From cb6b76d3b55bd6af97eac5907ba0cba5dfc5e5df Mon Sep 17 00:00:00 2001 From: Alex Haynes Date: Sun, 24 Mar 2019 15:08:36 -0400 Subject: [PATCH 001/131] added rpc.proto from go repo --- rpc.proto | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 rpc.proto diff --git a/rpc.proto b/rpc.proto new file mode 100644 index 0000000..2ae2ef2 --- /dev/null +++ b/rpc.proto @@ -0,0 +1,78 @@ +// Borrowed from https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto + +syntax = "proto2"; + +package pubsub.pb; + +message RPC { + repeated SubOpts subscriptions = 1; + repeated Message publish = 2; + + message SubOpts { + optional bool subscribe = 1; // subscribe or unsubcribe + optional string topicid = 2; + } + + optional ControlMessage control = 3; +} + +message Message { + optional bytes from = 1; + optional bytes data = 2; + optional bytes seqno = 3; + repeated string topicIDs = 4; + optional bytes signature = 5; + optional bytes key = 6; +} + +message ControlMessage { + repeated ControlIHave ihave = 1; + repeated ControlIWant iwant = 2; + repeated ControlGraft graft = 3; + repeated ControlPrune prune = 4; +} + +message ControlIHave { + optional string topicID = 1; + repeated string messageIDs = 2; +} + +message ControlIWant { + repeated string messageIDs = 1; +} + +message ControlGraft { + optional string topicID = 1; +} + +message ControlPrune { + optional string topicID = 1; +} + +message TopicDescriptor { + optional string name = 1; + optional AuthOpts auth = 2; + optional EncOpts enc = 3; + + message AuthOpts { + optional AuthMode mode = 1; + repeated bytes keys = 2; // root keys to trust + + enum AuthMode { + NONE = 0; // no authentication, anyone can publish + KEY = 1; // only messages signed by keys in the topic descriptor are accepted + WOT = 2; // web of trust, certificates can allow publisher set to grow + } + } + + message EncOpts { + optional EncMode mode = 1; + repeated bytes keyHashes = 2; // the hashes of the shared keys used (salted) + + enum EncMode { + NONE = 0; // no encryption, anyone can read + SHAREDKEY = 1; // messages are encrypted with shared key + WOT = 2; // web of trust, certificates can allow publisher set to grow + } + } +} \ No newline at end of file From 041e0fbe3453e1eb17143820dc48298adae44e61 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 09:41:38 -0400 Subject: [PATCH 002/131] add pubsub proto --- libp2p/pubsub/pb/rpc.proto | 76 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 libp2p/pubsub/pb/rpc.proto diff --git a/libp2p/pubsub/pb/rpc.proto b/libp2p/pubsub/pb/rpc.proto new file mode 100644 index 0000000..764b033 --- /dev/null +++ b/libp2p/pubsub/pb/rpc.proto @@ -0,0 +1,76 @@ +syntax = "proto2"; + +package pubsub.pb; + +message RPC { + repeated SubOpts subscriptions = 1; + repeated Message publish = 2; + + message SubOpts { + optional bool subscribe = 1; // subscribe or unsubcribe + optional string topicid = 2; + } + + optional ControlMessage control = 3; +} + +message Message { + optional bytes from = 1; + optional bytes data = 2; + optional bytes seqno = 3; + repeated string topicIDs = 4; + optional bytes signature = 5; + optional bytes key = 6; +} + +message ControlMessage { + repeated ControlIHave ihave = 1; + repeated ControlIWant iwant = 2; + repeated ControlGraft graft = 3; + repeated ControlPrune prune = 4; +} + +message ControlIHave { + optional string topicID = 1; + repeated string messageIDs = 2; +} + +message ControlIWant { + repeated string messageIDs = 1; +} + +message ControlGraft { + optional string topicID = 1; +} + +message ControlPrune { + optional string topicID = 1; +} + +message TopicDescriptor { + optional string name = 1; + optional AuthOpts auth = 2; + optional EncOpts enc = 3; + + message AuthOpts { + optional AuthMode mode = 1; + repeated bytes keys = 2; // root keys to trust + + enum AuthMode { + NONE = 0; // no authentication, anyone can publish + KEY = 1; // only messages signed by keys in the topic descriptor are accepted + WOT = 2; // web of trust, certificates can allow publisher set to grow + } + } + + message EncOpts { + optional EncMode mode = 1; + repeated bytes keyHashes = 2; // the hashes of the shared keys used (salted) + + enum EncMode { + NONE = 0; // no encryption, anyone can read + SHAREDKEY = 1; // messages are encrypted with shared key + WOT = 2; // web of trust, certificates can allow publisher set to grow + } + } +} \ No newline at end of file From 1e8d93fcf6777b62297c0d524a7dd1bae933ca20 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 09:55:14 -0400 Subject: [PATCH 003/131] add generated rpc code --- libp2p/pubsub/pb/rpc_pb2.py | 638 +++++++++++++++++++++++++++++++ libp2p/pubsub/pb/rpc_pb2_grpc.py | 3 + 2 files changed, 641 insertions(+) create mode 100644 libp2p/pubsub/pb/rpc_pb2.py create mode 100644 libp2p/pubsub/pb/rpc_pb2_grpc.py diff --git a/libp2p/pubsub/pb/rpc_pb2.py b/libp2p/pubsub/pb/rpc_pb2.py new file mode 100644 index 0000000..57035ed --- /dev/null +++ b/libp2p/pubsub/pb/rpc_pb2.py @@ -0,0 +1,638 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: rpc.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='rpc.proto', + package='pubsub.pb', + syntax='proto2', + serialized_options=None, + serialized_pb=_b('\n\trpc.proto\x12\tpubsub.pb\"\xb4\x01\n\x03RPC\x12-\n\rsubscriptions\x18\x01 \x03(\x0b\x32\x16.pubsub.pb.RPC.SubOpts\x12#\n\x07publish\x18\x02 \x03(\x0b\x32\x12.pubsub.pb.Message\x12*\n\x07\x63ontrol\x18\x03 \x01(\x0b\x32\x19.pubsub.pb.ControlMessage\x1a-\n\x07SubOpts\x12\x11\n\tsubscribe\x18\x01 \x01(\x08\x12\x0f\n\x07topicid\x18\x02 \x01(\t\"f\n\x07Message\x12\x0c\n\x04\x66rom\x18\x01 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\r\n\x05seqno\x18\x03 \x01(\x0c\x12\x10\n\x08topicIDs\x18\x04 \x03(\t\x12\x11\n\tsignature\x18\x05 \x01(\x0c\x12\x0b\n\x03key\x18\x06 \x01(\x0c\"\xb0\x01\n\x0e\x43ontrolMessage\x12&\n\x05ihave\x18\x01 \x03(\x0b\x32\x17.pubsub.pb.ControlIHave\x12&\n\x05iwant\x18\x02 \x03(\x0b\x32\x17.pubsub.pb.ControlIWant\x12&\n\x05graft\x18\x03 \x03(\x0b\x32\x17.pubsub.pb.ControlGraft\x12&\n\x05prune\x18\x04 \x03(\x0b\x32\x17.pubsub.pb.ControlPrune\"3\n\x0c\x43ontrolIHave\x12\x0f\n\x07topicID\x18\x01 \x01(\t\x12\x12\n\nmessageIDs\x18\x02 \x03(\t\"\"\n\x0c\x43ontrolIWant\x12\x12\n\nmessageIDs\x18\x01 \x03(\t\"\x1f\n\x0c\x43ontrolGraft\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x1f\n\x0c\x43ontrolPrune\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x87\x03\n\x0fTopicDescriptor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x31\n\x04\x61uth\x18\x02 \x01(\x0b\x32#.pubsub.pb.TopicDescriptor.AuthOpts\x12/\n\x03\x65nc\x18\x03 \x01(\x0b\x32\".pubsub.pb.TopicDescriptor.EncOpts\x1a|\n\x08\x41uthOpts\x12:\n\x04mode\x18\x01 \x01(\x0e\x32,.pubsub.pb.TopicDescriptor.AuthOpts.AuthMode\x12\x0c\n\x04keys\x18\x02 \x03(\x0c\"&\n\x08\x41uthMode\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03KEY\x10\x01\x12\x07\n\x03WOT\x10\x02\x1a\x83\x01\n\x07\x45ncOpts\x12\x38\n\x04mode\x18\x01 \x01(\x0e\x32*.pubsub.pb.TopicDescriptor.EncOpts.EncMode\x12\x11\n\tkeyHashes\x18\x02 \x03(\x0c\"+\n\x07\x45ncMode\x12\x08\n\x04NONE\x10\x00\x12\r\n\tSHAREDKEY\x10\x01\x12\x07\n\x03WOT\x10\x02') +) + + + +_TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE = _descriptor.EnumDescriptor( + name='AuthMode', + full_name='pubsub.pb.TopicDescriptor.AuthOpts.AuthMode', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='NONE', index=0, number=0, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='KEY', index=1, number=1, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='WOT', index=2, number=2, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=865, + serialized_end=903, +) +_sym_db.RegisterEnumDescriptor(_TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE) + +_TOPICDESCRIPTOR_ENCOPTS_ENCMODE = _descriptor.EnumDescriptor( + name='EncMode', + full_name='pubsub.pb.TopicDescriptor.EncOpts.EncMode', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='NONE', index=0, number=0, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='SHAREDKEY', index=1, number=1, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='WOT', index=2, number=2, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=994, + serialized_end=1037, +) +_sym_db.RegisterEnumDescriptor(_TOPICDESCRIPTOR_ENCOPTS_ENCMODE) + + +_RPC_SUBOPTS = _descriptor.Descriptor( + name='SubOpts', + full_name='pubsub.pb.RPC.SubOpts', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='subscribe', full_name='pubsub.pb.RPC.SubOpts.subscribe', index=0, + number=1, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='topicid', full_name='pubsub.pb.RPC.SubOpts.topicid', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=160, + serialized_end=205, +) + +_RPC = _descriptor.Descriptor( + name='RPC', + full_name='pubsub.pb.RPC', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='subscriptions', full_name='pubsub.pb.RPC.subscriptions', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='publish', full_name='pubsub.pb.RPC.publish', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='control', full_name='pubsub.pb.RPC.control', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_RPC_SUBOPTS, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=25, + serialized_end=205, +) + + +_MESSAGE = _descriptor.Descriptor( + name='Message', + full_name='pubsub.pb.Message', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='from', full_name='pubsub.pb.Message.from', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='data', full_name='pubsub.pb.Message.data', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='seqno', full_name='pubsub.pb.Message.seqno', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='topicIDs', full_name='pubsub.pb.Message.topicIDs', index=3, + number=4, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='signature', full_name='pubsub.pb.Message.signature', index=4, + number=5, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='key', full_name='pubsub.pb.Message.key', index=5, + number=6, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=207, + serialized_end=309, +) + + +_CONTROLMESSAGE = _descriptor.Descriptor( + name='ControlMessage', + full_name='pubsub.pb.ControlMessage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='ihave', full_name='pubsub.pb.ControlMessage.ihave', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='iwant', full_name='pubsub.pb.ControlMessage.iwant', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='graft', full_name='pubsub.pb.ControlMessage.graft', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='prune', full_name='pubsub.pb.ControlMessage.prune', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=312, + serialized_end=488, +) + + +_CONTROLIHAVE = _descriptor.Descriptor( + name='ControlIHave', + full_name='pubsub.pb.ControlIHave', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='topicID', full_name='pubsub.pb.ControlIHave.topicID', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='messageIDs', full_name='pubsub.pb.ControlIHave.messageIDs', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=490, + serialized_end=541, +) + + +_CONTROLIWANT = _descriptor.Descriptor( + name='ControlIWant', + full_name='pubsub.pb.ControlIWant', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='messageIDs', full_name='pubsub.pb.ControlIWant.messageIDs', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=543, + serialized_end=577, +) + + +_CONTROLGRAFT = _descriptor.Descriptor( + name='ControlGraft', + full_name='pubsub.pb.ControlGraft', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='topicID', full_name='pubsub.pb.ControlGraft.topicID', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=579, + serialized_end=610, +) + + +_CONTROLPRUNE = _descriptor.Descriptor( + name='ControlPrune', + full_name='pubsub.pb.ControlPrune', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='topicID', full_name='pubsub.pb.ControlPrune.topicID', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=612, + serialized_end=643, +) + + +_TOPICDESCRIPTOR_AUTHOPTS = _descriptor.Descriptor( + name='AuthOpts', + full_name='pubsub.pb.TopicDescriptor.AuthOpts', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='mode', full_name='pubsub.pb.TopicDescriptor.AuthOpts.mode', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='keys', full_name='pubsub.pb.TopicDescriptor.AuthOpts.keys', index=1, + number=2, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE, + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=779, + serialized_end=903, +) + +_TOPICDESCRIPTOR_ENCOPTS = _descriptor.Descriptor( + name='EncOpts', + full_name='pubsub.pb.TopicDescriptor.EncOpts', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='mode', full_name='pubsub.pb.TopicDescriptor.EncOpts.mode', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='keyHashes', full_name='pubsub.pb.TopicDescriptor.EncOpts.keyHashes', index=1, + number=2, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _TOPICDESCRIPTOR_ENCOPTS_ENCMODE, + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=906, + serialized_end=1037, +) + +_TOPICDESCRIPTOR = _descriptor.Descriptor( + name='TopicDescriptor', + full_name='pubsub.pb.TopicDescriptor', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='pubsub.pb.TopicDescriptor.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='auth', full_name='pubsub.pb.TopicDescriptor.auth', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='enc', full_name='pubsub.pb.TopicDescriptor.enc', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_TOPICDESCRIPTOR_AUTHOPTS, _TOPICDESCRIPTOR_ENCOPTS, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=646, + serialized_end=1037, +) + +_RPC_SUBOPTS.containing_type = _RPC +_RPC.fields_by_name['subscriptions'].message_type = _RPC_SUBOPTS +_RPC.fields_by_name['publish'].message_type = _MESSAGE +_RPC.fields_by_name['control'].message_type = _CONTROLMESSAGE +_CONTROLMESSAGE.fields_by_name['ihave'].message_type = _CONTROLIHAVE +_CONTROLMESSAGE.fields_by_name['iwant'].message_type = _CONTROLIWANT +_CONTROLMESSAGE.fields_by_name['graft'].message_type = _CONTROLGRAFT +_CONTROLMESSAGE.fields_by_name['prune'].message_type = _CONTROLPRUNE +_TOPICDESCRIPTOR_AUTHOPTS.fields_by_name['mode'].enum_type = _TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE +_TOPICDESCRIPTOR_AUTHOPTS.containing_type = _TOPICDESCRIPTOR +_TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE.containing_type = _TOPICDESCRIPTOR_AUTHOPTS +_TOPICDESCRIPTOR_ENCOPTS.fields_by_name['mode'].enum_type = _TOPICDESCRIPTOR_ENCOPTS_ENCMODE +_TOPICDESCRIPTOR_ENCOPTS.containing_type = _TOPICDESCRIPTOR +_TOPICDESCRIPTOR_ENCOPTS_ENCMODE.containing_type = _TOPICDESCRIPTOR_ENCOPTS +_TOPICDESCRIPTOR.fields_by_name['auth'].message_type = _TOPICDESCRIPTOR_AUTHOPTS +_TOPICDESCRIPTOR.fields_by_name['enc'].message_type = _TOPICDESCRIPTOR_ENCOPTS +DESCRIPTOR.message_types_by_name['RPC'] = _RPC +DESCRIPTOR.message_types_by_name['Message'] = _MESSAGE +DESCRIPTOR.message_types_by_name['ControlMessage'] = _CONTROLMESSAGE +DESCRIPTOR.message_types_by_name['ControlIHave'] = _CONTROLIHAVE +DESCRIPTOR.message_types_by_name['ControlIWant'] = _CONTROLIWANT +DESCRIPTOR.message_types_by_name['ControlGraft'] = _CONTROLGRAFT +DESCRIPTOR.message_types_by_name['ControlPrune'] = _CONTROLPRUNE +DESCRIPTOR.message_types_by_name['TopicDescriptor'] = _TOPICDESCRIPTOR +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +RPC = _reflection.GeneratedProtocolMessageType('RPC', (_message.Message,), dict( + + SubOpts = _reflection.GeneratedProtocolMessageType('SubOpts', (_message.Message,), dict( + DESCRIPTOR = _RPC_SUBOPTS, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.RPC.SubOpts) + )) + , + DESCRIPTOR = _RPC, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.RPC) + )) +_sym_db.RegisterMessage(RPC) +_sym_db.RegisterMessage(RPC.SubOpts) + +Message = _reflection.GeneratedProtocolMessageType('Message', (_message.Message,), dict( + DESCRIPTOR = _MESSAGE, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.Message) + )) +_sym_db.RegisterMessage(Message) + +ControlMessage = _reflection.GeneratedProtocolMessageType('ControlMessage', (_message.Message,), dict( + DESCRIPTOR = _CONTROLMESSAGE, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlMessage) + )) +_sym_db.RegisterMessage(ControlMessage) + +ControlIHave = _reflection.GeneratedProtocolMessageType('ControlIHave', (_message.Message,), dict( + DESCRIPTOR = _CONTROLIHAVE, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlIHave) + )) +_sym_db.RegisterMessage(ControlIHave) + +ControlIWant = _reflection.GeneratedProtocolMessageType('ControlIWant', (_message.Message,), dict( + DESCRIPTOR = _CONTROLIWANT, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlIWant) + )) +_sym_db.RegisterMessage(ControlIWant) + +ControlGraft = _reflection.GeneratedProtocolMessageType('ControlGraft', (_message.Message,), dict( + DESCRIPTOR = _CONTROLGRAFT, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlGraft) + )) +_sym_db.RegisterMessage(ControlGraft) + +ControlPrune = _reflection.GeneratedProtocolMessageType('ControlPrune', (_message.Message,), dict( + DESCRIPTOR = _CONTROLPRUNE, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlPrune) + )) +_sym_db.RegisterMessage(ControlPrune) + +TopicDescriptor = _reflection.GeneratedProtocolMessageType('TopicDescriptor', (_message.Message,), dict( + + AuthOpts = _reflection.GeneratedProtocolMessageType('AuthOpts', (_message.Message,), dict( + DESCRIPTOR = _TOPICDESCRIPTOR_AUTHOPTS, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.TopicDescriptor.AuthOpts) + )) + , + + EncOpts = _reflection.GeneratedProtocolMessageType('EncOpts', (_message.Message,), dict( + DESCRIPTOR = _TOPICDESCRIPTOR_ENCOPTS, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.TopicDescriptor.EncOpts) + )) + , + DESCRIPTOR = _TOPICDESCRIPTOR, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.TopicDescriptor) + )) +_sym_db.RegisterMessage(TopicDescriptor) +_sym_db.RegisterMessage(TopicDescriptor.AuthOpts) +_sym_db.RegisterMessage(TopicDescriptor.EncOpts) + + +# @@protoc_insertion_point(module_scope) diff --git a/libp2p/pubsub/pb/rpc_pb2_grpc.py b/libp2p/pubsub/pb/rpc_pb2_grpc.py new file mode 100644 index 0000000..a894352 --- /dev/null +++ b/libp2p/pubsub/pb/rpc_pb2_grpc.py @@ -0,0 +1,3 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + From 81d121a0296739902ece55711ecc351fea7fad12 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 15:23:56 -0400 Subject: [PATCH 004/131] update from to from_id in proto --- libp2p/pubsub/pb/rpc.proto | 2 +- libp2p/pubsub/pb/rpc_pb2.py | 46 ++++++++++++++++++------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/libp2p/pubsub/pb/rpc.proto b/libp2p/pubsub/pb/rpc.proto index 764b033..f07da20 100644 --- a/libp2p/pubsub/pb/rpc.proto +++ b/libp2p/pubsub/pb/rpc.proto @@ -15,7 +15,7 @@ message RPC { } message Message { - optional bytes from = 1; + optional bytes from_id = 1; optional bytes data = 2; optional bytes seqno = 3; repeated string topicIDs = 4; diff --git a/libp2p/pubsub/pb/rpc_pb2.py b/libp2p/pubsub/pb/rpc_pb2.py index 57035ed..d315782 100644 --- a/libp2p/pubsub/pb/rpc_pb2.py +++ b/libp2p/pubsub/pb/rpc_pb2.py @@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( package='pubsub.pb', syntax='proto2', serialized_options=None, - serialized_pb=_b('\n\trpc.proto\x12\tpubsub.pb\"\xb4\x01\n\x03RPC\x12-\n\rsubscriptions\x18\x01 \x03(\x0b\x32\x16.pubsub.pb.RPC.SubOpts\x12#\n\x07publish\x18\x02 \x03(\x0b\x32\x12.pubsub.pb.Message\x12*\n\x07\x63ontrol\x18\x03 \x01(\x0b\x32\x19.pubsub.pb.ControlMessage\x1a-\n\x07SubOpts\x12\x11\n\tsubscribe\x18\x01 \x01(\x08\x12\x0f\n\x07topicid\x18\x02 \x01(\t\"f\n\x07Message\x12\x0c\n\x04\x66rom\x18\x01 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\r\n\x05seqno\x18\x03 \x01(\x0c\x12\x10\n\x08topicIDs\x18\x04 \x03(\t\x12\x11\n\tsignature\x18\x05 \x01(\x0c\x12\x0b\n\x03key\x18\x06 \x01(\x0c\"\xb0\x01\n\x0e\x43ontrolMessage\x12&\n\x05ihave\x18\x01 \x03(\x0b\x32\x17.pubsub.pb.ControlIHave\x12&\n\x05iwant\x18\x02 \x03(\x0b\x32\x17.pubsub.pb.ControlIWant\x12&\n\x05graft\x18\x03 \x03(\x0b\x32\x17.pubsub.pb.ControlGraft\x12&\n\x05prune\x18\x04 \x03(\x0b\x32\x17.pubsub.pb.ControlPrune\"3\n\x0c\x43ontrolIHave\x12\x0f\n\x07topicID\x18\x01 \x01(\t\x12\x12\n\nmessageIDs\x18\x02 \x03(\t\"\"\n\x0c\x43ontrolIWant\x12\x12\n\nmessageIDs\x18\x01 \x03(\t\"\x1f\n\x0c\x43ontrolGraft\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x1f\n\x0c\x43ontrolPrune\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x87\x03\n\x0fTopicDescriptor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x31\n\x04\x61uth\x18\x02 \x01(\x0b\x32#.pubsub.pb.TopicDescriptor.AuthOpts\x12/\n\x03\x65nc\x18\x03 \x01(\x0b\x32\".pubsub.pb.TopicDescriptor.EncOpts\x1a|\n\x08\x41uthOpts\x12:\n\x04mode\x18\x01 \x01(\x0e\x32,.pubsub.pb.TopicDescriptor.AuthOpts.AuthMode\x12\x0c\n\x04keys\x18\x02 \x03(\x0c\"&\n\x08\x41uthMode\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03KEY\x10\x01\x12\x07\n\x03WOT\x10\x02\x1a\x83\x01\n\x07\x45ncOpts\x12\x38\n\x04mode\x18\x01 \x01(\x0e\x32*.pubsub.pb.TopicDescriptor.EncOpts.EncMode\x12\x11\n\tkeyHashes\x18\x02 \x03(\x0c\"+\n\x07\x45ncMode\x12\x08\n\x04NONE\x10\x00\x12\r\n\tSHAREDKEY\x10\x01\x12\x07\n\x03WOT\x10\x02') + serialized_pb=_b('\n\trpc.proto\x12\tpubsub.pb\"\xb4\x01\n\x03RPC\x12-\n\rsubscriptions\x18\x01 \x03(\x0b\x32\x16.pubsub.pb.RPC.SubOpts\x12#\n\x07publish\x18\x02 \x03(\x0b\x32\x12.pubsub.pb.Message\x12*\n\x07\x63ontrol\x18\x03 \x01(\x0b\x32\x19.pubsub.pb.ControlMessage\x1a-\n\x07SubOpts\x12\x11\n\tsubscribe\x18\x01 \x01(\x08\x12\x0f\n\x07topicid\x18\x02 \x01(\t\"i\n\x07Message\x12\x0f\n\x07\x66rom_id\x18\x01 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\r\n\x05seqno\x18\x03 \x01(\x0c\x12\x10\n\x08topicIDs\x18\x04 \x03(\t\x12\x11\n\tsignature\x18\x05 \x01(\x0c\x12\x0b\n\x03key\x18\x06 \x01(\x0c\"\xb0\x01\n\x0e\x43ontrolMessage\x12&\n\x05ihave\x18\x01 \x03(\x0b\x32\x17.pubsub.pb.ControlIHave\x12&\n\x05iwant\x18\x02 \x03(\x0b\x32\x17.pubsub.pb.ControlIWant\x12&\n\x05graft\x18\x03 \x03(\x0b\x32\x17.pubsub.pb.ControlGraft\x12&\n\x05prune\x18\x04 \x03(\x0b\x32\x17.pubsub.pb.ControlPrune\"3\n\x0c\x43ontrolIHave\x12\x0f\n\x07topicID\x18\x01 \x01(\t\x12\x12\n\nmessageIDs\x18\x02 \x03(\t\"\"\n\x0c\x43ontrolIWant\x12\x12\n\nmessageIDs\x18\x01 \x03(\t\"\x1f\n\x0c\x43ontrolGraft\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x1f\n\x0c\x43ontrolPrune\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x87\x03\n\x0fTopicDescriptor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x31\n\x04\x61uth\x18\x02 \x01(\x0b\x32#.pubsub.pb.TopicDescriptor.AuthOpts\x12/\n\x03\x65nc\x18\x03 \x01(\x0b\x32\".pubsub.pb.TopicDescriptor.EncOpts\x1a|\n\x08\x41uthOpts\x12:\n\x04mode\x18\x01 \x01(\x0e\x32,.pubsub.pb.TopicDescriptor.AuthOpts.AuthMode\x12\x0c\n\x04keys\x18\x02 \x03(\x0c\"&\n\x08\x41uthMode\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03KEY\x10\x01\x12\x07\n\x03WOT\x10\x02\x1a\x83\x01\n\x07\x45ncOpts\x12\x38\n\x04mode\x18\x01 \x01(\x0e\x32*.pubsub.pb.TopicDescriptor.EncOpts.EncMode\x12\x11\n\tkeyHashes\x18\x02 \x03(\x0c\"+\n\x07\x45ncMode\x12\x08\n\x04NONE\x10\x00\x12\r\n\tSHAREDKEY\x10\x01\x12\x07\n\x03WOT\x10\x02') ) @@ -45,8 +45,8 @@ _TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE = _descriptor.EnumDescriptor( ], containing_type=None, serialized_options=None, - serialized_start=865, - serialized_end=903, + serialized_start=868, + serialized_end=906, ) _sym_db.RegisterEnumDescriptor(_TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE) @@ -71,8 +71,8 @@ _TOPICDESCRIPTOR_ENCOPTS_ENCMODE = _descriptor.EnumDescriptor( ], containing_type=None, serialized_options=None, - serialized_start=994, - serialized_end=1037, + serialized_start=997, + serialized_end=1040, ) _sym_db.RegisterEnumDescriptor(_TOPICDESCRIPTOR_ENCOPTS_ENCMODE) @@ -167,7 +167,7 @@ _MESSAGE = _descriptor.Descriptor( containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='from', full_name='pubsub.pb.Message.from', index=0, + name='from_id', full_name='pubsub.pb.Message.from_id', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, @@ -221,7 +221,7 @@ _MESSAGE = _descriptor.Descriptor( oneofs=[ ], serialized_start=207, - serialized_end=309, + serialized_end=312, ) @@ -272,8 +272,8 @@ _CONTROLMESSAGE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=312, - serialized_end=488, + serialized_start=315, + serialized_end=491, ) @@ -310,8 +310,8 @@ _CONTROLIHAVE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=490, - serialized_end=541, + serialized_start=493, + serialized_end=544, ) @@ -341,8 +341,8 @@ _CONTROLIWANT = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=543, - serialized_end=577, + serialized_start=546, + serialized_end=580, ) @@ -372,8 +372,8 @@ _CONTROLGRAFT = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=579, - serialized_end=610, + serialized_start=582, + serialized_end=613, ) @@ -403,8 +403,8 @@ _CONTROLPRUNE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=612, - serialized_end=643, + serialized_start=615, + serialized_end=646, ) @@ -442,8 +442,8 @@ _TOPICDESCRIPTOR_AUTHOPTS = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=779, - serialized_end=903, + serialized_start=782, + serialized_end=906, ) _TOPICDESCRIPTOR_ENCOPTS = _descriptor.Descriptor( @@ -480,8 +480,8 @@ _TOPICDESCRIPTOR_ENCOPTS = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=906, - serialized_end=1037, + serialized_start=909, + serialized_end=1040, ) _TOPICDESCRIPTOR = _descriptor.Descriptor( @@ -524,8 +524,8 @@ _TOPICDESCRIPTOR = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=646, - serialized_end=1037, + serialized_start=649, + serialized_end=1040, ) _RPC_SUBOPTS.containing_type = _RPC From 3cfeccaf17b1b6eb43ea052bb646b573afa3680a Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 15:25:33 -0400 Subject: [PATCH 005/131] rewrote get_hello_packet --- libp2p/pubsub/pubsub.py | 74 +++++++++++++---------------------------- 1 file changed, 24 insertions(+), 50 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 9bd072f..17ecdd5 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -1,5 +1,7 @@ import asyncio +from .pb import rpc_pb2_grpc +from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee from .message import MessageSub from .message import create_message_talk, create_message_sub @@ -7,37 +9,6 @@ from. message import generate_message_id class Pubsub(): - """ - For now, because I'm on a plane and don't have access to the go repo/protobuf stuff, - this is going to be the message format for the two types: subscription and talk - subscription indicates subscribing or unsubscribing from a topic - talk is sending a message on topic(s) - subscription format: - subscription - 'from' - :'topicid' - :'topicid' - ... - Ex. - subscription - msg_sender_peer_id - origin_peer_id - sub:topic1 - sub:topic2 - unsub:fav_topic - talk format: - talk - 'from' - 'origin' - [topic_ids comma-delimited] - 'data' - Ex. - talk - msg_sender_peer_id - origin_peer_id - topic1,topics_are_cool,foo - I like tacos - """ # pylint: disable=too-many-instance-attributes def __init__(self, host, router, my_id): @@ -90,34 +61,37 @@ class Pubsub(): """ Generate subscription message with all topics we are subscribed to """ - subs_map = {} - for topic in self.my_topics: - subs_map[topic] = True - sub_msg = MessageSub( - str(self.host.get_id()),\ - str(self.host.get_id()), subs_map, generate_message_id()\ - ) - return sub_msg.to_str() + packet = rpc_pb2.RPC() + message = rpc_pb2.Message( + from_id=str(self.host.get_id()).encode('utf-8'), + seqno=str(generate_message_id()).encode('utf-8') + ) + packet.publish.extend([message]) + for topic_id in self.my_topics: + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe=True, topicid=topic_id)]) + + return packet.SerializeToString() async def continously_read_stream(self, stream): """ Read from input stream in an infinite loop. Process messages from other nodes, which for now are considered MessageTalk and MessageSub messages. - TODO: Handle RPC messages instead of my Aspyn's own custom message format :param stream: stream to continously read from """ while True: - incoming = (await stream.read()).decode() - msg_comps = incoming.split('\n') - msg_type = msg_comps[0] + incoming = (await stream.read()) + rpc_incoming = rpc_pb2.RPC() + rpc_incoming.ParseFromString(incoming) + print (rpc_incoming.publish) - msg_sender = msg_comps[1] + # msg_type = msg_comps[0] + + msg_sender = rpc_incoming.publish.from_id # msg_origin = msg_comps[2] - msg_id = msg_comps[3] - print("HIT ME1") + msg_id = rpc_incoming.publish.seqno if msg_id not in self.seen_messages: - print("HIT ME") # Do stuff with incoming unseen message should_publish = True if msg_type == "subscription": @@ -161,7 +135,7 @@ class Pubsub(): # Send hello packet hello = self.get_hello_packet() - await stream.write(hello.encode()) + await stream.write(hello) # Pass stream off to stream reader asyncio.ensure_future(self.continously_read_stream(stream)) @@ -187,7 +161,7 @@ class Pubsub(): # Send hello packet hello = self.get_hello_packet() - await stream.write(hello.encode()) + await stream.write(hello) # Pass stream off to stream reader asyncio.ensure_future(self.continously_read_stream(stream)) @@ -291,4 +265,4 @@ class Pubsub(): stream = self.peers[peer] # Write message to stream - await stream.write(encoded_msg) + await stream.write(encoded_msg) \ No newline at end of file From a81628f99d605db35ecb8bca6c778970d824874f Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 15:42:55 -0400 Subject: [PATCH 006/131] update dependencies --- requirements_dev.txt | 2 ++ setup.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/requirements_dev.txt b/requirements_dev.txt index 4bf18f6..e3d8c0e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -3,3 +3,5 @@ codecov pytest-cov pytest-asyncio pylint +grpcio +grpcio-tools diff --git a/setup.py b/setup.py index abaa1c7..c44a276 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,8 @@ setuptools.setup( "base58", "pymultihash", "multiaddr", + "grpcio", + "grpcio-tools" ], packages=["libp2p"], zip_safe=False, From bf17f424b38bc5704f99996e35123564c7466c4c Mon Sep 17 00:00:00 2001 From: Alex Haynes Date: Fri, 29 Mar 2019 16:23:30 -0400 Subject: [PATCH 007/131] RPC conversion progress --- libp2p/pubsub/floodsub.py | 41 ++++++++++++++----- libp2p/pubsub/pubsub.py | 85 +++++++++++++++++++++++++++------------ 2 files changed, 91 insertions(+), 35 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 68e542c..1cea17e 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -1,4 +1,6 @@ from .pubsub_router_interface import IPubsubRouter +from .pb import rpc_pb2 +from .message import MessageSub, MessageTalk from .message import create_message_talk class FloodSub(IPubsubRouter): @@ -56,28 +58,49 @@ class FloodSub(IPubsubRouter): """ # Encode message - encoded_msg = message.encode() + # encoded_msg = message.encode() + + if isinstance(message, str): + msg_talk = create_message_talk(message) + message = rpc_pb2.Message( + from_id=str(msg_talk.origin_id).encode('utf-8'), + seqno=str(msg_talk.message_id).encode('utf-8'), + topicIDs=msg_talk.topics, + data=msg_talk.data.encode() + + ) + packet = rpc_pb2.RPC() + print("YEET") + print(type(message)) + packet.publish.extend([message]) + + # Get message sender, origin, and topics - msg_talk = create_message_talk(message) + # msg_talk = create_message_talk(message) msg_sender = str(sender_peer_id) - msg_origin = msg_talk.origin_id - topics = msg_talk.topics + # msg_origin = msg_talk.origin_id + # topics = msg_talk.topics # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message - if msg_sender == msg_origin and msg_sender == str(self.pubsub.host.get_id()): - await self.pubsub.handle_talk(message) + if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): + old_format = MessageTalk(sender_peer_id, + message.from_id, + message.topicIDs, + message.data, + message.seqno) + await self.pubsub.handle_talk(old_format) # Deliver to self and peers - for topic in topics: + for topic in message.topicIDs: if topic in self.pubsub.peer_topics: for peer_id_in_topic in self.pubsub.peer_topics[topic]: # Forward to all known peers in the topic that are not the # message sender and are not the message origin - if peer_id_in_topic not in (msg_sender, msg_origin): + if peer_id_in_topic not in (msg_sender, message.from_id): stream = self.pubsub.peers[peer_id_in_topic] - await stream.write(encoded_msg) + await stream.write(packet.SerializeToString()) else: # Implies publish did not write print("publish did not write") diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 17ecdd5..b1ee363 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -3,7 +3,7 @@ import asyncio from .pb import rpc_pb2_grpc from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee -from .message import MessageSub +from .message import MessageSub, MessageTalk from .message import create_message_talk, create_message_sub from. message import generate_message_id @@ -61,6 +61,7 @@ class Pubsub(): """ Generate subscription message with all topics we are subscribed to """ + print("MAKING HELLO PACKET") packet = rpc_pb2.RPC() message = rpc_pb2.Message( from_id=str(self.host.get_id()).encode('utf-8'), @@ -80,43 +81,74 @@ class Pubsub(): and MessageSub messages. :param stream: stream to continously read from """ + + # TODO check on types here + peer_id = stream.mplex_conn.peer_id + while True: incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() + print("JUST GOT STRING") + print(incoming) rpc_incoming.ParseFromString(incoming) - print (rpc_incoming.publish) + print("JUST GOT") + print (rpc_incoming) + + if hasattr(rpc_incoming, 'publish'): + # deal with "talk messages" + for msg in rpc_incoming.publish: + # TODO check what from_id is in go? is it origin or peer + old_format = MessageTalk(peer_id, + msg.from_id, + msg.topicIDs, + msg.data, + msg.seqno) + self.seen_messages.append(msg.seqno) + await self.handle_talk(old_format) + await self.router.publish(peer_id, msg) + + if hasattr(rpc_incoming, 'subscriptions'): + # deal with "subscription messages" + subs_map = {} + for msg in rpc_incoming.subscriptions: + if msg.subscribe: + subs_map[msg.topic_id] = "sub" + else: + subs_map[msg.topic_id] = "unsub" + old_format = MessageSub(peer_id, peer_id, subs_map, generate_message_id()) + self.handle_subscription(old_format) # msg_type = msg_comps[0] - msg_sender = rpc_incoming.publish.from_id - # msg_origin = msg_comps[2] - msg_id = rpc_incoming.publish.seqno - if msg_id not in self.seen_messages: - # Do stuff with incoming unseen message - should_publish = True - if msg_type == "subscription": - self.handle_subscription(incoming) + # msg_sender = rpc_incoming.publish.from_id + # # msg_origin = msg_comps[2] + # msg_id = rpc_incoming.publish.seqno + # if msg_id not in self.seen_messages: + # # Do stuff with incoming unseen message + # should_publish = True + # if msg_type == "subscription": + # self.handle_subscription(incoming) - # We don't need to relay the subscription to our - # peers because a given node only needs its peers - # to know that it is subscribed to the topic (doesn't - # need everyone to know) - should_publish = False - elif msg_type == "talk": - await self.handle_talk(incoming) + # # We don't need to relay the subscription to our + # # peers because a given node only needs its peers + # # to know that it is subscribed to the topic (doesn't + # # need everyone to know) + # should_publish = False + # elif msg_type == "talk": + # await self.handle_talk(incoming) # Add message id to seen - self.seen_messages.append(msg_id) + # self.seen_messages.append(msg_id) # Publish message using router's publish - if should_publish: - msg = create_message_talk(incoming) + # if should_publish: + # msg = create_message_talk(incoming) - # Adjust raw_msg to that the message sender - # is now our peer_id - msg.from_id = str(self.host.get_id()) + # # Adjust raw_msg to that the message sender + # # is now our peer_id + # msg.from_id = str(self.host.get_id()) - await self.router.publish(msg_sender, msg.to_str()) + # await self.router.publish(msg_sender, msg.to_str()) # Force context switch await asyncio.sleep(0) @@ -135,6 +167,7 @@ class Pubsub(): # Send hello packet hello = self.get_hello_packet() + print(hello) await stream.write(hello) # Pass stream off to stream reader asyncio.ensure_future(self.continously_read_stream(stream)) @@ -176,7 +209,7 @@ class Pubsub(): defined in the subscription message :param subscription: raw data constituting a subscription message """ - sub_msg = create_message_sub(subscription) + sub_msg = subscription if sub_msg.subs_map: print("handle_subscription my_id: " + self.my_id + ", subber: " + sub_msg.origin_id) for topic_id in sub_msg.subs_map: @@ -198,7 +231,7 @@ class Pubsub(): custom message that is published on a given topic(s) :param talk: raw data constituting a talk message """ - msg = create_message_talk(talk) + msg = talk # Check if this message has any topics that we are subscribed to for topic in msg.topics: From aec783b843ccd2cea79bee289953d2cea0d77498 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sat, 30 Mar 2019 17:59:08 -0400 Subject: [PATCH 008/131] reworked subscribe unsubsrcibe --- libp2p/pubsub/pubsub.py | 113 +++++++++++++++------------------------- 1 file changed, 43 insertions(+), 70 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index b1ee363..85abacb 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -61,7 +61,7 @@ class Pubsub(): """ Generate subscription message with all topics we are subscribed to """ - print("MAKING HELLO PACKET") + packet = rpc_pb2.RPC() message = rpc_pb2.Message( from_id=str(self.host.get_id()).encode('utf-8'), @@ -74,7 +74,7 @@ class Pubsub(): return packet.SerializeToString() - async def continously_read_stream(self, stream): + async def continuously_read_stream(self, stream): """ Read from input stream in an infinite loop. Process messages from other nodes, which for now are considered MessageTalk @@ -88,16 +88,12 @@ class Pubsub(): while True: incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() - print("JUST GOT STRING") - print(incoming) rpc_incoming.ParseFromString(incoming) - print("JUST GOT") + print ("CONTINUOUSLY") print (rpc_incoming) - - if hasattr(rpc_incoming, 'publish'): + if rpc_incoming.publish: # deal with "talk messages" for msg in rpc_incoming.publish: - # TODO check what from_id is in go? is it origin or peer old_format = MessageTalk(peer_id, msg.from_id, msg.topicIDs, @@ -107,48 +103,16 @@ class Pubsub(): await self.handle_talk(old_format) await self.router.publish(peer_id, msg) - if hasattr(rpc_incoming, 'subscriptions'): + if rpc_incoming.subscriptions: # deal with "subscription messages" subs_map = {} for msg in rpc_incoming.subscriptions: if msg.subscribe: - subs_map[msg.topic_id] = "sub" + subs_map[msg.topicid] = "sub" else: - subs_map[msg.topic_id] = "unsub" - old_format = MessageSub(peer_id, peer_id, subs_map, generate_message_id()) - self.handle_subscription(old_format) + subs_map[msg.topicid] = "unsub" - # msg_type = msg_comps[0] - - # msg_sender = rpc_incoming.publish.from_id - # # msg_origin = msg_comps[2] - # msg_id = rpc_incoming.publish.seqno - # if msg_id not in self.seen_messages: - # # Do stuff with incoming unseen message - # should_publish = True - # if msg_type == "subscription": - # self.handle_subscription(incoming) - - # # We don't need to relay the subscription to our - # # peers because a given node only needs its peers - # # to know that it is subscribed to the topic (doesn't - # # need everyone to know) - # should_publish = False - # elif msg_type == "talk": - # await self.handle_talk(incoming) - - # Add message id to seen - # self.seen_messages.append(msg_id) - - # Publish message using router's publish - # if should_publish: - # msg = create_message_talk(incoming) - - # # Adjust raw_msg to that the message sender - # # is now our peer_id - # msg.from_id = str(self.host.get_id()) - - # await self.router.publish(msg_sender, msg.to_str()) + self.handle_subscription(rpc_incoming) # Force context switch await asyncio.sleep(0) @@ -167,10 +131,10 @@ class Pubsub(): # Send hello packet hello = self.get_hello_packet() - print(hello) + await stream.write(hello) # Pass stream off to stream reader - asyncio.ensure_future(self.continously_read_stream(stream)) + asyncio.ensure_future(self.continuously_read_stream(stream)) async def handle_peer_queue(self): """ @@ -197,30 +161,30 @@ class Pubsub(): await stream.write(hello) # Pass stream off to stream reader - asyncio.ensure_future(self.continously_read_stream(stream)) + asyncio.ensure_future(self.continuously_read_stream(stream)) # Force context switch await asyncio.sleep(0) - def handle_subscription(self, subscription): + def handle_subscription(self, rpc_message): """ Handle an incoming subscription message from a peer. Update internal mapping to mark the peer as subscribed or unsubscribed to topics as defined in the subscription message :param subscription: raw data constituting a subscription message """ - sub_msg = subscription - if sub_msg.subs_map: - print("handle_subscription my_id: " + self.my_id + ", subber: " + sub_msg.origin_id) - for topic_id in sub_msg.subs_map: + for sub_msg in rpc_message.subscriptions: + # Look at each subscription in the msg individually - if sub_msg.subs_map[topic_id]: - if topic_id not in self.peer_topics: + if sub_msg.subscribe: + origin_id = rpc_message.publish[0].from_id + + if sub_msg.topicid not in self.peer_topics: # Create topic list if it did not yet exist - self.peer_topics[topic_id] = [sub_msg.origin_id] - elif sub_msg.origin_id not in self.peer_topics[topic_id]: + self.peer_topics[sub_msg.topicid] = origin_id + elif orgin_id not in self.peer_topics[sub_msg.topicid]: # Add peer to topic - self.peer_topics[topic_id].append(sub_msg.origin_id) + self.peer_topics[sub_msg.topicid].append(origin_id) else: # TODO: Remove peer from topic pass @@ -250,13 +214,18 @@ class Pubsub(): self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message - sub_msg = MessageSub( - str(self.host.get_id()),\ - str(self.host.get_id()), {topic_id: True}, generate_message_id()\ - ) + packet = rpc_pb2.RPC() + packet.publish.extend([rpc_pb2.Message( + from_id=str(self.host.get_id()).encode('utf-8'), + seqno=str(generate_message_id()).encode('utf-8') + )]) + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe = True, + topicid = topic_id.encode('utf-8') + )]) # Send out subscribe message to all peers - await self.message_all_peers(sub_msg.to_str()) + await self.message_all_peers(packet.SerializeToString()) # Tell router we are joining this topic self.router.join(topic_id) @@ -275,27 +244,31 @@ class Pubsub(): del self.my_topics[topic_id] # Create unsubscribe message - unsub_msg = MessageSub(str(self.host.get_id()), str(self.host.get_id()),\ - {topic_id: False}, generate_message_id()) + packet = rpc_pb2.RPC() + packet.publish.extend([rpc_pb2.Message( + from_id=str(self.host.get_id()).encode('utf-8'), + seqno=str(generate_message_id()).encode('utf-8') + )]) + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe = False, + topicid = topic_id.encode('utf-8') + )]) # Send out unsubscribe message to all peers - await self.message_all_peers(unsub_msg.to_str()) + await self.message_all_peers(packet.SerializeToString()) # Tell router we are leaving this topic self.router.leave(topic_id) - async def message_all_peers(self, raw_msg): + async def message_all_peers(self, rpc_msg): """ Broadcast a message to peers :param raw_msg: raw contents of the message to broadcast """ - # Encode message for sending - encoded_msg = raw_msg.encode() - # Broadcast message for peer in self.peers: stream = self.peers[peer] # Write message to stream - await stream.write(encoded_msg) \ No newline at end of file + await stream.write(rpc_msg) From f5af4b9016d791599ff61465b7e699d82774679a Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sat, 30 Mar 2019 18:49:50 -0400 Subject: [PATCH 009/131] remove message.py --- libp2p/pubsub/floodsub.py | 31 +--------- libp2p/pubsub/message.py | 118 -------------------------------------- libp2p/pubsub/pubsub.py | 28 +++++---- 3 files changed, 15 insertions(+), 162 deletions(-) delete mode 100644 libp2p/pubsub/message.py diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 1cea17e..dd4dfc1 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -1,7 +1,6 @@ from .pubsub_router_interface import IPubsubRouter from .pb import rpc_pb2 -from .message import MessageSub, MessageTalk -from .message import create_message_talk + class FloodSub(IPubsubRouter): @@ -57,40 +56,14 @@ class FloodSub(IPubsubRouter): :param message: message to forward """ - # Encode message - # encoded_msg = message.encode() - - if isinstance(message, str): - msg_talk = create_message_talk(message) - message = rpc_pb2.Message( - from_id=str(msg_talk.origin_id).encode('utf-8'), - seqno=str(msg_talk.message_id).encode('utf-8'), - topicIDs=msg_talk.topics, - data=msg_talk.data.encode() - - ) packet = rpc_pb2.RPC() - print("YEET") - print(type(message)) packet.publish.extend([message]) - - - - # Get message sender, origin, and topics - # msg_talk = create_message_talk(message) msg_sender = str(sender_peer_id) - # msg_origin = msg_talk.origin_id - # topics = msg_talk.topics # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): - old_format = MessageTalk(sender_peer_id, - message.from_id, - message.topicIDs, - message.data, - message.seqno) - await self.pubsub.handle_talk(old_format) + await self.pubsub.handle_talk(sender_peer_id, message) # Deliver to self and peers for topic in message.topicIDs: diff --git a/libp2p/pubsub/message.py b/libp2p/pubsub/message.py deleted file mode 100644 index 2f839dc..0000000 --- a/libp2p/pubsub/message.py +++ /dev/null @@ -1,118 +0,0 @@ -import uuid - - -class MessageTalk(): - - """ - Object to make parsing talk messages easier, where talk messages are - defined as custom messages published to a set of topics - """ - # pylint: disable=too-few-public-methods - def __init__(self, from_id, origin_id, topics, data, message_id): - # pylint: disable=too-many-arguments - self.msg_type = "talk" - self.from_id = from_id - self.origin_id = origin_id - self.topics = topics - self.data = data - self.message_id = message_id - - def to_str(self): - """ - Convert to string - :return: MessageTalk object in string representation - """ - out = self.msg_type + '\n' - out += self.from_id + '\n' - out += self.origin_id + '\n' - out += self.message_id + '\n' - for i in range(len(self.topics)): - out += self.topics[i] - if i < len(self.topics) - 1: - out += ',' - out += '\n' + self.data - return out - - -class MessageSub(): - """ - Object to make parsing subscription messages easier, where subscription - messages are defined as indicating the topics a node wishes to subscribe to - or unsubscribe from - """ - # pylint: disable=too-few-public-methods - def __init__(self, from_id, origin_id, subs_map, message_id): - self.msg_type = "subscription" - self.from_id = from_id - self.origin_id = origin_id - self.subs_map = subs_map - self.message_id = message_id - - def to_str(self): - """ - Convert to string - :return: MessageSub object in string representation - """ - out = self.msg_type + '\n' - out += self.from_id + '\n' - out += self.origin_id + '\n' - out += self.message_id - - if self.subs_map: - out += '\n' - - keys = list(self.subs_map) - - for i, topic in enumerate(keys): - sub = self.subs_map[topic] - if sub: - out += "sub:" - else: - out += "unsub:" - out += topic - if i < len(keys) - 1: - out += '\n' - - return out - -def create_message_talk(msg_talk_as_str): - """ - Create a MessageTalk object from a MessageTalk string representation - :param msg_talk_as_str: a MessageTalk object in its string representation - :return: MessageTalk object - """ - msg_comps = msg_talk_as_str.split('\n') - from_id = msg_comps[1] - origin_id = msg_comps[2] - message_id = msg_comps[3] - topics = msg_comps[4].split(',') - data = msg_comps[5] - return MessageTalk(from_id, origin_id, topics, data, message_id) - -def create_message_sub(msg_sub_as_str): - """ - Create a MessageSub object from a MessageSub string representation - :param msg_talk_as_str: a MessageSub object in its string representation - :return: MessageSub object - """ - msg_comps = msg_sub_as_str.split('\n') - from_id = msg_comps[1] - origin_id = msg_comps[2] - message_id = msg_comps[3] - - subs_map = {} - for i in range(4, len(msg_comps)): - sub_comps = msg_comps[i].split(":") - topic = sub_comps[1] - if sub_comps[0] == "sub": - subs_map[topic] = True - else: - subs_map[topic] = False - return MessageSub(from_id, origin_id, subs_map, message_id) - -def generate_message_id(): - """ - Generate a unique message id - :return: messgae id - """ - return str(uuid.uuid1()) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 85abacb..976e6b4 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -1,11 +1,9 @@ import asyncio +import uuid from .pb import rpc_pb2_grpc from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee -from .message import MessageSub, MessageTalk -from .message import create_message_talk, create_message_sub -from. message import generate_message_id class Pubsub(): @@ -77,8 +75,7 @@ class Pubsub(): async def continuously_read_stream(self, stream): """ Read from input stream in an infinite loop. Process - messages from other nodes, which for now are considered MessageTalk - and MessageSub messages. + messages from other nodes :param stream: stream to continously read from """ @@ -94,13 +91,8 @@ class Pubsub(): if rpc_incoming.publish: # deal with "talk messages" for msg in rpc_incoming.publish: - old_format = MessageTalk(peer_id, - msg.from_id, - msg.topicIDs, - msg.data, - msg.seqno) self.seen_messages.append(msg.seqno) - await self.handle_talk(old_format) + await self.handle_talk(peer_id, msg) await self.router.publish(peer_id, msg) if rpc_incoming.subscriptions: @@ -189,21 +181,20 @@ class Pubsub(): # TODO: Remove peer from topic pass - async def handle_talk(self, talk): + async def handle_talk(self, peer_id, publish_message): """ Handle incoming Talk message from a peer. A Talk message contains some custom message that is published on a given topic(s) :param talk: raw data constituting a talk message """ - msg = talk # Check if this message has any topics that we are subscribed to - for topic in msg.topics: + for topic in publish_message.topicIDs: if topic in self.my_topics: # we are subscribed to a topic this message was sent for, # so add message to the subscription output queue # for each topic - await self.my_topics[topic].put(talk) + await self.my_topics[topic].put(publish_message) async def subscribe(self, topic_id): """ @@ -272,3 +263,10 @@ class Pubsub(): # Write message to stream await stream.write(rpc_msg) + +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) \ No newline at end of file From ec7bc45a5805faa95d2a389541c246b6412661d7 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sat, 30 Mar 2019 19:12:31 -0400 Subject: [PATCH 010/131] remove Message from dummy account --- tests/pubsub/dummy_account_node.py | 57 +++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index f328a6e..de78211 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -1,12 +1,12 @@ import asyncio +import uuid import multiaddr from libp2p import new_node -from libp2p.pubsub.message import create_message_talk +from libp2p.pubsub.pb import rpc_pb2_grpc +from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub -from libp2p.pubsub.message import MessageTalk -from libp2p.pubsub.message import generate_message_id SUPPORTED_PUBSUB_PROTOCOLS = ["/floodsub/1.0.0"] CRYPTO_TOPIC = "ethereum" @@ -53,16 +53,15 @@ class DummyAccountNode(): Handle all incoming messages on the CRYPTO_TOPIC from peers """ while True: - message_raw = await self.q.get() - message = create_message_talk(message_raw) - contents = message.data - - msg_comps = contents.split(",") - - if msg_comps[0] == "send": - self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) - elif msg_comps[0] == "set": - self.handle_set_crypto_for_user(msg_comps[1], int(msg_comps[2])) + incoming = await self.q.get() + rpc_incoming = rpc_pb2.RPC() + rpc_incoming.ParseFromString(incoming) + for message in rpc_incoming.publish: + msg_comps = message.split(",") + if msg_comps[0] == "send": + self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) + elif msg_comps[0] == "set": + self.handle_set_crypto_for_user(msg_comps[1], int(msg_comps[2])) async def setup_crypto_networking(self): """ @@ -82,8 +81,8 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "send," + source_user + "," + dest_user + "," + str(amount) - msg = MessageTalk(my_id, my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) - await self.floodsub.publish(my_id, msg.to_str()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + await self.floodsub.publish(my_id, msg.SerializeToString()) async def publish_set_crypto(self, user, amount): """ @@ -93,8 +92,9 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "set," + user + "," + str(amount) - msg = MessageTalk(my_id, my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) - await self.floodsub.publish(my_id, msg.to_str()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + + await self.floodsub.publish(my_id, packet.SerializeToString()) def handle_send_crypto(self, source_user, dest_user, amount): """ @@ -132,3 +132,26 @@ class DummyAccountNode(): else: return -1 +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) + +def generate_RPC_packet(origin_id, topics, msg_content, msg_id): + packet = rpc_pb2.RPC() + packet.publish.extend([rpc_pb2.Message( + from_id=origin_id.encode('utf-8'), + seqno=msg_id.encode('utf-8'), + data=msg_content.encode('utf-8'), + topicIDs=topics.encode('utf-8')) + ]) + + for topic in topics: + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe=True, + topicid = topic.encode('utf-8') + )]) + + return packet From 971dbe1a968f7df704bab20bb4264015a3307731 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sat, 30 Mar 2019 19:30:58 -0400 Subject: [PATCH 011/131] fix encoding issue --- libp2p/pubsub/floodsub.py | 3 +++ tests/pubsub/dummy_account_node.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index dd4dfc1..8b9ba1c 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -57,6 +57,9 @@ class FloodSub(IPubsubRouter): """ packet = rpc_pb2.RPC() + print ("IN FLOOODSUB PUBLISH") + print (message) + print ("++++++++++++++++") packet.publish.extend([message]) msg_sender = str(sender_peer_id) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index de78211..930dc57 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -141,17 +141,21 @@ def generate_message_id(): def generate_RPC_packet(origin_id, topics, msg_content, msg_id): packet = rpc_pb2.RPC() - packet.publish.extend([rpc_pb2.Message( + message = rpc_pb2.Message( from_id=origin_id.encode('utf-8'), seqno=msg_id.encode('utf-8'), - data=msg_content.encode('utf-8'), - topicIDs=topics.encode('utf-8')) - ]) + data=msg_content.encode('utf-8') + ) for topic in topics: + message.topicIDs.extend([topic.encode('utf-8')]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe=True, topicid = topic.encode('utf-8') )]) + packet.publish.extend([message]) + print ("HEYHO") + print (packet) + return packet From 89a19a9213aff1bae5d7dfb8408e4eae8f8aa30f Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 31 Mar 2019 22:16:28 -0400 Subject: [PATCH 012/131] reworked floodsub logic --- libp2p/pubsub/floodsub.py | 46 +++++++++++++--------- libp2p/pubsub/pubsub.py | 83 ++++++++++++++++++++++----------------- 2 files changed, 74 insertions(+), 55 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 8b9ba1c..d4736d0 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -41,7 +41,7 @@ class FloodSub(IPubsubRouter): :param rpc: rpc message """ - async def publish(self, sender_peer_id, message): + async def publish(self, sender_peer_id, rpc_message): """ Invoked to forward a new message that has been validated. This is where the "flooding" part of floodsub happens @@ -53,33 +53,41 @@ class FloodSub(IPubsubRouter): It also never forwards a message back to the source or the peer that forwarded the message. :param sender_peer_id: peer_id of message sender - :param message: message to forward + :param rpc_message: pubsub message in RPC string format """ packet = rpc_pb2.RPC() + packet.ParseFromString(rpc_message) print ("IN FLOOODSUB PUBLISH") - print (message) + print (packet) print ("++++++++++++++++") - packet.publish.extend([message]) msg_sender = str(sender_peer_id) - # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message - if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): - await self.pubsub.handle_talk(sender_peer_id, message) + for message in packet.publish: + if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): + await self.pubsub.handle_talk(sender_peer_id, message) - # Deliver to self and peers - for topic in message.topicIDs: - if topic in self.pubsub.peer_topics: - for peer_id_in_topic in self.pubsub.peer_topics[topic]: - # Forward to all known peers in the topic that are not the - # message sender and are not the message origin - if peer_id_in_topic not in (msg_sender, message.from_id): - stream = self.pubsub.peers[peer_id_in_topic] - await stream.write(packet.SerializeToString()) - else: - # Implies publish did not write - print("publish did not write") + + print ("OHOHOHOH") + print (self.pubsub.peer_topics) + print ("UUUJUJUJ") + print (self.pubsub.peers) + print ("********") + # Deliver to self and peers + for topic in message.topicIDs: + if topic in self.pubsub.peer_topics: + for peer_id_in_topic in self.pubsub.peer_topics[topic]: + # Forward to all known peers in the topic that are not the + # message sender and are not the message origin + print ("PEERID") + print (peer_id_in_topic) + if peer_id_in_topic not in (msg_sender, message.from_id): + stream = self.pubsub.peers[peer_id_in_topic] + await stream.write(packet.SerializeToString()) + else: + # Implies publish did not write + print("publish did not write") def join(self, topic): """ diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 976e6b4..7705dfb 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -80,31 +80,43 @@ class Pubsub(): """ # TODO check on types here - peer_id = stream.mplex_conn.peer_id + peer_id = str(stream.mplex_conn.peer_id) while True: incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() rpc_incoming.ParseFromString(incoming) - print ("CONTINUOUSLY") + + print ("IN PUBSUB CONTINUOUSLY READ") print (rpc_incoming) + print ("###########################") + + should_publish = True + if rpc_incoming.publish: - # deal with "talk messages" - for msg in rpc_incoming.publish: - self.seen_messages.append(msg.seqno) - await self.handle_talk(peer_id, msg) - await self.router.publish(peer_id, msg) + # deal with RPC.publish + for message in rpc_incoming.publish: + self.seen_messages.append(message.seqno) + await self.handle_talk(peer_id, message) + if rpc_incoming.subscriptions: - # deal with "subscription messages" - subs_map = {} - for msg in rpc_incoming.subscriptions: - if msg.subscribe: - subs_map[msg.topicid] = "sub" - else: - subs_map[msg.topicid] = "unsub" + # deal with RPC.subscriptions + # We don't need to relay the subscription to our + # peers because a given node only needs its peers + # to know that it is subscribed to the topic (doesn't + # need everyone to know) + should_publish = False - self.handle_subscription(rpc_incoming) + # TODO check that peer_id is the same as origin_id + from_id = str(rpc_incoming.publish[0].from_id.decode('utf-8')) + for message in rpc_incoming.subscriptions: + if message.subscribe: + self.handle_subscription(from_id, message) + + if should_publish: + # relay message to peers with router + await self.router.publish(peer_id, incoming) # Force context switch await asyncio.sleep(0) @@ -136,7 +148,10 @@ class Pubsub(): pubsub protocols we support """ while True: + print ("PUBSUB HANDLE PEER QUEUE") peer_id = await self.peer_queue.get() + print (peer_id) + print ("++++++++++++++++++++++++") # Open a stream to peer on existing connection # (we know connection exists since that's the only way @@ -158,34 +173,30 @@ class Pubsub(): # Force context switch await asyncio.sleep(0) - def handle_subscription(self, rpc_message): + def handle_subscription(self, peer_id, sub_message): """ Handle an incoming subscription message from a peer. Update internal mapping to mark the peer as subscribed or unsubscribed to topics as defined in the subscription message - :param subscription: raw data constituting a subscription message + :param origin_id: id of the peer who subscribe to the message + :param sub_message: RPC.SubOpts """ - for sub_msg in rpc_message.subscriptions: - - # Look at each subscription in the msg individually - if sub_msg.subscribe: - origin_id = rpc_message.publish[0].from_id - - if sub_msg.topicid not in self.peer_topics: - # Create topic list if it did not yet exist - self.peer_topics[sub_msg.topicid] = origin_id - elif orgin_id not in self.peer_topics[sub_msg.topicid]: - # Add peer to topic - self.peer_topics[sub_msg.topicid].append(origin_id) - else: - # TODO: Remove peer from topic - pass + # TODO verify logic here + if sub_message.subscribe: + if sub_message.topicid not in self.peer_topics: + self.peer_topics[sub_message.topicid] = [peer_id] + elif peer_id not in self.peer_topics[sub_message.topicid]: + # Add peer to topic + self.peer_topics[sub_message.topicid].append(peer_id) + else: + # TODO: Remove peer from topic + pass async def handle_talk(self, peer_id, publish_message): """ - Handle incoming Talk message from a peer. A Talk message contains some - custom message that is published on a given topic(s) - :param talk: raw data constituting a talk message + Put incoming message from a peer onto my blocking queue + :param peer_id: peer id whom forwarded this message + :param talk: RPC.Message format """ # Check if this message has any topics that we are subscribed to @@ -269,4 +280,4 @@ def generate_message_id(): Generate a unique message id :return: messgae id """ - return str(uuid.uuid1()) \ No newline at end of file + return str(uuid.uuid1()) From de6bc011f01ce506d40c5a22e1c1e2008ad93670 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 31 Mar 2019 22:16:49 -0400 Subject: [PATCH 013/131] update dummy account node --- tests/pubsub/dummy_account_node.py | 32 +++++++++++++++++++----------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 930dc57..0773533 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -37,6 +37,7 @@ class DummyAccountNode(): We use create as this serves as a factory function and allows us to use async await, unlike the init function """ + print ("**DUMMY** CREATE ACCOUNT NODE") self = DummyAccountNode() libp2p_node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -54,14 +55,17 @@ class DummyAccountNode(): """ while True: incoming = await self.q.get() - rpc_incoming = rpc_pb2.RPC() - rpc_incoming.ParseFromString(incoming) - for message in rpc_incoming.publish: - msg_comps = message.split(",") - if msg_comps[0] == "send": - self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) - elif msg_comps[0] == "set": - self.handle_set_crypto_for_user(msg_comps[1], int(msg_comps[2])) + print ("**DUMMY** HANDLE INCOMING") + print (incoming) + print ("========================") + + msg_comps = incoming.data.decode('utf-8').split(",") + print (msg_comps) + print ("--------") + if msg_comps[0] == "send": + self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) + elif msg_comps[0] == "set": + self.handle_set_crypto(msg_comps[1], int(msg_comps[2])) async def setup_crypto_networking(self): """ @@ -103,6 +107,7 @@ class DummyAccountNode(): :param dest_user: user to send crypto to :param amount: amount of crypto to send """ + print ("**DUMMY** IN HANDLE SEND CRYPTO") if source_user in self.balances: self.balances[source_user] -= amount else: @@ -113,12 +118,15 @@ class DummyAccountNode(): else: self.balances[dest_user] = amount - def handle_set_crypto_for_user(self, dest_user, amount): + def handle_set_crypto(self, dest_user, amount): """ Handle incoming set_crypto message :param dest_user: user to set crypto for :param amount: amount of crypto """ + print ("**DUMMY** IN HANDLE SET CRYPTO") + print (type (dest_user)) + print (amount) self.balances[dest_user] = amount def get_balance(self, user): @@ -127,6 +135,9 @@ class DummyAccountNode(): :param user: user to get balance for :return: balance of user """ + print ("GET BALACNCE") + print (user) + print (self.balances) if user in self.balances: return self.balances[user] else: @@ -155,7 +166,4 @@ def generate_RPC_packet(origin_id, topics, msg_content, msg_id): )]) packet.publish.extend([message]) - print ("HEYHO") - print (packet) - return packet From 2e5e7e3c10da71178506ed8445fcfe06d387c01d Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Mon, 1 Apr 2019 15:04:20 -0400 Subject: [PATCH 014/131] remove message from test dummy --- tests/pubsub/dummy_account_node.py | 2 +- tests/pubsub/test_dummyaccount_demo.py | 148 ++++++++++++------------- 2 files changed, 74 insertions(+), 76 deletions(-) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 0773533..eaca614 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -125,7 +125,7 @@ class DummyAccountNode(): :param amount: amount of crypto """ print ("**DUMMY** IN HANDLE SET CRYPTO") - print (type (dest_user)) + print (dest_user) print (amount) self.balances[dest_user] = amount diff --git a/tests/pubsub/test_dummyaccount_demo.py b/tests/pubsub/test_dummyaccount_demo.py index a071fa6..9b3a149 100644 --- a/tests/pubsub/test_dummyaccount_demo.py +++ b/tests/pubsub/test_dummyaccount_demo.py @@ -8,8 +8,6 @@ from libp2p import new_node from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub -from libp2p.pubsub.message import MessageTalk -from libp2p.pubsub.message import create_message_talk from dummy_account_node import DummyAccountNode # pylint: disable=too-many-locals @@ -66,7 +64,7 @@ async def perform_test(num_nodes, adjacency_map, action_func, assertion_func): await action_func(dummy_nodes) # Allow time for action function to be performed (i.e. messages to propogate) - await asyncio.sleep(0.25) + await asyncio.sleep(1) # Perform assertion function for dummy_node in dummy_nodes: @@ -88,102 +86,102 @@ async def test_simple_two_nodes(): await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_simple_three_nodes_line_topography(): - num_nodes = 3 - adj_map = {0: [1], 1: [2]} +# @pytest.mark.asyncio +# async def test_simple_three_nodes_line_topography(): +# num_nodes = 3 +# adj_map = {0: [1], 1: [2]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 10) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 10) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 10 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 10 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_simple_three_nodes_triangle_topography(): - num_nodes = 3 - adj_map = {0: [1, 2], 1: [2]} +# @pytest.mark.asyncio +# async def test_simple_three_nodes_triangle_topography(): +# num_nodes = 3 +# adj_map = {0: [1, 2], 1: [2]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 20) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 20) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 20 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 20 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_simple_seven_nodes_tree_topography(): - num_nodes = 7 - adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +# @pytest.mark.asyncio +# async def test_simple_seven_nodes_tree_topography(): +# num_nodes = 7 +# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 20) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 20) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 20 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 20 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_set_then_send_from_root_seven_nodes_tree_topography(): - num_nodes = 7 - adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +# @pytest.mark.asyncio +# async def test_set_then_send_from_root_seven_nodes_tree_topography(): +# num_nodes = 7 +# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 20) - await asyncio.sleep(0.25) - await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 20) +# await asyncio.sleep(0.25) +# await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 15 - assert dummy_node.get_balance("alex") == 5 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 15 +# assert dummy_node.get_balance("alex") == 5 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): - num_nodes = 7 - adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +# @pytest.mark.asyncio +# async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): +# num_nodes = 7 +# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} - async def action_func(dummy_nodes): - await dummy_nodes[6].publish_set_crypto("aspyn", 20) - await asyncio.sleep(0.25) - await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) +# async def action_func(dummy_nodes): +# await dummy_nodes[6].publish_set_crypto("aspyn", 20) +# await asyncio.sleep(0.25) +# await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 15 - assert dummy_node.get_balance("alex") == 5 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 15 +# assert dummy_node.get_balance("alex") == 5 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_simple_five_nodes_ring_topography(): - num_nodes = 5 - adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} +# @pytest.mark.asyncio +# async def test_simple_five_nodes_ring_topography(): +# num_nodes = 5 +# adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 20) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 20) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 20 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 20 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): - num_nodes = 5 - adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} +# @pytest.mark.asyncio +# async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): +# num_nodes = 5 +# adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("alex", 20) - await asyncio.sleep(0.25) - await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("alex", 20) +# await asyncio.sleep(0.25) +# await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) - def assertion_func(dummy_node): - assert dummy_node.get_balance("alex") == 8 - assert dummy_node.get_balance("rob") == 12 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("alex") == 8 +# assert dummy_node.get_balance("rob") == 12 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) From 6eb070b78e53c4bcdab6d82bf073bbba86523517 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Mon, 1 Apr 2019 16:23:20 -0400 Subject: [PATCH 015/131] fix all tests --- libp2p/pubsub/floodsub.py | 18 ++- libp2p/pubsub/pubsub.py | 62 ++++++----- tests/pubsub/dummy_account_node.py | 4 +- tests/pubsub/test_dummyaccount_demo.py | 144 ++++++++++++------------ tests/pubsub/test_floodsub.py | 147 ++++++++++++++++--------- 5 files changed, 221 insertions(+), 154 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index d4736d0..305607e 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -65,7 +65,16 @@ class FloodSub(IPubsubRouter): # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message for message in packet.publish: - if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): + decoded_from_id = message.from_id.decode('utf-8') + + print ("MESSAGE SENDER") + print (msg_sender) + print ("FROM ID") + print (message.from_id) + print (str(self.pubsub.host.get_id())) + + + if msg_sender == decoded_from_id and msg_sender == str(self.pubsub.host.get_id()): await self.pubsub.handle_talk(sender_peer_id, message) @@ -82,9 +91,12 @@ class FloodSub(IPubsubRouter): # message sender and are not the message origin print ("PEERID") print (peer_id_in_topic) - if peer_id_in_topic not in (msg_sender, message.from_id): + if peer_id_in_topic not in (msg_sender, decoded_from_id): stream = self.pubsub.peers[peer_id_in_topic] - await stream.write(packet.SerializeToString()) + # create new packet with just publish message + new_packet = rpc_pb2.RPC() + new_packet.publish.extend([message]) + await stream.write(new_packet.SerializeToString()) else: # Implies publish did not write print("publish did not write") diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 7705dfb..c7bfe06 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -58,17 +58,19 @@ class Pubsub(): def get_hello_packet(self): """ Generate subscription message with all topics we are subscribed to + only send hello packet if we have subscribed topics """ - packet = rpc_pb2.RPC() - message = rpc_pb2.Message( - from_id=str(self.host.get_id()).encode('utf-8'), - seqno=str(generate_message_id()).encode('utf-8') - ) - packet.publish.extend([message]) - for topic_id in self.my_topics: - packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - subscribe=True, topicid=topic_id)]) + if self.my_topics: + for topic_id in self.my_topics: + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe=True, topicid=topic_id)]) + + # message = rpc_pb2.Message( + # from_id=str(self.host.get_id()).encode('utf-8'), + # seqno=str(generate_message_id()).encode('utf-8') + # ) + # packet.publish.extend([message]) return packet.SerializeToString() @@ -80,9 +82,11 @@ class Pubsub(): """ # TODO check on types here + print ("++++++++++ASPYN+++++++++++++++++") peer_id = str(stream.mplex_conn.peer_id) while True: + print ("HIT ME") incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() rpc_incoming.ParseFromString(incoming) @@ -91,13 +95,15 @@ class Pubsub(): print (rpc_incoming) print ("###########################") - should_publish = True + should_publish = False if rpc_incoming.publish: # deal with RPC.publish for message in rpc_incoming.publish: - self.seen_messages.append(message.seqno) - await self.handle_talk(peer_id, message) + if message.seqno not in self.seen_messages: + should_publish = True + self.seen_messages.append(message.seqno) + await self.handle_talk(peer_id, message) if rpc_incoming.subscriptions: @@ -106,13 +112,9 @@ class Pubsub(): # peers because a given node only needs its peers # to know that it is subscribed to the topic (doesn't # need everyone to know) - should_publish = False - - # TODO check that peer_id is the same as origin_id - from_id = str(rpc_incoming.publish[0].from_id.decode('utf-8')) for message in rpc_incoming.subscriptions: if message.subscribe: - self.handle_subscription(from_id, message) + self.handle_subscription(peer_id, message) if should_publish: # relay message to peers with router @@ -182,6 +184,8 @@ class Pubsub(): :param sub_message: RPC.SubOpts """ # TODO verify logic here + + if sub_message.subscribe: if sub_message.topicid not in self.peer_topics: self.peer_topics[sub_message.topicid] = [peer_id] @@ -213,19 +217,23 @@ class Pubsub(): :param topic_id: topic_id to subscribe to """ # Map topic_id to blocking queue + + print ("**PUBSUB** in SUBSCRIBE") self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message packet = rpc_pb2.RPC() - packet.publish.extend([rpc_pb2.Message( - from_id=str(self.host.get_id()).encode('utf-8'), - seqno=str(generate_message_id()).encode('utf-8') - )]) + # packet.publish.extend([rpc_pb2.Message( + # from_id=str(self.host.get_id()).encode('utf-8'), + # seqno=str(generate_message_id()).encode('utf-8') + # )]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe = True, topicid = topic_id.encode('utf-8') )]) - + print (packet) + print ("**PUBSUB** PEEERS") + print (self.peers) # Send out subscribe message to all peers await self.message_all_peers(packet.SerializeToString()) @@ -247,10 +255,10 @@ class Pubsub(): # Create unsubscribe message packet = rpc_pb2.RPC() - packet.publish.extend([rpc_pb2.Message( - from_id=str(self.host.get_id()).encode('utf-8'), - seqno=str(generate_message_id()).encode('utf-8') - )]) + # packet.publish.extend([rpc_pb2.Message( + # from_id=str(self.host.get_id()).encode('utf-8'), + # seqno=str(generate_message_id()).encode('utf-8') + # )]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe = False, topicid = topic_id.encode('utf-8') @@ -267,6 +275,8 @@ class Pubsub(): Broadcast a message to peers :param raw_msg: raw contents of the message to broadcast """ + print ("**PUBSU** IN MESSAGE ALL PEERS") + print (rpc_msg) # Broadcast message for peer in self.peers: diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index eaca614..1951fc6 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -86,7 +86,7 @@ class DummyAccountNode(): my_id = str(self.libp2p_node.get_id()) msg_contents = "send," + source_user + "," + dest_user + "," + str(amount) packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) - await self.floodsub.publish(my_id, msg.SerializeToString()) + await self.floodsub.publish(my_id, packet.SerializeToString()) async def publish_set_crypto(self, user, amount): """ @@ -128,6 +128,8 @@ class DummyAccountNode(): print (dest_user) print (amount) self.balances[dest_user] = amount + print (self.balances) + print ("^^ balance") def get_balance(self, user): """ diff --git a/tests/pubsub/test_dummyaccount_demo.py b/tests/pubsub/test_dummyaccount_demo.py index 9b3a149..1f08c8d 100644 --- a/tests/pubsub/test_dummyaccount_demo.py +++ b/tests/pubsub/test_dummyaccount_demo.py @@ -86,102 +86,102 @@ async def test_simple_two_nodes(): await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_simple_three_nodes_line_topography(): -# num_nodes = 3 -# adj_map = {0: [1], 1: [2]} +@pytest.mark.asyncio +async def test_simple_three_nodes_line_topography(): + num_nodes = 3 + adj_map = {0: [1], 1: [2]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 10) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 10) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 10 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 10 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_simple_three_nodes_triangle_topography(): -# num_nodes = 3 -# adj_map = {0: [1, 2], 1: [2]} +@pytest.mark.asyncio +async def test_simple_three_nodes_triangle_topography(): + num_nodes = 3 + adj_map = {0: [1, 2], 1: [2]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 20) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 20 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_simple_seven_nodes_tree_topography(): -# num_nodes = 7 -# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +@pytest.mark.asyncio +async def test_simple_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 20) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 20 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_set_then_send_from_root_seven_nodes_tree_topography(): -# num_nodes = 7 -# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +@pytest.mark.asyncio +async def test_set_then_send_from_root_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 20) -# await asyncio.sleep(0.25) -# await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) + await asyncio.sleep(0.25) + await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 15 -# assert dummy_node.get_balance("alex") == 5 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 15 + assert dummy_node.get_balance("alex") == 5 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): -# num_nodes = 7 -# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +@pytest.mark.asyncio +async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} -# async def action_func(dummy_nodes): -# await dummy_nodes[6].publish_set_crypto("aspyn", 20) -# await asyncio.sleep(0.25) -# await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) + async def action_func(dummy_nodes): + await dummy_nodes[6].publish_set_crypto("aspyn", 20) + await asyncio.sleep(0.25) + await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 15 -# assert dummy_node.get_balance("alex") == 5 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 15 + assert dummy_node.get_balance("alex") == 5 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_simple_five_nodes_ring_topography(): -# num_nodes = 5 -# adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} +@pytest.mark.asyncio +async def test_simple_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 20) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 20 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): -# num_nodes = 5 -# adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} +@pytest.mark.asyncio +async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("alex", 20) -# await asyncio.sleep(0.25) -# await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("alex", 20) + await asyncio.sleep(0.25) + await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("alex") == 8 -# assert dummy_node.get_balance("rob") == 12 + def assertion_func(dummy_node): + assert dummy_node.get_balance("alex") == 8 + assert dummy_node.get_balance("rob") == 12 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index f4a5826..c4c2681 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -1,15 +1,14 @@ import asyncio +import uuid import multiaddr import pytest from tests.utils import cleanup from libp2p import new_node from libp2p.peer.peerinfo import info_from_p2p_addr +from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub -from libp2p.pubsub.message import MessageTalk -from libp2p.pubsub.message import create_message_talk -from libp2p.pubsub.message import generate_message_id # pylint: disable=too-many-locals @@ -22,7 +21,7 @@ async def connect(node1, node2): await node1.connect(info) @pytest.mark.asyncio -async def test_simple_two_nodes(): +async def test_simple_two_nodes_RPC(): node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -45,71 +44,77 @@ async def test_simple_two_nodes(): node_a_id = str(node_a.get_id()) - msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) - - await floodsub_a.publish(node_a.get_id(), msg.to_str()) - + msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", generate_message_id()) + await floodsub_a.publish(node_a_id, msg.SerializeToString()) + print ("MESSAGE B") + print (msg.SerializeToString()) + print ("=========") await asyncio.sleep(0.25) res_b = await qb.get() + print ("RES B") + print (res_b) + print ("-----") # Check that the msg received by node_b is the same # as the message sent by node_a - assert res_b == msg.to_str() + assert res_b.SerializeToString() == msg.publish[0].SerializeToString() + + # Success, terminate pending tasks. await cleanup() -@pytest.mark.asyncio -async def test_simple_three_nodes(): - # Want to pass message from A -> B -> C - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_c = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) +# @pytest.mark.asyncio +# async def test_simple_three_nodes(): +# # Want to pass message from A -> B -> C +# node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) +# node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) +# node_c = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) - await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) - await node_c.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) +# await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) +# await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) +# await node_c.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) - supported_protocols = ["/floodsub/1.0.0"] +# supported_protocols = ["/floodsub/1.0.0"] - floodsub_a = FloodSub(supported_protocols) - pubsub_a = Pubsub(node_a, floodsub_a, "a") - floodsub_b = FloodSub(supported_protocols) - pubsub_b = Pubsub(node_b, floodsub_b, "b") - floodsub_c = FloodSub(supported_protocols) - pubsub_c = Pubsub(node_c, floodsub_c, "c") +# floodsub_a = FloodSub(supported_protocols) +# pubsub_a = Pubsub(node_a, floodsub_a, "a") +# floodsub_b = FloodSub(supported_protocols) +# pubsub_b = Pubsub(node_b, floodsub_b, "b") +# floodsub_c = FloodSub(supported_protocols) +# pubsub_c = Pubsub(node_c, floodsub_c, "c") - await connect(node_a, node_b) - await connect(node_b, node_c) +# await connect(node_a, node_b) +# await connect(node_b, node_c) - await asyncio.sleep(0.25) - qb = await pubsub_b.subscribe("my_topic") - qc = await pubsub_c.subscribe("my_topic") - await asyncio.sleep(0.25) +# await asyncio.sleep(0.25) +# qb = await pubsub_b.subscribe("my_topic") +# qc = await pubsub_c.subscribe("my_topic") +# await asyncio.sleep(0.25) - node_a_id = str(node_a.get_id()) +# node_a_id = str(node_a.get_id()) - msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) +# msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) - await floodsub_a.publish(node_a.get_id(), msg.to_str()) +# await floodsub_a.publish(node_a.get_id(), msg.to_str()) - await asyncio.sleep(0.25) - res_b = await qb.get() - res_c = await qc.get() +# await asyncio.sleep(0.25) +# res_b = await qb.get() +# res_c = await qc.get() - # Check that the msg received by node_b is the same - # as the message sent by node_a - assert res_b == msg.to_str() +# # Check that the msg received by node_b is the same +# # as the message sent by node_a +# assert res_b == msg.to_str() - # res_c should match original msg but with b as sender - node_b_id = str(node_b.get_id()) - msg.from_id = node_b_id +# # res_c should match original msg but with b as sender +# node_b_id = str(node_b.get_id()) +# msg.from_id = node_b_id - assert res_c == msg.to_str() +# assert res_c == msg.to_str() - # Success, terminate pending tasks. - await cleanup() +# # Success, terminate pending tasks. +# await cleanup() async def perform_test_from_obj(obj): """ @@ -237,11 +242,14 @@ async def perform_test_from_obj(obj): actual_node_id = str(node_map[node_id].get_id()) # Create correctly formatted message - msg_talk = MessageTalk(actual_node_id, actual_node_id, topics, data, generate_message_id()) + msg_talk = generate_RPC_packet(actual_node_id, topics, data, generate_message_id()) + print ("**TEST FLOODSUB** MESSAGE TALK") + print (msg_talk) # Publish message # await floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()) - tasks_publish.append(asyncio.ensure_future(floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()))) + tasks_publish.append(asyncio.ensure_future(floodsub_map[node_id].publish(\ + actual_node_id, msg_talk.SerializeToString()))) # For each topic in topics, add topic, msg_talk tuple to ordered test list # TODO: Update message sender to be correct message sender before @@ -261,12 +269,20 @@ async def perform_test_from_obj(obj): for node_id in topic_map[topic]: # Get message from subscription queue msg_on_node_str = await queues_map[node_id][topic].get() - msg_on_node = create_message_talk(msg_on_node_str) + + print ("MESSAGE ON NODE STR") + print (msg_on_node_str) + + print ("ACTUAL MESSSSAGE") + print (actual_msg) + + assert actual_msg.publish[0].SerializeToString() == msg_on_node_str.SerializeToString() + # msg_on_node = create_message_talk(msg_on_node_str) # Perform checks - assert actual_msg.origin_id == msg_on_node.origin_id - assert actual_msg.topics == msg_on_node.topics - assert actual_msg.data == msg_on_node.data + # assert actual_msg.origin_id == msg_on_node.origin_id + # assert actual_msg.topics == msg_on_node.topics + # assert actual_msg.data == msg_on_node.data # Success, terminate pending tasks. await cleanup() @@ -484,3 +500,30 @@ async def test_three_nodes_clique_two_topic_diff_origin_test_obj(): ] } await perform_test_from_obj(test_obj) + +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) + +def generate_RPC_packet(origin_id, topics, msg_content, msg_id): + packet = rpc_pb2.RPC() + message = rpc_pb2.Message( + from_id=origin_id.encode('utf-8'), + seqno=msg_id.encode('utf-8'), + data=msg_content.encode('utf-8'), + ) + for topic in topics: + message.topicIDs.extend([topic.encode('utf-8')]) + + # for topic in topics: + # message.topicIDs.extend([topic.encode('utf-8')]) + # packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + # subscribe=True, + # topicid = topic.encode('utf-8') + # )]) + + packet.publish.extend([message]) + return packet From 41d1aae55b8a2f65212254ee712b6ca2083c1885 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Mon, 1 Apr 2019 16:55:44 -0400 Subject: [PATCH 016/131] clean up --- libp2p/pubsub/floodsub.py | 25 +----- libp2p/pubsub/pubsub.py | 57 +++---------- libp2p/pubsub/pubsub_router_interface.py | 4 +- tests/pubsub/dummy_account_node.py | 47 +---------- tests/pubsub/test_floodsub.py | 103 +---------------------- tests/pubsub/utils.py | 30 +++++++ 6 files changed, 54 insertions(+), 212 deletions(-) create mode 100644 tests/pubsub/utils.py diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 305607e..a4f2e86 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -1,8 +1,9 @@ -from .pubsub_router_interface import IPubsubRouter from .pb import rpc_pb2 +from .pubsub_router_interface import IPubsubRouter class FloodSub(IPubsubRouter): + # pylint: disable=no-member def __init__(self, protocols): self.protocols = protocols @@ -55,42 +56,22 @@ class FloodSub(IPubsubRouter): :param sender_peer_id: peer_id of message sender :param rpc_message: pubsub message in RPC string format """ - packet = rpc_pb2.RPC() packet.ParseFromString(rpc_message) - print ("IN FLOOODSUB PUBLISH") - print (packet) - print ("++++++++++++++++") msg_sender = str(sender_peer_id) # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message for message in packet.publish: decoded_from_id = message.from_id.decode('utf-8') - - print ("MESSAGE SENDER") - print (msg_sender) - print ("FROM ID") - print (message.from_id) - print (str(self.pubsub.host.get_id())) - - if msg_sender == decoded_from_id and msg_sender == str(self.pubsub.host.get_id()): - await self.pubsub.handle_talk(sender_peer_id, message) + await self.pubsub.handle_talk(message) - - print ("OHOHOHOH") - print (self.pubsub.peer_topics) - print ("UUUJUJUJ") - print (self.pubsub.peers) - print ("********") # Deliver to self and peers for topic in message.topicIDs: if topic in self.pubsub.peer_topics: for peer_id_in_topic in self.pubsub.peer_topics[topic]: # Forward to all known peers in the topic that are not the # message sender and are not the message origin - print ("PEERID") - print (peer_id_in_topic) if peer_id_in_topic not in (msg_sender, decoded_from_id): stream = self.pubsub.peers[peer_id_in_topic] # create new packet with just publish message diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index c7bfe06..bfad873 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -1,13 +1,12 @@ import asyncio import uuid -from .pb import rpc_pb2_grpc from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee class Pubsub(): - # pylint: disable=too-many-instance-attributes + # pylint: disable=too-many-instance-attributes, no-member def __init__(self, host, router, my_id): """ @@ -65,12 +64,6 @@ class Pubsub(): for topic_id in self.my_topics: packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe=True, topicid=topic_id)]) - - # message = rpc_pb2.Message( - # from_id=str(self.host.get_id()).encode('utf-8'), - # seqno=str(generate_message_id()).encode('utf-8') - # ) - # packet.publish.extend([message]) return packet.SerializeToString() @@ -82,19 +75,13 @@ class Pubsub(): """ # TODO check on types here - print ("++++++++++ASPYN+++++++++++++++++") peer_id = str(stream.mplex_conn.peer_id) while True: - print ("HIT ME") incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() rpc_incoming.ParseFromString(incoming) - print ("IN PUBSUB CONTINUOUSLY READ") - print (rpc_incoming) - print ("###########################") - should_publish = False if rpc_incoming.publish: @@ -103,8 +90,7 @@ class Pubsub(): if message.seqno not in self.seen_messages: should_publish = True self.seen_messages.append(message.seqno) - await self.handle_talk(peer_id, message) - + await self.handle_talk(message) if rpc_incoming.subscriptions: # deal with RPC.subscriptions @@ -150,10 +136,8 @@ class Pubsub(): pubsub protocols we support """ while True: - print ("PUBSUB HANDLE PEER QUEUE") + peer_id = await self.peer_queue.get() - print (peer_id) - print ("++++++++++++++++++++++++") # Open a stream to peer on existing connection # (we know connection exists since that's the only way @@ -175,7 +159,7 @@ class Pubsub(): # Force context switch await asyncio.sleep(0) - def handle_subscription(self, peer_id, sub_message): + def handle_subscription(self, origin_id, sub_message): """ Handle an incoming subscription message from a peer. Update internal mapping to mark the peer as subscribed or unsubscribed to topics as @@ -183,23 +167,19 @@ class Pubsub(): :param origin_id: id of the peer who subscribe to the message :param sub_message: RPC.SubOpts """ - # TODO verify logic here - - if sub_message.subscribe: if sub_message.topicid not in self.peer_topics: - self.peer_topics[sub_message.topicid] = [peer_id] - elif peer_id not in self.peer_topics[sub_message.topicid]: + self.peer_topics[sub_message.topicid] = [origin_id] + elif origin_id not in self.peer_topics[sub_message.topicid]: # Add peer to topic - self.peer_topics[sub_message.topicid].append(peer_id) + self.peer_topics[sub_message.topicid].append(origin_id) else: # TODO: Remove peer from topic pass - async def handle_talk(self, peer_id, publish_message): + async def handle_talk(self, publish_message): """ Put incoming message from a peer onto my blocking queue - :param peer_id: peer id whom forwarded this message :param talk: RPC.Message format """ @@ -216,9 +196,8 @@ class Pubsub(): Subscribe ourself to a topic :param topic_id: topic_id to subscribe to """ - # Map topic_id to blocking queue - print ("**PUBSUB** in SUBSCRIBE") + # Map topic_id to blocking queue self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message @@ -228,12 +207,10 @@ class Pubsub(): # seqno=str(generate_message_id()).encode('utf-8') # )]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - subscribe = True, - topicid = topic_id.encode('utf-8') + subscribe=True, + topicid=topic_id.encode('utf-8') )]) - print (packet) - print ("**PUBSUB** PEEERS") - print (self.peers) + # Send out subscribe message to all peers await self.message_all_peers(packet.SerializeToString()) @@ -255,13 +232,9 @@ class Pubsub(): # Create unsubscribe message packet = rpc_pb2.RPC() - # packet.publish.extend([rpc_pb2.Message( - # from_id=str(self.host.get_id()).encode('utf-8'), - # seqno=str(generate_message_id()).encode('utf-8') - # )]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - subscribe = False, - topicid = topic_id.encode('utf-8') + subscribe=False, + topicid=topic_id.encode('utf-8') )]) # Send out unsubscribe message to all peers @@ -275,8 +248,6 @@ class Pubsub(): Broadcast a message to peers :param raw_msg: raw contents of the message to broadcast """ - print ("**PUBSU** IN MESSAGE ALL PEERS") - print (rpc_msg) # Broadcast message for peer in self.peers: diff --git a/libp2p/pubsub/pubsub_router_interface.py b/libp2p/pubsub/pubsub_router_interface.py index 727b39e..ec5132e 100644 --- a/libp2p/pubsub/pubsub_router_interface.py +++ b/libp2p/pubsub/pubsub_router_interface.py @@ -39,11 +39,11 @@ class IPubsubRouter(ABC): """ @abstractmethod - def publish(self, sender_peer_id, message): + def publish(self, sender_peer_id, rpc_message): """ Invoked to forward a new message that has been validated :param sender_peer_id: peer_id of message sender - :param message: message to forward + :param rpc_message: message to forward """ @abstractmethod diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 1951fc6..05a02bf 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -1,10 +1,8 @@ import asyncio -import uuid import multiaddr +from utils import generate_message_id, generate_RPC_packet from libp2p import new_node -from libp2p.pubsub.pb import rpc_pb2_grpc -from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub @@ -37,7 +35,6 @@ class DummyAccountNode(): We use create as this serves as a factory function and allows us to use async await, unlike the init function """ - print ("**DUMMY** CREATE ACCOUNT NODE") self = DummyAccountNode() libp2p_node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -54,14 +51,9 @@ class DummyAccountNode(): Handle all incoming messages on the CRYPTO_TOPIC from peers """ while True: - incoming = await self.q.get() - print ("**DUMMY** HANDLE INCOMING") - print (incoming) - print ("========================") - + incoming = await self.q.get() msg_comps = incoming.data.decode('utf-8').split(",") - print (msg_comps) - print ("--------") + if msg_comps[0] == "send": self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) elif msg_comps[0] == "set": @@ -107,7 +99,6 @@ class DummyAccountNode(): :param dest_user: user to send crypto to :param amount: amount of crypto to send """ - print ("**DUMMY** IN HANDLE SEND CRYPTO") if source_user in self.balances: self.balances[source_user] -= amount else: @@ -124,12 +115,7 @@ class DummyAccountNode(): :param dest_user: user to set crypto for :param amount: amount of crypto """ - print ("**DUMMY** IN HANDLE SET CRYPTO") - print (dest_user) - print (amount) self.balances[dest_user] = amount - print (self.balances) - print ("^^ balance") def get_balance(self, user): """ @@ -137,35 +123,8 @@ class DummyAccountNode(): :param user: user to get balance for :return: balance of user """ - print ("GET BALACNCE") - print (user) - print (self.balances) if user in self.balances: return self.balances[user] else: return -1 -def generate_message_id(): - """ - Generate a unique message id - :return: messgae id - """ - return str(uuid.uuid1()) - -def generate_RPC_packet(origin_id, topics, msg_content, msg_id): - packet = rpc_pb2.RPC() - message = rpc_pb2.Message( - from_id=origin_id.encode('utf-8'), - seqno=msg_id.encode('utf-8'), - data=msg_content.encode('utf-8') - ) - - for topic in topics: - message.topicIDs.extend([topic.encode('utf-8')]) - packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - subscribe=True, - topicid = topic.encode('utf-8') - )]) - - packet.publish.extend([message]) - return packet diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index c4c2681..4fc059b 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -1,5 +1,4 @@ import asyncio -import uuid import multiaddr import pytest @@ -9,6 +8,7 @@ from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub +from utils import generate_message_id, generate_RPC_packet # pylint: disable=too-many-locals @@ -46,16 +46,10 @@ async def test_simple_two_nodes_RPC(): msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", generate_message_id()) await floodsub_a.publish(node_a_id, msg.SerializeToString()) - print ("MESSAGE B") - print (msg.SerializeToString()) - print ("=========") await asyncio.sleep(0.25) res_b = await qb.get() - print ("RES B") - print (res_b) - print ("-----") # Check that the msg received by node_b is the same # as the message sent by node_a assert res_b.SerializeToString() == msg.publish[0].SerializeToString() @@ -65,57 +59,6 @@ async def test_simple_two_nodes_RPC(): # Success, terminate pending tasks. await cleanup() -# @pytest.mark.asyncio -# async def test_simple_three_nodes(): -# # Want to pass message from A -> B -> C -# node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) -# node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) -# node_c = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - -# await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) -# await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) -# await node_c.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) - -# supported_protocols = ["/floodsub/1.0.0"] - -# floodsub_a = FloodSub(supported_protocols) -# pubsub_a = Pubsub(node_a, floodsub_a, "a") -# floodsub_b = FloodSub(supported_protocols) -# pubsub_b = Pubsub(node_b, floodsub_b, "b") -# floodsub_c = FloodSub(supported_protocols) -# pubsub_c = Pubsub(node_c, floodsub_c, "c") - -# await connect(node_a, node_b) -# await connect(node_b, node_c) - -# await asyncio.sleep(0.25) -# qb = await pubsub_b.subscribe("my_topic") -# qc = await pubsub_c.subscribe("my_topic") -# await asyncio.sleep(0.25) - -# node_a_id = str(node_a.get_id()) - -# msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) - -# await floodsub_a.publish(node_a.get_id(), msg.to_str()) - -# await asyncio.sleep(0.25) -# res_b = await qb.get() -# res_c = await qc.get() - -# # Check that the msg received by node_b is the same -# # as the message sent by node_a -# assert res_b == msg.to_str() - -# # res_c should match original msg but with b as sender -# node_b_id = str(node_b.get_id()) -# msg.from_id = node_b_id - -# assert res_c == msg.to_str() - -# # Success, terminate pending tasks. -# await cleanup() - async def perform_test_from_obj(obj): """ Perform a floodsub test from a test obj. @@ -243,9 +186,7 @@ async def perform_test_from_obj(obj): # Create correctly formatted message msg_talk = generate_RPC_packet(actual_node_id, topics, data, generate_message_id()) - - print ("**TEST FLOODSUB** MESSAGE TALK") - print (msg_talk) + # Publish message # await floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()) tasks_publish.append(asyncio.ensure_future(floodsub_map[node_id].publish(\ @@ -269,20 +210,7 @@ async def perform_test_from_obj(obj): for node_id in topic_map[topic]: # Get message from subscription queue msg_on_node_str = await queues_map[node_id][topic].get() - - print ("MESSAGE ON NODE STR") - print (msg_on_node_str) - - print ("ACTUAL MESSSSAGE") - print (actual_msg) - assert actual_msg.publish[0].SerializeToString() == msg_on_node_str.SerializeToString() - # msg_on_node = create_message_talk(msg_on_node_str) - - # Perform checks - # assert actual_msg.origin_id == msg_on_node.origin_id - # assert actual_msg.topics == msg_on_node.topics - # assert actual_msg.data == msg_on_node.data # Success, terminate pending tasks. await cleanup() @@ -500,30 +428,3 @@ async def test_three_nodes_clique_two_topic_diff_origin_test_obj(): ] } await perform_test_from_obj(test_obj) - -def generate_message_id(): - """ - Generate a unique message id - :return: messgae id - """ - return str(uuid.uuid1()) - -def generate_RPC_packet(origin_id, topics, msg_content, msg_id): - packet = rpc_pb2.RPC() - message = rpc_pb2.Message( - from_id=origin_id.encode('utf-8'), - seqno=msg_id.encode('utf-8'), - data=msg_content.encode('utf-8'), - ) - for topic in topics: - message.topicIDs.extend([topic.encode('utf-8')]) - - # for topic in topics: - # message.topicIDs.extend([topic.encode('utf-8')]) - # packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - # subscribe=True, - # topicid = topic.encode('utf-8') - # )]) - - packet.publish.extend([message]) - return packet diff --git a/tests/pubsub/utils.py b/tests/pubsub/utils.py new file mode 100644 index 0000000..d4695d7 --- /dev/null +++ b/tests/pubsub/utils.py @@ -0,0 +1,30 @@ +import uuid +from libp2p.pubsub.pb import rpc_pb2 + +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) + +def generate_RPC_packet(origin_id, topics, msg_content, msg_id): + """ + Generate RPC packet to send over wire + :param origin_id: peer id of the message origin + :param topics: list of topics + :param msg_content: string of content in data + :param msg_id: seqno for the message + """ + packet = rpc_pb2.RPC() + message = rpc_pb2.Message( + from_id=origin_id.encode('utf-8'), + seqno=msg_id.encode('utf-8'), + data=msg_content.encode('utf-8'), + ) + + for topic in topics: + message.topicIDs.extend([topic.encode('utf-8')]) + + packet.publish.extend([message]) + return packet From 0238dff2173600a109cbbe73a4725e7104360e7e Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Tue, 2 Apr 2019 21:17:48 -0400 Subject: [PATCH 017/131] remove unused code --- libp2p/pubsub/pubsub.py | 12 ------------ tests/pubsub/test_floodsub.py | 2 -- 2 files changed, 14 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index bfad873..bee5fba 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -1,5 +1,4 @@ import asyncio -import uuid from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee @@ -202,10 +201,6 @@ class Pubsub(): # Create subscribe message packet = rpc_pb2.RPC() - # packet.publish.extend([rpc_pb2.Message( - # from_id=str(self.host.get_id()).encode('utf-8'), - # seqno=str(generate_message_id()).encode('utf-8') - # )]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe=True, topicid=topic_id.encode('utf-8') @@ -255,10 +250,3 @@ class Pubsub(): # Write message to stream await stream.write(rpc_msg) - -def generate_message_id(): - """ - Generate a unique message id - :return: messgae id - """ - return str(uuid.uuid1()) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index 4fc059b..272af79 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -54,8 +54,6 @@ async def test_simple_two_nodes_RPC(): # as the message sent by node_a assert res_b.SerializeToString() == msg.publish[0].SerializeToString() - - # Success, terminate pending tasks. await cleanup() From 71282678c4a1f79d91e38f17284c7bc6427f316f Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Tue, 2 Apr 2019 22:05:14 -0400 Subject: [PATCH 018/131] Add priority queues to handle seqno --- libp2p/pubsub/pubsub.py | 14 ++++++++------ libp2p/pubsub/read_only_queue.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 libp2p/pubsub/read_only_queue.py diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index bfad873..37b8714 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -3,6 +3,7 @@ import uuid from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee +from .read_only_queue import ReadOnlyQueue class Pubsub(): @@ -188,8 +189,9 @@ class Pubsub(): if topic in self.my_topics: # we are subscribed to a topic this message was sent for, # so add message to the subscription output queue - # for each topic - await self.my_topics[topic].put(publish_message) + # for each topic with priority being the message's seqno. + # Note: asyncio.PriorityQueue item format is (priority, data) + await self.my_topics[topic].put((publish_message.seqno, publish_message)) async def subscribe(self, topic_id): """ @@ -197,8 +199,8 @@ class Pubsub(): :param topic_id: topic_id to subscribe to """ - # Map topic_id to blocking queue - self.my_topics[topic_id] = asyncio.Queue() + # Map topic_id to a priority blocking queue + self.my_topics[topic_id] = asyncio.PriorityQueue() # Create subscribe message packet = rpc_pb2.RPC() @@ -217,8 +219,8 @@ class Pubsub(): # Tell router we are joining this topic self.router.join(topic_id) - # Return the asyncio queue for messages on this topic - return self.my_topics[topic_id] + # Return the readonly queue for messages on this topic + return ReadOnlyQueue(self.my_topics[topic_id]) async def unsubscribe(self, topic_id): """ diff --git a/libp2p/pubsub/read_only_queue.py b/libp2p/pubsub/read_only_queue.py new file mode 100644 index 0000000..1656768 --- /dev/null +++ b/libp2p/pubsub/read_only_queue.py @@ -0,0 +1,14 @@ +import asyncio + +class ReadOnlyQueue(): + + def __init__(self, queue): + self.queue = queue + + async def get(self): + """ + Get the next item from queue, which has items in the format (priority, data) + :return: next item from the queue + """ + return (await self.queue.get())[1] + From 129658980062e7c9b410b451ba34b1fdeb0473df Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Tue, 2 Apr 2019 22:05:32 -0400 Subject: [PATCH 019/131] Adjust floodsub tests for new seqno util --- tests/pubsub/test_floodsub.py | 11 ++++++++--- tests/pubsub/utils.py | 22 ++++++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index 4fc059b..ba9d33c 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -8,7 +8,7 @@ from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub -from utils import generate_message_id, generate_RPC_packet +from utils import message_id_generator, generate_RPC_packet # pylint: disable=too-many-locals @@ -44,7 +44,8 @@ async def test_simple_two_nodes_RPC(): node_a_id = str(node_a.get_id()) - msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", generate_message_id()) + next_msg_id_func = message_id_generator(0) + msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", next_msg_id_func()) await floodsub_a.publish(node_a_id, msg.SerializeToString()) await asyncio.sleep(0.25) @@ -175,6 +176,8 @@ async def perform_test_from_obj(obj): topics_in_msgs_ordered = [] messages = obj["messages"] tasks_publish = [] + next_msg_id_func = message_id_generator(0) + for msg in messages: topics = msg["topics"] @@ -185,7 +188,7 @@ async def perform_test_from_obj(obj): actual_node_id = str(node_map[node_id].get_id()) # Create correctly formatted message - msg_talk = generate_RPC_packet(actual_node_id, topics, data, generate_message_id()) + msg_talk = generate_RPC_packet(actual_node_id, topics, data, next_msg_id_func()) # Publish message # await floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()) @@ -207,6 +210,8 @@ async def perform_test_from_obj(obj): # TODO: Check message sender too for i in range(len(topics_in_msgs_ordered)): topic, actual_msg = topics_in_msgs_ordered[i] + + # Look at each node in each topic for node_id in topic_map[topic]: # Get message from subscription queue msg_on_node_str = await queues_map[node_id][topic].get() diff --git a/tests/pubsub/utils.py b/tests/pubsub/utils.py index d4695d7..8aa3f88 100644 --- a/tests/pubsub/utils.py +++ b/tests/pubsub/utils.py @@ -1,12 +1,26 @@ import uuid +import struct from libp2p.pubsub.pb import rpc_pb2 -def generate_message_id(): + +def message_id_generator(start_val): """ Generate a unique message id - :return: messgae id + :param start_val: value to start generating messages at + :return: message id """ - return str(uuid.uuid1()) + val = start_val + def generator(): + # Allow manipulation of val within closure + nonlocal val + + # Increment id + val += 1 + + # Convert val to big endian + return struct.pack('>I', val) + + return generator def generate_RPC_packet(origin_id, topics, msg_content, msg_id): """ @@ -19,7 +33,7 @@ def generate_RPC_packet(origin_id, topics, msg_content, msg_id): packet = rpc_pb2.RPC() message = rpc_pb2.Message( from_id=origin_id.encode('utf-8'), - seqno=msg_id.encode('utf-8'), + seqno=msg_id, data=msg_content.encode('utf-8'), ) From 726d083a3acd414c110b66fb18632f89449d4e51 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Tue, 2 Apr 2019 22:34:01 -0400 Subject: [PATCH 020/131] Modify pubsub to have seen message check incorporate seqno and node id --- libp2p/pubsub/pubsub.py | 16 +++++++--------- libp2p/pubsub/read_only_queue.py | 14 -------------- tests/pubsub/dummy_account_node.py | 13 +++++++++---- 3 files changed, 16 insertions(+), 27 deletions(-) delete mode 100644 libp2p/pubsub/read_only_queue.py diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 37b8714..d5d4b52 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -3,7 +3,6 @@ import uuid from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee -from .read_only_queue import ReadOnlyQueue class Pubsub(): @@ -90,7 +89,7 @@ class Pubsub(): for message in rpc_incoming.publish: if message.seqno not in self.seen_messages: should_publish = True - self.seen_messages.append(message.seqno) + self.seen_messages.append((message.seqno, message.from_id)) await self.handle_talk(message) if rpc_incoming.subscriptions: @@ -189,9 +188,8 @@ class Pubsub(): if topic in self.my_topics: # we are subscribed to a topic this message was sent for, # so add message to the subscription output queue - # for each topic with priority being the message's seqno. - # Note: asyncio.PriorityQueue item format is (priority, data) - await self.my_topics[topic].put((publish_message.seqno, publish_message)) + # for each topic + await self.my_topics[topic].put(publish_message) async def subscribe(self, topic_id): """ @@ -199,8 +197,8 @@ class Pubsub(): :param topic_id: topic_id to subscribe to """ - # Map topic_id to a priority blocking queue - self.my_topics[topic_id] = asyncio.PriorityQueue() + # Map topic_id to blocking queue + self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message packet = rpc_pb2.RPC() @@ -219,8 +217,8 @@ class Pubsub(): # Tell router we are joining this topic self.router.join(topic_id) - # Return the readonly queue for messages on this topic - return ReadOnlyQueue(self.my_topics[topic_id]) + # Return the asyncio queue for messages on this topic + return self.my_topics[topic_id] async def unsubscribe(self, topic_id): """ diff --git a/libp2p/pubsub/read_only_queue.py b/libp2p/pubsub/read_only_queue.py deleted file mode 100644 index 1656768..0000000 --- a/libp2p/pubsub/read_only_queue.py +++ /dev/null @@ -1,14 +0,0 @@ -import asyncio - -class ReadOnlyQueue(): - - def __init__(self, queue): - self.queue = queue - - async def get(self): - """ - Get the next item from queue, which has items in the format (priority, data) - :return: next item from the queue - """ - return (await self.queue.get())[1] - diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 05a02bf..8fb7310 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -1,7 +1,8 @@ import asyncio import multiaddr +import uuid -from utils import generate_message_id, generate_RPC_packet +from utils import message_id_generator, generate_RPC_packet from libp2p import new_node from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub @@ -25,6 +26,8 @@ class DummyAccountNode(): def __init__(self): self.balances = {} + self.next_msg_id_func = message_id_generator(0) + self.node_id = str(uuid.uuid1()) @classmethod async def create(cls): @@ -51,7 +54,7 @@ class DummyAccountNode(): Handle all incoming messages on the CRYPTO_TOPIC from peers """ while True: - incoming = await self.q.get() + incoming = await self.q.get() msg_comps = incoming.data.decode('utf-8').split(",") if msg_comps[0] == "send": @@ -77,7 +80,7 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "send," + source_user + "," + dest_user + "," + str(amount) - packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, self.next_msg_id_func()) await self.floodsub.publish(my_id, packet.SerializeToString()) async def publish_set_crypto(self, user, amount): @@ -88,7 +91,7 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "set," + user + "," + str(amount) - packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, self.next_msg_id_func()) await self.floodsub.publish(my_id, packet.SerializeToString()) @@ -99,6 +102,7 @@ class DummyAccountNode(): :param dest_user: user to send crypto to :param amount: amount of crypto to send """ + print("handle send " + self.node_id) if source_user in self.balances: self.balances[source_user] -= amount else: @@ -115,6 +119,7 @@ class DummyAccountNode(): :param dest_user: user to set crypto for :param amount: amount of crypto """ + print("handle set " + self.node_id) self.balances[dest_user] = amount def get_balance(self, user): From 583b79fad72c0b658a45505436d350d4b4192fec Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 00:34:39 -0400 Subject: [PATCH 021/131] Fix seen messages bug --- libp2p/pubsub/pubsub.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index d5d4b52..d8fb3c0 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -87,9 +87,10 @@ class Pubsub(): if rpc_incoming.publish: # deal with RPC.publish for message in rpc_incoming.publish: - if message.seqno not in self.seen_messages: + id_in_seen_msgs = (message.seqno, message.from_id) + if id_in_seen_msgs not in self.seen_messages: should_publish = True - self.seen_messages.append((message.seqno, message.from_id)) + self.seen_messages.append(id_in_seen_msgs) await self.handle_talk(message) if rpc_incoming.subscriptions: From e6605f22e9397abe932d18bc6aa8eb46f0a330ac Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 00:35:39 -0400 Subject: [PATCH 022/131] Add dummy node test --- tests/pubsub/test_dummyaccount_demo.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/pubsub/test_dummyaccount_demo.py b/tests/pubsub/test_dummyaccount_demo.py index 1f08c8d..9fa2aa7 100644 --- a/tests/pubsub/test_dummyaccount_demo.py +++ b/tests/pubsub/test_dummyaccount_demo.py @@ -185,3 +185,28 @@ async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): assert dummy_node.get_balance("rob") == 12 await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_set_then_send_from_five_diff_nodes_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("alex", 20) + await asyncio.sleep(1) + await dummy_nodes[1].publish_send_crypto("alex", "rob", 3) + await asyncio.sleep(1) + await dummy_nodes[2].publish_send_crypto("rob", "aspyn", 2) + await asyncio.sleep(1) + await dummy_nodes[3].publish_send_crypto("aspyn", "zx", 1) + await asyncio.sleep(1) + await dummy_nodes[4].publish_send_crypto("zx", "raul", 1) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("alex") == 17 + assert dummy_node.get_balance("rob") == 1 + assert dummy_node.get_balance("aspyn") == 1 + assert dummy_node.get_balance("zx") == 0 + assert dummy_node.get_balance("raul") == 1 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) From 3a52d29cb784d332045349c26f6c5a8d214752d2 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Wed, 3 Apr 2019 14:05:37 -0400 Subject: [PATCH 023/131] remove redundant proto file --- rpc.proto | 78 ------------------------------------------------------- 1 file changed, 78 deletions(-) delete mode 100644 rpc.proto diff --git a/rpc.proto b/rpc.proto deleted file mode 100644 index 2ae2ef2..0000000 --- a/rpc.proto +++ /dev/null @@ -1,78 +0,0 @@ -// Borrowed from https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto - -syntax = "proto2"; - -package pubsub.pb; - -message RPC { - repeated SubOpts subscriptions = 1; - repeated Message publish = 2; - - message SubOpts { - optional bool subscribe = 1; // subscribe or unsubcribe - optional string topicid = 2; - } - - optional ControlMessage control = 3; -} - -message Message { - optional bytes from = 1; - optional bytes data = 2; - optional bytes seqno = 3; - repeated string topicIDs = 4; - optional bytes signature = 5; - optional bytes key = 6; -} - -message ControlMessage { - repeated ControlIHave ihave = 1; - repeated ControlIWant iwant = 2; - repeated ControlGraft graft = 3; - repeated ControlPrune prune = 4; -} - -message ControlIHave { - optional string topicID = 1; - repeated string messageIDs = 2; -} - -message ControlIWant { - repeated string messageIDs = 1; -} - -message ControlGraft { - optional string topicID = 1; -} - -message ControlPrune { - optional string topicID = 1; -} - -message TopicDescriptor { - optional string name = 1; - optional AuthOpts auth = 2; - optional EncOpts enc = 3; - - message AuthOpts { - optional AuthMode mode = 1; - repeated bytes keys = 2; // root keys to trust - - enum AuthMode { - NONE = 0; // no authentication, anyone can publish - KEY = 1; // only messages signed by keys in the topic descriptor are accepted - WOT = 2; // web of trust, certificates can allow publisher set to grow - } - } - - message EncOpts { - optional EncMode mode = 1; - repeated bytes keyHashes = 2; // the hashes of the shared keys used (salted) - - enum EncMode { - NONE = 0; // no encryption, anyone can read - SHAREDKEY = 1; // messages are encrypted with shared key - WOT = 2; // web of trust, certificates can allow publisher set to grow - } - } -} \ No newline at end of file From 225bd390dfc154a11238460097152a09b0150673 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Wed, 3 Apr 2019 14:08:05 -0400 Subject: [PATCH 024/131] add source to rpc.proto --- libp2p/pubsub/pb/rpc.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libp2p/pubsub/pb/rpc.proto b/libp2p/pubsub/pb/rpc.proto index f07da20..df38bad 100644 --- a/libp2p/pubsub/pb/rpc.proto +++ b/libp2p/pubsub/pb/rpc.proto @@ -1,3 +1,5 @@ +// Modified from https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto + syntax = "proto2"; package pubsub.pb; From 0ff155660d04fc63e1a3e1281d31d0f0cd6cfb6b Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 14:17:33 -0400 Subject: [PATCH 025/131] Add test for multiple messages from two origins --- tests/pubsub/test_floodsub.py | 54 +++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index ba9d33c..00ac956 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -433,3 +433,57 @@ async def test_three_nodes_clique_two_topic_diff_origin_test_obj(): ] } await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_four_nodes_clique_two_topic_diff_origin_many_msgs_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2", "3", "4"], + "2": ["1", "3", "4"], + "3": ["1", "2", "4"], + "4": ["1", "2", "3"] + }, + "topic_map": { + "astrophysics": ["1", "2", "3", "4"], + "school": ["1", "2", "3", "4"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar2", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar3", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic3", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) From c61f4297b06ca14ea9cc98426efef665db70db68 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 14:20:50 -0400 Subject: [PATCH 026/131] Add test for ring topology multiple messages from two origins --- tests/pubsub/test_floodsub.py | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index 00ac956..2d4ed8d 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -487,3 +487,58 @@ async def test_four_nodes_clique_two_topic_diff_origin_many_msgs_test_obj(): ] } await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_five_nodes_ring_two_topic_diff_origin_many_msgs_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2"], + "2": ["3"], + "3": ["4"], + "4": ["5"], + "5": ["1"] + }, + "topic_map": { + "astrophysics": ["1", "2", "3", "4", "5"], + "school": ["1", "2", "3", "4", "5"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar2", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar3", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic3", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) \ No newline at end of file From 510a5eaa3482766f3ec8d440e23390bf742d777b Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 18:12:09 -0500 Subject: [PATCH 027/131] Add notifee interface --- libp2p/network/notifee_interface.py | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 libp2p/network/notifee_interface.py diff --git a/libp2p/network/notifee_interface.py b/libp2p/network/notifee_interface.py new file mode 100644 index 0000000..fbc61f4 --- /dev/null +++ b/libp2p/network/notifee_interface.py @@ -0,0 +1,46 @@ +from abc import ABC, abstractmethod + + +class INotifee(ABC): + + @abstractmethod + def opened_stream(self, network, stream): + """ + :param network: network the stream was opened on + :param stream: stream that was opened + """ + + @abstractmethod + def closed_stream(self, network, stream): + """ + :param network: network the stream was closed on + :param stream: stream that was closed + """ + + @abstractmethod + def connected(self, network, conn): + """ + :param network: network the connection was opened on + :param conn: connection that was opened + """ + + @abstractmethod + def disconnected(self, network, conn): + """ + :param network: network the connection was closed on + :param conn: connection that was closed + """ + + @abstractmethod + def listen(self, network, multiaddr): + """ + :param network: network the listener is listening on + :param multiaddr: multiaddress listener is listening on + """ + + @abstractmethod + def listen_close(self, network, multiaddr): + """ + :param network: network the connection was opened on + :param multiaddr: multiaddress listener is no longer listening on + """ From 95f1f0bbf07d1a33f570889b6e7605dcb48a69e8 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 18:12:33 -0500 Subject: [PATCH 028/131] Add notify function to network interface --- libp2p/network/network_interface.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libp2p/network/network_interface.py b/libp2p/network/network_interface.py index 1867efc..acd20b1 100644 --- a/libp2p/network/network_interface.py +++ b/libp2p/network/network_interface.py @@ -41,3 +41,9 @@ class INetwork(ABC): :param *args: one or many multiaddrs to start listening on :return: True if at least one success """ + + @abstractmethod + def notify(self, notifee): + """ + :param notifee: object implementing Notifee interface + """ From 3dbb969b0b020e260c332fe65b0c936c9af1238c Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 18:18:58 -0500 Subject: [PATCH 029/131] Implement notify feature --- libp2p/network/swarm.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 705ee31..1ecc0df 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -24,6 +24,9 @@ class Swarm(INetwork): self.multiselect = Multiselect() self.multiselect_client = MultiselectClient() + # Create Notifee array + self.notifees = [] + # Create generic protocol handler self.generic_protocol_handler = create_generic_protocol_handler(self) @@ -61,6 +64,7 @@ class Swarm(INetwork): # set muxed connection equal to existing muxed connection muxed_conn = self.connections[peer_id] else: + # Dial peer (connection to peer does not yet exist) # Transport dials peer (gets back a raw conn) raw_conn = await self.transport.dial(multiaddr, self.self_id) @@ -70,6 +74,10 @@ class Swarm(INetwork): # Store muxed connection in connections self.connections[peer_id] = muxed_conn + # Call notifiers since event occurred + for notifee in self.notifees: + notifee.connected(self, muxed_conn) + return muxed_conn async def new_stream(self, peer_id, protocol_ids): @@ -100,6 +108,10 @@ class Swarm(INetwork): net_stream = NetStream(muxed_stream) net_stream.set_protocol(selected_protocol) + # Call notifiers since event occurred + for notifee in self.notifees: + notifee.opened_stream(self, net_stream) + return net_stream async def listen(self, *args): @@ -137,11 +149,20 @@ class Swarm(INetwork): # Store muxed_conn with peer id self.connections[peer_id] = muxed_conn + # Call notifiers since event occurred + for notifee in self.notifees: + notifee.connected(self, muxed_conn) + try: # Success listener = self.transport.create_listener(conn_handler) self.listeners[str(multiaddr)] = listener await listener.listen(multiaddr) + + # Call notifiers since event occurred + for notifee in self.notifees: + notifee.listen(self, multiaddr) + return True except IOError: # Failed. Continue looping. @@ -150,6 +171,13 @@ class Swarm(INetwork): # No multiaddr succeeded return False + def notify(self, notifee): + """ + :param notifee: object implementing Notifee interface + """ + # TODO: Add check to ensure notifee conforms to Notifee interface + self.notifees.append(notifee) + def add_transport(self, transport): # TODO: Support more than one transport self.transport = transport From 5a4af0c81c1a6b8c3544693bf1d86ad9c67454d3 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 19:08:13 -0500 Subject: [PATCH 030/131] Add tests for notify --- libp2p/network/notifee_interface.py | 1 - tests/libp2p/test_notify.py | 176 ++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 tests/libp2p/test_notify.py diff --git a/libp2p/network/notifee_interface.py b/libp2p/network/notifee_interface.py index fbc61f4..4ce9ed5 100644 --- a/libp2p/network/notifee_interface.py +++ b/libp2p/network/notifee_interface.py @@ -1,6 +1,5 @@ from abc import ABC, abstractmethod - class INotifee(ABC): @abstractmethod diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py new file mode 100644 index 0000000..1df267c --- /dev/null +++ b/tests/libp2p/test_notify.py @@ -0,0 +1,176 @@ +import multiaddr +import pytest, asyncio + +from tests.utils import cleanup +from libp2p import new_node +from libp2p.peer.peerinfo import info_from_p2p_addr +from libp2p.network.notifee_interface import INotifee + +# pylint: disable=too-many-locals + +""" +Test Notify and Notifee by ensuring that the proper events get +called at the proper time, and that the stream passed into +opened_stream is correct + +Note: Listen event does not get hit because MyNotifee is passed +into network after network has already started listening + +TODO: Add tests to ensure conn is the correct connection +TODO: Add tests for closed_stream disconnected, listen_close as those +features are implemented in network +passed into connected +""" +class MyNotifee(INotifee): + # pylint: disable=too-many-instance-attributes, cell-var-from-loop + + def __init__(self, events, val_to_append_to_event): + self.events = events + self.val_to_append_to_event = val_to_append_to_event + + def opened_stream(self, network, stream): + self.events.append(["opened_stream" + \ + self.val_to_append_to_event, stream]) + + def closed_stream(self, network, stream): + pass + + def connected(self, network, conn): + self.events.append("connected" + self.val_to_append_to_event) + + def disconnected(self, network, conn): + pass + + def listen(self, network, multiaddr): + pass + + def listen_close(self, network, multiaddr): + pass + +@pytest.mark.asyncio +async def test_one_notifier(): + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + async def stream_handler(stream): + while True: + read_string = (await stream.read()).decode() + + response = "ack:" + read_string + await stream.write(response.encode()) + + node_b.set_stream_handler("/echo/1.0.0", stream_handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + + # Add notifee for node_a + events = [] + node_a.get_network().notify(MyNotifee(events, "0")) + + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # Ensure the connected and opened_stream events were hit in MyNotifee obj + # and that stream passed into opened_stream matches the stream created on + # node_a + assert events == ["connected0", ["opened_stream0", stream]] + + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() + +@pytest.mark.asyncio +async def test_two_notifiers(): + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + async def stream_handler(stream): + while True: + read_string = (await stream.read()).decode() + + response = "ack:" + read_string + await stream.write(response.encode()) + + node_b.set_stream_handler("/echo/1.0.0", stream_handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + + # Add notifee for node_a + events0 = [] + node_a.get_network().notify(MyNotifee(events0, "0")) + + events1 = [] + node_a.get_network().notify(MyNotifee(events1, "1")) + + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # Ensure the connected and opened_stream events were hit in both Notifee objs + # and that the stream passed into opened_stream matches the stream created on + # node_a + assert events0 == ["connected0", ["opened_stream0", stream]] + assert events1 == ["connected1", ["opened_stream1", stream]] + + + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() + +@pytest.mark.asyncio +async def test_ten_notifiers(): + num_notifiers = 10 + + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + async def stream_handler(stream): + while True: + read_string = (await stream.read()).decode() + + response = "ack:" + read_string + await stream.write(response.encode()) + + node_b.set_stream_handler("/echo/1.0.0", stream_handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + + # Add notifee for node_a + events_lst = [] + for i in range(num_notifiers): + events_lst.append([]) + node_a.get_network().notify(MyNotifee(events_lst[i], str(i))) + + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # Ensure the connected and opened_stream events were hit in both Notifee objs + # and that the stream passed into opened_stream matches the stream created on + # node_a + for i in range(num_notifiers): + assert events_lst[i] == ["connected" + str(i), \ + ["opened_stream" + str(i), stream]] + + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() \ No newline at end of file From 0d2aa3f5b1873b9134b6a44969a552f993ba867f Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 19:11:04 -0500 Subject: [PATCH 031/131] Make notifee functions all async --- libp2p/network/notifee_interface.py | 13 +++++++------ libp2p/network/swarm.py | 8 ++++---- tests/libp2p/test_notify.py | 12 ++++++------ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/libp2p/network/notifee_interface.py b/libp2p/network/notifee_interface.py index 4ce9ed5..837ecd1 100644 --- a/libp2p/network/notifee_interface.py +++ b/libp2p/network/notifee_interface.py @@ -1,44 +1,45 @@ +import asyncio from abc import ABC, abstractmethod class INotifee(ABC): @abstractmethod - def opened_stream(self, network, stream): + async def opened_stream(self, network, stream): """ :param network: network the stream was opened on :param stream: stream that was opened """ @abstractmethod - def closed_stream(self, network, stream): + async def closed_stream(self, network, stream): """ :param network: network the stream was closed on :param stream: stream that was closed """ @abstractmethod - def connected(self, network, conn): + async def connected(self, network, conn): """ :param network: network the connection was opened on :param conn: connection that was opened """ @abstractmethod - def disconnected(self, network, conn): + async def disconnected(self, network, conn): """ :param network: network the connection was closed on :param conn: connection that was closed """ @abstractmethod - def listen(self, network, multiaddr): + async def listen(self, network, multiaddr): """ :param network: network the listener is listening on :param multiaddr: multiaddress listener is listening on """ @abstractmethod - def listen_close(self, network, multiaddr): + async def listen_close(self, network, multiaddr): """ :param network: network the connection was opened on :param multiaddr: multiaddress listener is no longer listening on diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 1ecc0df..e6bf288 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -76,7 +76,7 @@ class Swarm(INetwork): # Call notifiers since event occurred for notifee in self.notifees: - notifee.connected(self, muxed_conn) + await notifee.connected(self, muxed_conn) return muxed_conn @@ -110,7 +110,7 @@ class Swarm(INetwork): # Call notifiers since event occurred for notifee in self.notifees: - notifee.opened_stream(self, net_stream) + await notifee.opened_stream(self, net_stream) return net_stream @@ -151,7 +151,7 @@ class Swarm(INetwork): # Call notifiers since event occurred for notifee in self.notifees: - notifee.connected(self, muxed_conn) + await notifee.connected(self, muxed_conn) try: # Success @@ -161,7 +161,7 @@ class Swarm(INetwork): # Call notifiers since event occurred for notifee in self.notifees: - notifee.listen(self, multiaddr) + await notifee.listen(self, multiaddr) return True except IOError: diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 1df267c..05b2c4f 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -28,23 +28,23 @@ class MyNotifee(INotifee): self.events = events self.val_to_append_to_event = val_to_append_to_event - def opened_stream(self, network, stream): + async def opened_stream(self, network, stream): self.events.append(["opened_stream" + \ self.val_to_append_to_event, stream]) - def closed_stream(self, network, stream): + async def closed_stream(self, network, stream): pass - def connected(self, network, conn): + async def connected(self, network, conn): self.events.append("connected" + self.val_to_append_to_event) - def disconnected(self, network, conn): + async def disconnected(self, network, conn): pass - def listen(self, network, multiaddr): + async def listen(self, network, multiaddr): pass - def listen_close(self, network, multiaddr): + async def listen_close(self, network, multiaddr): pass @pytest.mark.asyncio From a1c0165484c96c5e1ad55158f2169f7916ff9521 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 19:13:26 -0500 Subject: [PATCH 032/131] Fix linting issue --- libp2p/network/notifee_interface.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libp2p/network/notifee_interface.py b/libp2p/network/notifee_interface.py index 837ecd1..17ba1fe 100644 --- a/libp2p/network/notifee_interface.py +++ b/libp2p/network/notifee_interface.py @@ -1,4 +1,3 @@ -import asyncio from abc import ABC, abstractmethod class INotifee(ABC): From 86b4bb5e55dcdfce5e3d06196e2ad8632ff7f146 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 19:17:11 -0500 Subject: [PATCH 033/131] Fix linting issue --- tests/libp2p/test_notify.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 05b2c4f..e9a62cd 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -1,26 +1,23 @@ -import multiaddr -import pytest, asyncio +import pytest from tests.utils import cleanup from libp2p import new_node -from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.network.notifee_interface import INotifee # pylint: disable=too-many-locals """ Test Notify and Notifee by ensuring that the proper events get -called at the proper time, and that the stream passed into -opened_stream is correct +called, and that the stream passed into opened_stream is correct Note: Listen event does not get hit because MyNotifee is passed into network after network has already started listening TODO: Add tests to ensure conn is the correct connection -TODO: Add tests for closed_stream disconnected, listen_close as those -features are implemented in network -passed into connected +TODO: Add tests for closed_stream disconnected, listen_close when those +features are implemented in swarm """ + class MyNotifee(INotifee): # pylint: disable=too-many-instance-attributes, cell-var-from-loop @@ -173,4 +170,4 @@ async def test_ten_notifiers(): assert response == ("ack:" + message) # Success, terminate pending tasks. - await cleanup() \ No newline at end of file + await cleanup() From 91b4d66d50b97c4c68e61e77ff3c16882caf95ec Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Sun, 3 Mar 2019 23:34:49 +1100 Subject: [PATCH 034/131] Add conn to net stream and conn tests --- libp2p/network/stream/net_stream.py | 1 + tests/libp2p/test_notify.py | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/libp2p/network/stream/net_stream.py b/libp2p/network/stream/net_stream.py index 8cfb635..f708962 100644 --- a/libp2p/network/stream/net_stream.py +++ b/libp2p/network/stream/net_stream.py @@ -5,6 +5,7 @@ class NetStream(INetStream): def __init__(self, muxed_stream): self.muxed_stream = muxed_stream + self.mplex_conn = muxed_stream.mplex_conn self.protocol_id = None def get_protocol(self): diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index e9a62cd..3021cf1 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -33,7 +33,8 @@ class MyNotifee(INotifee): pass async def connected(self, network, conn): - self.events.append("connected" + self.val_to_append_to_event) + self.events.append(["connected" + self.val_to_append_to_event,\ + conn]) async def disconnected(self, network, conn): pass @@ -70,7 +71,8 @@ async def test_one_notifier(): # Ensure the connected and opened_stream events were hit in MyNotifee obj # and that stream passed into opened_stream matches the stream created on # node_a - assert events == ["connected0", ["opened_stream0", stream]] + assert events == [["connected0", stream.mplex_conn], \ + ["opened_stream0", stream]] messages = ["hello", "hello"] for message in messages: @@ -112,8 +114,8 @@ async def test_two_notifiers(): # Ensure the connected and opened_stream events were hit in both Notifee objs # and that the stream passed into opened_stream matches the stream created on # node_a - assert events0 == ["connected0", ["opened_stream0", stream]] - assert events1 == ["connected1", ["opened_stream1", stream]] + assert events0 == [["connected0", stream.mplex_conn], ["opened_stream0", stream]] + assert events1 == [["connected1", stream.mplex_conn], ["opened_stream1", stream]] messages = ["hello", "hello"] @@ -158,7 +160,7 @@ async def test_ten_notifiers(): # and that the stream passed into opened_stream matches the stream created on # node_a for i in range(num_notifiers): - assert events_lst[i] == ["connected" + str(i), \ + assert events_lst[i] == [["connected" + str(i), stream.mplex_conn], \ ["opened_stream" + str(i), stream]] messages = ["hello", "hello"] From b340b8981985a906bbb97b9a080d9688650425b1 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Sun, 3 Mar 2019 23:44:15 +1100 Subject: [PATCH 035/131] Refactor test setup to remove duplicate code --- tests/libp2p/test_notify.py | 38 ++++++++----------------------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 3021cf1..80dd6c1 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -45,8 +45,7 @@ class MyNotifee(INotifee): async def listen_close(self, network, multiaddr): pass -@pytest.mark.asyncio -async def test_one_notifier(): +async def perform_two_host_simple_set_up(): node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -61,6 +60,11 @@ async def test_one_notifier(): # Associate the peer with local ip address (see default parameters of Libp2p()) node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + return node_a, node_b + +@pytest.mark.asyncio +async def test_one_notifier(): + node_a, node_b = await perform_two_host_simple_set_up() # Add notifee for node_a events = [] @@ -87,20 +91,7 @@ async def test_one_notifier(): @pytest.mark.asyncio async def test_two_notifiers(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - - async def stream_handler(stream): - while True: - read_string = (await stream.read()).decode() - - response = "ack:" + read_string - await stream.write(response.encode()) - - node_b.set_stream_handler("/echo/1.0.0", stream_handler) - - # Associate the peer with local ip address (see default parameters of Libp2p()) - node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + node_a, node_b = await perform_two_host_simple_set_up() # Add notifee for node_a events0 = [] @@ -133,20 +124,7 @@ async def test_two_notifiers(): async def test_ten_notifiers(): num_notifiers = 10 - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - - async def stream_handler(stream): - while True: - read_string = (await stream.read()).decode() - - response = "ack:" + read_string - await stream.write(response.encode()) - - node_b.set_stream_handler("/echo/1.0.0", stream_handler) - - # Associate the peer with local ip address (see default parameters of Libp2p()) - node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + node_a, node_b = await perform_two_host_simple_set_up() # Add notifee for node_a events_lst = [] From 15883fcf096513b848f218cbf22d8d4c7a65cc15 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Sun, 3 Mar 2019 23:49:28 +1100 Subject: [PATCH 036/131] Fix linting issues --- tests/libp2p/test_notify.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 80dd6c1..5ebed1b 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -49,14 +49,14 @@ async def perform_two_host_simple_set_up(): node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - async def stream_handler(stream): + async def my_stream_handler(stream): while True: read_string = (await stream.read()).decode() - response = "ack:" + read_string - await stream.write(response.encode()) + resp = "ack:" + read_string + await stream.write(resp.encode()) - node_b.set_stream_handler("/echo/1.0.0", stream_handler) + node_b.set_stream_handler("/echo/1.0.0", my_stream_handler) # Associate the peer with local ip address (see default parameters of Libp2p()) node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) From ef827f90ad190a6d22c54e6d434b5903bc8da6b6 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Sun, 3 Mar 2019 23:54:04 +1100 Subject: [PATCH 037/131] Fix linting issue --- tests/libp2p/test_notify.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 5ebed1b..fdf7368 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -1,11 +1,3 @@ -import pytest - -from tests.utils import cleanup -from libp2p import new_node -from libp2p.network.notifee_interface import INotifee - -# pylint: disable=too-many-locals - """ Test Notify and Notifee by ensuring that the proper events get called, and that the stream passed into opened_stream is correct @@ -13,11 +5,18 @@ called, and that the stream passed into opened_stream is correct Note: Listen event does not get hit because MyNotifee is passed into network after network has already started listening -TODO: Add tests to ensure conn is the correct connection TODO: Add tests for closed_stream disconnected, listen_close when those features are implemented in swarm """ +import pytest + +from tests.utils import cleanup +from libp2p import new_node +from libp2p.network.notifee_interface import INotifee + +# pylint: disable=too-many-locals + class MyNotifee(INotifee): # pylint: disable=too-many-instance-attributes, cell-var-from-loop From ce22f06773c6b1b7fbadc0aada6a288548b0e123 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Sun, 3 Mar 2019 23:57:20 +1100 Subject: [PATCH 038/131] Fix linting issue --- libp2p/protocol_muxer/multiselect_client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libp2p/protocol_muxer/multiselect_client.py b/libp2p/protocol_muxer/multiselect_client.py index 8e5463a..4cc07d9 100644 --- a/libp2p/protocol_muxer/multiselect_client.py +++ b/libp2p/protocol_muxer/multiselect_client.py @@ -104,9 +104,8 @@ class MultiselectClient(IMultiselectClient): if response == protocol: return protocol if response == PROTOCOL_NOT_FOUND_MSG: - raise MultiselectClientError("protocol not supported") - else: - raise MultiselectClientError("unrecognized response: " + response) + raise MultiselectClientError("protocol not supported") + raise MultiselectClientError("unrecognized response: " + response) def validate_handshake(handshake_contents): From 1384cebf9cf04f2c381f4288bfc95ab596c67e98 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Sun, 3 Mar 2019 23:59:38 +1100 Subject: [PATCH 039/131] Fix outstanding unrelated lint issue in multiselect_client --- libp2p/protocol_muxer/multiselect_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libp2p/protocol_muxer/multiselect_client.py b/libp2p/protocol_muxer/multiselect_client.py index 4cc07d9..dec5493 100644 --- a/libp2p/protocol_muxer/multiselect_client.py +++ b/libp2p/protocol_muxer/multiselect_client.py @@ -104,7 +104,7 @@ class MultiselectClient(IMultiselectClient): if response == protocol: return protocol if response == PROTOCOL_NOT_FOUND_MSG: - raise MultiselectClientError("protocol not supported") + raise MultiselectClientError("protocol not supported") raise MultiselectClientError("unrecognized response: " + response) From d800b3ef41bdcb12ceff35ba2078379e6bdd5fcc Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 14 Mar 2019 12:47:40 -0400 Subject: [PATCH 040/131] Add opened_stream call for non-initiator --- libp2p/network/swarm.py | 9 +++- tests/libp2p/test_notify.py | 100 ++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index e6bf288..daabb64 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -5,6 +5,7 @@ from libp2p.protocol_muxer.multiselect import Multiselect from libp2p.peer.id import id_b58_decode from .network_interface import INetwork +from .notifee_interface import INotifee from .stream.net_stream import NetStream from .connection.raw_connection import RawConnection @@ -175,8 +176,8 @@ class Swarm(INetwork): """ :param notifee: object implementing Notifee interface """ - # TODO: Add check to ensure notifee conforms to Notifee interface - self.notifees.append(notifee) + if isinstance(notifee, INotifee): + self.notifees.append(notifee) def add_transport(self, transport): # TODO: Support more than one transport @@ -195,6 +196,10 @@ def create_generic_protocol_handler(swarm): # Perform protocol muxing to determine protocol to use _, handler = await multiselect.negotiate(muxed_stream) + # Call notifiers since event occurred + for notifee in swarm.notifees: + await notifee.opened_stream(swarm, muxed_stream) + # Give to stream handler asyncio.ensure_future(handler(muxed_stream)) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index fdf7368..37b287a 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -44,6 +44,27 @@ class MyNotifee(INotifee): async def listen_close(self, network, multiaddr): pass +class InvalidNotifee(): + # pylint: disable=too-many-instance-attributes, cell-var-from-loop + + def __init__(self): + pass + + async def opened_stream(self, network, stream): + assert False + + async def closed_stream(self, network, stream): + assert False + + async def connected(self, network, conn): + assert False + + async def disconnected(self, network, conn): + assert False + + async def listen(self, network, multiaddr): + assert False + async def perform_two_host_simple_set_up(): node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -61,6 +82,16 @@ async def perform_two_host_simple_set_up(): node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) return node_a, node_b +async def perform_two_host_simple_set_up_custom_handler(handler): + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + node_b.set_stream_handler("/echo/1.0.0", handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + return node_a, node_b + @pytest.mark.asyncio async def test_one_notifier(): node_a, node_b = await perform_two_host_simple_set_up() @@ -88,6 +119,47 @@ async def test_one_notifier(): # Success, terminate pending tasks. await cleanup() +@pytest.mark.asyncio +async def test_one_notifier_on_two_nodes(): + events_b = [] + + async def my_stream_handler(stream): + assert events_b == [["connectedb", stream.mplex_conn], \ + ["opened_streamb", stream]] + while True: + read_string = (await stream.read()).decode() + + resp = "ack:" + read_string + await stream.write(resp.encode()) + + node_a, node_b = await perform_two_host_simple_set_up_custom_handler(my_stream_handler) + + # Add notifee for node_a + events_a = [] + node_a.get_network().notify(MyNotifee(events_a, "a")) + + # Add notifee for node_b + node_b.get_network().notify(MyNotifee(events_b, "b")) + + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # Ensure the connected and opened_stream events were hit in MyNotifee obj + # and that stream passed into opened_stream matches the stream created on + # node_a + assert events_a == [["connecteda", stream.mplex_conn], \ + ["opened_streama", stream]] + + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() + @pytest.mark.asyncio async def test_two_notifiers(): node_a, node_b = await perform_two_host_simple_set_up() @@ -150,3 +222,31 @@ async def test_ten_notifiers(): # Success, terminate pending tasks. await cleanup() + +@pytest.mark.asyncio +async def test_invalid_notifee(): + num_notifiers = 10 + + node_a, node_b = await perform_two_host_simple_set_up() + + # Add notifee for node_a + events_lst = [] + for i in range(num_notifiers): + events_lst.append([]) + node_a.get_network().notify(InvalidNotifee()) + + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # If this point is reached, this implies that the InvalidNotifee instance + # did not assert false, i.e. no functions of InvalidNotifee were called (which is correct + # given that InvalidNotifee should not have been added as a notifee) + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() From 7d45131f376917f38f60c33bcf3be3c19343c108 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 14 Mar 2019 14:01:37 -0400 Subject: [PATCH 041/131] Add return value to Notify --- libp2p/network/network_interface.py | 1 + libp2p/network/swarm.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/libp2p/network/network_interface.py b/libp2p/network/network_interface.py index acd20b1..71e813c 100644 --- a/libp2p/network/network_interface.py +++ b/libp2p/network/network_interface.py @@ -46,4 +46,5 @@ class INetwork(ABC): def notify(self, notifee): """ :param notifee: object implementing Notifee interface + :return: true if notifee registered successfully, false otherwise """ diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index daabb64..f2943ba 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -175,9 +175,12 @@ class Swarm(INetwork): def notify(self, notifee): """ :param notifee: object implementing Notifee interface + :return: true if notifee registered successfully, false otherwise """ if isinstance(notifee, INotifee): self.notifees.append(notifee) + return True + return False def add_transport(self, transport): # TODO: Support more than one transport From 51cd9c8e5654338dc779348f1b07164a887fc8fb Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 14 Mar 2019 14:01:59 -0400 Subject: [PATCH 042/131] Add additional initiator and non-initiator notifee tests --- tests/libp2p/test_notify.py | 79 ++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 37b287a..108f04a 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -50,19 +50,19 @@ class InvalidNotifee(): def __init__(self): pass - async def opened_stream(self, network, stream): + async def opened_stream(self): assert False - async def closed_stream(self, network, stream): + async def closed_stream(self): assert False - async def connected(self, network, conn): + async def connected(self): assert False - async def disconnected(self, network, conn): + async def disconnected(self): assert False - async def listen(self, network, multiaddr): + async def listen(self): assert False async def perform_two_host_simple_set_up(): @@ -98,7 +98,7 @@ async def test_one_notifier(): # Add notifee for node_a events = [] - node_a.get_network().notify(MyNotifee(events, "0")) + assert node_a.get_network().notify(MyNotifee(events, "0")) stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) @@ -122,8 +122,11 @@ async def test_one_notifier(): @pytest.mark.asyncio async def test_one_notifier_on_two_nodes(): events_b = [] - + async def my_stream_handler(stream): + # Ensure the connected and opened_stream events were hit in Notifee obj + # and that the stream passed into opened_stream matches the stream created on + # node_b assert events_b == [["connectedb", stream.mplex_conn], \ ["opened_streamb", stream]] while True: @@ -136,10 +139,10 @@ async def test_one_notifier_on_two_nodes(): # Add notifee for node_a events_a = [] - node_a.get_network().notify(MyNotifee(events_a, "a")) + assert node_a.get_network().notify(MyNotifee(events_a, "a")) # Add notifee for node_b - node_b.get_network().notify(MyNotifee(events_b, "b")) + assert node_b.get_network().notify(MyNotifee(events_b, "b")) stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) @@ -166,10 +169,10 @@ async def test_two_notifiers(): # Add notifee for node_a events0 = [] - node_a.get_network().notify(MyNotifee(events0, "0")) + assert node_a.get_network().notify(MyNotifee(events0, "0")) events1 = [] - node_a.get_network().notify(MyNotifee(events1, "1")) + assert node_a.get_network().notify(MyNotifee(events1, "1")) stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) @@ -201,7 +204,7 @@ async def test_ten_notifiers(): events_lst = [] for i in range(num_notifiers): events_lst.append([]) - node_a.get_network().notify(MyNotifee(events_lst[i], str(i))) + assert node_a.get_network().notify(MyNotifee(events_lst[i], str(i))) stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) @@ -223,6 +226,54 @@ async def test_ten_notifiers(): # Success, terminate pending tasks. await cleanup() +@pytest.mark.asyncio +async def test_ten_notifiers_on_two_nodes(): + num_notifiers = 10 + events_lst_b = [] + + async def my_stream_handler(stream): + # Ensure the connected and opened_stream events were hit in all Notifee objs + # and that the stream passed into opened_stream matches the stream created on + # node_b + for i in range(num_notifiers): + assert events_lst_b[i] == [["connectedb" + str(i), stream.mplex_conn], \ + ["opened_streamb" + str(i), stream]] + while True: + read_string = (await stream.read()).decode() + + resp = "ack:" + read_string + await stream.write(resp.encode()) + + node_a, node_b = await perform_two_host_simple_set_up_custom_handler(my_stream_handler) + + # Add notifee for node_a and node_b + events_lst_a = [] + for i in range(num_notifiers): + events_lst_a.append([]) + events_lst_b.append([]) + assert node_a.get_network().notify(MyNotifee(events_lst_a[i], "a" + str(i))) + assert node_b.get_network().notify(MyNotifee(events_lst_b[i], "b" + str(i))) + + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # Ensure the connected and opened_stream events were hit in all Notifee objs + # and that the stream passed into opened_stream matches the stream created on + # node_a + for i in range(num_notifiers): + assert events_lst_a[i] == [["connecteda" + str(i), stream.mplex_conn], \ + ["opened_streama" + str(i), stream]] + + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() + @pytest.mark.asyncio async def test_invalid_notifee(): num_notifiers = 10 @@ -231,9 +282,9 @@ async def test_invalid_notifee(): # Add notifee for node_a events_lst = [] - for i in range(num_notifiers): + for _ in range(num_notifiers): events_lst.append([]) - node_a.get_network().notify(InvalidNotifee()) + assert not node_a.get_network().notify(InvalidNotifee()) stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) From 474602c4f72bc6fd6ba17b17ca7303bc08e47e9d Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 17 Mar 2019 19:20:46 -0400 Subject: [PATCH 043/131] refactor new_node --- libp2p/__init__.py | 60 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/libp2p/__init__.py b/libp2p/__init__.py index 0fa2fdb..d83ec3c 100644 --- a/libp2p/__init__.py +++ b/libp2p/__init__.py @@ -11,6 +11,9 @@ from .transport.tcp.tcp import TCP async def cleanup_done_tasks(): + """ + clean up asyncio done tasks to free up resources + """ while True: for task in asyncio.all_tasks(): if task.done(): @@ -20,30 +23,59 @@ async def cleanup_done_tasks(): # Some sleep necessary to context switch await asyncio.sleep(3) -async def new_node( +def initialize_default_swarm( id_opt=None, transport_opt=None, - muxer_opt=None, sec_opt=None, peerstore=None): - - if id_opt is None: + muxer_opt=None, sec_opt=None, peerstore_opt=None): + """ + initialize swarm when no swarm is passed in + :param id_opt: optional id for host + :param transport_opt: optional choice of transport upgrade + :param muxer_opt: optional choice of stream muxer + :param sec_opt: optional choice of security upgrade + :param peerstore_opt: optional peerstore + :return: return a default swarm instance + """ + # pylint: disable=too-many-arguments + if not id_opt: new_key = RSA.generate(2048, e=65537) id_opt = id_from_public_key(new_key.publickey()) # private_key = new_key.exportKey("PEM") transport_opt = transport_opt or ["/ip4/127.0.0.1/tcp/8001"] - transport_opt = [multiaddr.Multiaddr(t) for t in transport_opt] - muxer_opt = muxer_opt or ["mplex/6.7.0"] - sec_opt = sec_opt or ["secio"] - peerstore = peerstore or PeerStore() - - upgrader = TransportUpgrader(sec_opt, transport_opt) - swarm = Swarm(id_opt, peerstore, upgrader) + transport = [multiaddr.Multiaddr(t) for t in transport_opt] + # TODO wire muxer up with swarm + # muxer = muxer_opt or ["mplex/6.7.0"] + sec = sec_opt or ["secio"] + peerstore = peerstore_opt or PeerStore() + upgrader = TransportUpgrader(sec, transport) + swarm_opt = Swarm(id_opt, peerstore, upgrader) tcp = TCP() - swarm.add_transport(tcp) - await swarm.listen(transport_opt[0]) + swarm_opt.add_transport(tcp) + + return swarm_opt + +async def new_node( + swarm_opt=None, id_opt=None, transport_opt=None, + muxer_opt=None, sec_opt=None, peerstore_opt=None): + """ + create new libp2p node + :param id_opt: optional id for host + :param transport_opt: optional choice of transport upgrade + :param muxer_opt: optional choice of stream muxer + :param sec_opt: optional choice of security upgrade + :param peerstore_opt: optional peerstore + :return: return a default swarm instance + """ + # pylint: disable=too-many-arguments + if not swarm_opt: + swarm_opt = initialize_default_swarm( + id_opt=id_opt, transport_opt=transport_opt, + muxer_opt=muxer_opt, sec_opt=sec_opt, + peerstore_opt=peerstore_opt) # TODO enable support for other host type # TODO routing unimplemented - host = BasicHost(swarm) + host = BasicHost(swarm_opt) # Kick off cleanup job asyncio.ensure_future(cleanup_done_tasks()) From 9aa41b106aa014b41a27b9b062ad3092d80c76db Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 17 Mar 2019 19:33:10 -0400 Subject: [PATCH 044/131] refactor host setup helper --- tests/utils.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index c995412..e7f9b85 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,8 @@ -import asyncio - from contextlib import suppress +import asyncio +import multiaddr + +from libp2p import new_node async def cleanup(): @@ -12,3 +14,18 @@ async def cleanup(): # Cancelled task raises asyncio.CancelledError that we can suppress: with suppress(asyncio.CancelledError): await task + +async def set_up_nodes_by_transport_opt(transport_opt_list): + nodes_list = [] + for transport_opt in transport_opt_list: + node = await new_node(transport_opt=transport_opt) + await node.get_network().listen(multiaddr.Multiaddr(transport_opt[0])) + nodes_list.append(node) + return tuple(nodes_list) + +async def echo_stream_handler(stream): + while True: + read_string = (await stream.read()).decode() + + resp = "ack:" + read_string + await stream.write(resp.encode()) From 3e0d6ea126617026cf87149071991d47f7a0fe1c Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 17 Mar 2019 19:33:40 -0400 Subject: [PATCH 045/131] fix existing tests --- tests/examples/test_chat.py | 8 ++--- tests/libp2p/test_libp2p.py | 35 ++++++++++--------- tests/libp2p/test_notify.py | 37 ++++++--------------- tests/protocol_muxer/test_protocol_muxer.py | 10 ++---- 4 files changed, 34 insertions(+), 56 deletions(-) diff --git a/tests/examples/test_chat.py b/tests/examples/test_chat.py index 0ea1979..ee192cc 100644 --- a/tests/examples/test_chat.py +++ b/tests/examples/test_chat.py @@ -1,7 +1,8 @@ import pytest import asyncio +import multiaddr -from tests.utils import cleanup +from tests.utils import cleanup, set_up_nodes_by_transport_opt from libp2p import new_node from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.protocol_muxer.multiselect_client import MultiselectClientError @@ -9,7 +10,6 @@ from libp2p.protocol_muxer.multiselect_client import MultiselectClientError PROTOCOL_ID = '/chat/1.0.0' - async def hello_world(host_a, host_b): async def stream_handler(stream): read = await stream.read() @@ -100,8 +100,8 @@ async def no_common_protocol(host_a, host_b): (no_common_protocol), ]) async def test_chat(test): - host_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - host_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (host_a, host_b) = await set_up_nodes_by_transport_opt(transport_opt_list) addr = host_a.get_addrs()[0] info = info_from_p2p_addr(addr) diff --git a/tests/libp2p/test_libp2p.py b/tests/libp2p/test_libp2p.py index 4b7e59c..3d74516 100644 --- a/tests/libp2p/test_libp2p.py +++ b/tests/libp2p/test_libp2p.py @@ -1,17 +1,16 @@ import multiaddr import pytest -from tests.utils import cleanup +from tests.utils import cleanup, set_up_nodes_by_transport_opt from libp2p import new_node from libp2p.peer.peerinfo import info_from_p2p_addr + # pylint: disable=too-many-locals - - @pytest.mark.asyncio async def test_simple_messages(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler(stream): while True: @@ -41,8 +40,8 @@ async def test_simple_messages(): @pytest.mark.asyncio async def test_double_response(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler(stream): while True: @@ -78,8 +77,8 @@ async def test_double_response(): async def test_multiple_streams(): # Node A should be able to open a stream with node B and then vice versa. # Stream IDs should be generated uniquely so that the stream state is not overwritten - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler_a(stream): while True: @@ -124,8 +123,8 @@ async def test_multiple_streams(): @pytest.mark.asyncio async def test_multiple_streams_same_initiator_different_protocols(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler_a1(stream): while True: @@ -184,8 +183,8 @@ async def test_multiple_streams_same_initiator_different_protocols(): @pytest.mark.asyncio async def test_multiple_streams_two_initiators(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler_a1(stream): while True: @@ -262,9 +261,9 @@ async def test_multiple_streams_two_initiators(): @pytest.mark.asyncio async def test_triangle_nodes_connection(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_c = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"],\ + ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b, node_c) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler(stream): while True: @@ -315,8 +314,8 @@ async def test_triangle_nodes_connection(): @pytest.mark.asyncio async def test_host_connect(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) assert not node_a.get_peerstore().peers() diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 108f04a..a2fecaf 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -11,7 +11,7 @@ features are implemented in swarm import pytest -from tests.utils import cleanup +from tests.utils import * from libp2p import new_node from libp2p.network.notifee_interface import INotifee @@ -65,26 +65,9 @@ class InvalidNotifee(): async def listen(self): assert False -async def perform_two_host_simple_set_up(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - - async def my_stream_handler(stream): - while True: - read_string = (await stream.read()).decode() - - resp = "ack:" + read_string - await stream.write(resp.encode()) - - node_b.set_stream_handler("/echo/1.0.0", my_stream_handler) - - # Associate the peer with local ip address (see default parameters of Libp2p()) - node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) - return node_a, node_b - -async def perform_two_host_simple_set_up_custom_handler(handler): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) +async def perform_two_host_set_up_custom_handler(handler): + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) node_b.set_stream_handler("/echo/1.0.0", handler) @@ -94,7 +77,7 @@ async def perform_two_host_simple_set_up_custom_handler(handler): @pytest.mark.asyncio async def test_one_notifier(): - node_a, node_b = await perform_two_host_simple_set_up() + node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) # Add notifee for node_a events = [] @@ -135,7 +118,7 @@ async def test_one_notifier_on_two_nodes(): resp = "ack:" + read_string await stream.write(resp.encode()) - node_a, node_b = await perform_two_host_simple_set_up_custom_handler(my_stream_handler) + node_a, node_b = await perform_two_host_set_up_custom_handler(my_stream_handler) # Add notifee for node_a events_a = [] @@ -165,7 +148,7 @@ async def test_one_notifier_on_two_nodes(): @pytest.mark.asyncio async def test_two_notifiers(): - node_a, node_b = await perform_two_host_simple_set_up() + node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) # Add notifee for node_a events0 = [] @@ -198,7 +181,7 @@ async def test_two_notifiers(): async def test_ten_notifiers(): num_notifiers = 10 - node_a, node_b = await perform_two_host_simple_set_up() + node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) # Add notifee for node_a events_lst = [] @@ -244,7 +227,7 @@ async def test_ten_notifiers_on_two_nodes(): resp = "ack:" + read_string await stream.write(resp.encode()) - node_a, node_b = await perform_two_host_simple_set_up_custom_handler(my_stream_handler) + node_a, node_b = await perform_two_host_set_up_custom_handler(my_stream_handler) # Add notifee for node_a and node_b events_lst_a = [] @@ -278,7 +261,7 @@ async def test_ten_notifiers_on_two_nodes(): async def test_invalid_notifee(): num_notifiers = 10 - node_a, node_b = await perform_two_host_simple_set_up() + node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) # Add notifee for node_a events_lst = [] diff --git a/tests/protocol_muxer/test_protocol_muxer.py b/tests/protocol_muxer/test_protocol_muxer.py index a7e19af..1eff211 100644 --- a/tests/protocol_muxer/test_protocol_muxer.py +++ b/tests/protocol_muxer/test_protocol_muxer.py @@ -1,6 +1,6 @@ import pytest -from tests.utils import cleanup +from tests.utils import cleanup, set_up_nodes_by_transport_opt from libp2p import new_node from libp2p.protocol_muxer.multiselect_client import MultiselectClientError @@ -15,12 +15,8 @@ from libp2p.protocol_muxer.multiselect_client import MultiselectClientError async def perform_simple_test(expected_selected_protocol, protocols_for_client, protocols_with_handlers): - transport_opt_a = ["/ip4/127.0.0.1/tcp/0"] - transport_opt_b = ["/ip4/127.0.0.1/tcp/0"] - node_a = await new_node( - transport_opt=transport_opt_a) - node_b = await new_node( - transport_opt=transport_opt_b) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler(stream): while True: From cc7c08bac4961526ce20ca0ba72a9724ad693423 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 17 Mar 2019 21:15:14 -0400 Subject: [PATCH 046/131] add test for listen event --- tests/libp2p/test_notify.py | 79 +++++++++++++++++++++++++++++++------ tests/utils.py | 10 +++++ 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index a2fecaf..edaba2f 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -12,8 +12,9 @@ features are implemented in swarm import pytest from tests.utils import * -from libp2p import new_node +from libp2p import new_node, initialize_default_swarm from libp2p.network.notifee_interface import INotifee +from libp2p.host.basic_host import BasicHost # pylint: disable=too-many-locals @@ -39,7 +40,8 @@ class MyNotifee(INotifee): pass async def listen(self, network, multiaddr): - pass + self.events.append(["listened" + self.val_to_append_to_event,\ + multiaddr]) async def listen_close(self, network, multiaddr): pass @@ -65,16 +67,6 @@ class InvalidNotifee(): async def listen(self): assert False -async def perform_two_host_set_up_custom_handler(handler): - transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] - (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) - - node_b.set_stream_handler("/echo/1.0.0", handler) - - # Associate the peer with local ip address (see default parameters of Libp2p()) - node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) - return node_a, node_b - @pytest.mark.asyncio async def test_one_notifier(): node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) @@ -146,6 +138,69 @@ async def test_one_notifier_on_two_nodes(): # Success, terminate pending tasks. await cleanup() +@pytest.mark.asyncio +async def test_one_notifier_on_two_nodes_with_listen(): + events_b = [] + + node_a_transport_opt = ["/ip4/127.0.0.1/tcp/0"] + node_a = await new_node(transport_opt=node_a_transport_opt) + await node_a.get_network().listen(multiaddr.Multiaddr(node_a_transport_opt[0])) + + # Set up node_b swarm to pass into host + node_b_transport_opt = ["/ip4/127.0.0.1/tcp/0"] + node_b_multiaddr = multiaddr.Multiaddr(node_b_transport_opt[0]) + node_b_swarm = initialize_default_swarm(transport_opt=node_b_transport_opt) + node_b = BasicHost(node_b_swarm) + + async def my_stream_handler(stream): + # Ensure the listened, connected and opened_stream events were hit in Notifee obj + # and that the stream passed into opened_stream matches the stream created on + # node_b + assert events_b == [ + ["listenedb", node_b_multiaddr], \ + ["connectedb", stream.mplex_conn], \ + ["opened_streamb", stream] + ] + while True: + read_string = (await stream.read()).decode() + + resp = "ack:" + read_string + await stream.write(resp.encode()) + + # Add notifee for node_a + events_a = [] + assert node_a.get_network().notify(MyNotifee(events_a, "a")) + + # Add notifee for node_b + assert node_b.get_network().notify(MyNotifee(events_b, "b")) + + # start listen on node_b_swarm + await node_b.get_network().listen(node_b_multiaddr) + + node_b.set_stream_handler("/echo/1.0.0", my_stream_handler) + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # Ensure the connected and opened_stream events were hit in MyNotifee obj + # and that stream passed into opened_stream matches the stream created on + # node_a + assert events_a == [ + ["connecteda", stream.mplex_conn], \ + ["opened_streama", stream] + ] + + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() + @pytest.mark.asyncio async def test_two_notifiers(): node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) diff --git a/tests/utils.py b/tests/utils.py index e7f9b85..28f217e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -29,3 +29,13 @@ async def echo_stream_handler(stream): resp = "ack:" + read_string await stream.write(resp.encode()) + +async def perform_two_host_set_up_custom_handler(handler): + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) + + node_b.set_stream_handler("/echo/1.0.0", handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + return node_a, node_b \ No newline at end of file From fa53c5a866c9bf6712c576cbf6379997e8b258b2 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 17 Mar 2019 21:30:56 -0400 Subject: [PATCH 047/131] fix linting issues --- libp2p/__init__.py | 2 +- tests/libp2p/test_libp2p.py | 3 +-- tests/libp2p/test_notify.py | 11 +++++++---- tests/protocol_muxer/test_protocol_muxer.py | 3 +-- tests/utils.py | 4 ++-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/libp2p/__init__.py b/libp2p/__init__.py index d83ec3c..62e014b 100644 --- a/libp2p/__init__.py +++ b/libp2p/__init__.py @@ -35,7 +35,7 @@ def initialize_default_swarm( :param peerstore_opt: optional peerstore :return: return a default swarm instance """ - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments, unused-argument if not id_opt: new_key = RSA.generate(2048, e=65537) id_opt = id_from_public_key(new_key.publickey()) diff --git a/tests/libp2p/test_libp2p.py b/tests/libp2p/test_libp2p.py index 3d74516..e9bfb83 100644 --- a/tests/libp2p/test_libp2p.py +++ b/tests/libp2p/test_libp2p.py @@ -2,7 +2,6 @@ import multiaddr import pytest from tests.utils import cleanup, set_up_nodes_by_transport_opt -from libp2p import new_node from libp2p.peer.peerinfo import info_from_p2p_addr @@ -10,7 +9,7 @@ from libp2p.peer.peerinfo import info_from_p2p_addr @pytest.mark.asyncio async def test_simple_messages(): transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] - (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler(stream): while True: diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index edaba2f..71b455b 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -10,8 +10,11 @@ features are implemented in swarm """ import pytest +import multiaddr -from tests.utils import * + +from tests.utils import cleanup, echo_stream_handler, \ + perform_two_host_set_up_custom_handler from libp2p import new_node, initialize_default_swarm from libp2p.network.notifee_interface import INotifee from libp2p.host.basic_host import BasicHost @@ -39,11 +42,11 @@ class MyNotifee(INotifee): async def disconnected(self, network, conn): pass - async def listen(self, network, multiaddr): + async def listen(self, network, _multiaddr): self.events.append(["listened" + self.val_to_append_to_event,\ - multiaddr]) + _multiaddr]) - async def listen_close(self, network, multiaddr): + async def listen_close(self, network, _multiaddr): pass class InvalidNotifee(): diff --git a/tests/protocol_muxer/test_protocol_muxer.py b/tests/protocol_muxer/test_protocol_muxer.py index 1eff211..00949fa 100644 --- a/tests/protocol_muxer/test_protocol_muxer.py +++ b/tests/protocol_muxer/test_protocol_muxer.py @@ -1,7 +1,6 @@ import pytest from tests.utils import cleanup, set_up_nodes_by_transport_opt -from libp2p import new_node from libp2p.protocol_muxer.multiselect_client import MultiselectClientError # TODO: Add tests for multiple streams being opened on different @@ -16,7 +15,7 @@ from libp2p.protocol_muxer.multiselect_client import MultiselectClientError async def perform_simple_test(expected_selected_protocol, protocols_for_client, protocols_with_handlers): transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] - (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler(stream): while True: diff --git a/tests/utils.py b/tests/utils.py index 28f217e..4efde83 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -32,10 +32,10 @@ async def echo_stream_handler(stream): async def perform_two_host_set_up_custom_handler(handler): transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] - (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) node_b.set_stream_handler("/echo/1.0.0", handler) # Associate the peer with local ip address (see default parameters of Libp2p()) node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) - return node_a, node_b \ No newline at end of file + return node_a, node_b From 8d0f40a378e12c1c5e52c163728d1403770baf8d Mon Sep 17 00:00:00 2001 From: stuckinaboot Date: Sat, 23 Mar 2019 13:52:02 -0400 Subject: [PATCH 048/131] [WIP] PubSub and FloodSub development (#133) * Add notifee interface * Add notify function to network interface * Implement notify feature * Add tests for notify * Make notifee functions all async * Fix linting issue * Fix linting issue * Scaffold pubsub router interface * Scaffold pubsub directory * Store peer_id in muxed connection * Implement pubsub notifee * Remove outdated files * Implement pubsub first attempt * Prepare pubsub for floodsub * Add mplex conn to net stream and add conn in notify tests * Implement floodsub * Use NetStream in generic protocol handler * Debugging async issues * Modify test to perform proper assert. Test passes * Remove callbacks. Reduce sleep time * Add simple three node test * Clean up code. Add message classes * Add test for two topics * Add conn to net stream and conn tests * Refactor test setup to remove duplicate code * Fix linting issues * Fix linting issue * Fix linting issue * Fix outstanding unrelated lint issue in multiselect_client * Add connect function * Remove debug prints * Remove debug prints from floodsub * Use MessageTalk in place of direct message breakdown * Remove extra prints * Remove outdated function * Add message to queues for all topics in message * Debugging * Add message self delivery * Increase read timeout to 5 to get pubsub tests passing * Refactor testing helper func. Add tests * Add tests and increase timeout to get tests passing * Add dummy account demo scaffolding * Attempt to use threads. Test fails * Implement basic dummy node tests using threads * Add generic testing function * Add simple seven node tree test * Add more complex seven node tree tests * Add five node ring tests * Remove unnecessary get_message_type func * Add documentation to classes * Add message id to messages * Add documentation to test helper func * Add docs to dummy account node helper func * Add more docs to dummy account node test helper func * fixed linting errors in floodsub * small notify bugfix * move pubsub into libp2p * fixed pubsub linting * fixing pubsub test failures * linting --- libp2p/network/swarm.py | 14 +- libp2p/pubsub/__init__.py | 0 libp2p/pubsub/floodsub.py | 98 ++++ libp2p/pubsub/message.py | 118 +++++ libp2p/pubsub/pubsub.py | 294 +++++++++++ libp2p/pubsub/pubsub_notifee.py | 40 ++ libp2p/pubsub/pubsub_router_interface.py | 64 +++ libp2p/stream_muxer/mplex/mplex.py | 10 +- .../muxed_connection_interface.py | 3 +- libp2p/transport/upgrader.py | 4 +- tests/libp2p/test_notify.py | 31 ++ tests/pubsub/dummy_account_node.py | 134 +++++ tests/pubsub/test_dummyaccount_demo.py | 189 +++++++ tests/pubsub/test_floodsub.py | 486 ++++++++++++++++++ 14 files changed, 1474 insertions(+), 11 deletions(-) create mode 100644 libp2p/pubsub/__init__.py create mode 100644 libp2p/pubsub/floodsub.py create mode 100644 libp2p/pubsub/message.py create mode 100644 libp2p/pubsub/pubsub.py create mode 100644 libp2p/pubsub/pubsub_notifee.py create mode 100644 libp2p/pubsub/pubsub_router_interface.py create mode 100644 tests/pubsub/dummy_account_node.py create mode 100644 tests/pubsub/test_dummyaccount_demo.py create mode 100644 tests/pubsub/test_floodsub.py diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index f2943ba..b30567b 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -70,7 +70,8 @@ class Swarm(INetwork): raw_conn = await self.transport.dial(multiaddr, self.self_id) # Use upgrader to upgrade raw conn to muxed conn - muxed_conn = self.upgrader.upgrade_connection(raw_conn, self.generic_protocol_handler) + muxed_conn = self.upgrader.upgrade_connection(raw_conn, \ + self.generic_protocol_handler, peer_id) # Store muxed connection in connections self.connections[peer_id] = muxed_conn @@ -145,7 +146,7 @@ class Swarm(INetwork): raw_conn = RawConnection(multiaddr.value_for_protocol('ip4'), multiaddr.value_for_protocol('tcp'), reader, writer, False) muxed_conn = self.upgrader.upgrade_connection(raw_conn, \ - self.generic_protocol_handler) + self.generic_protocol_handler, peer_id) # Store muxed_conn with peer id self.connections[peer_id] = muxed_conn @@ -197,14 +198,17 @@ def create_generic_protocol_handler(swarm): async def generic_protocol_handler(muxed_stream): # Perform protocol muxing to determine protocol to use - _, handler = await multiselect.negotiate(muxed_stream) + protocol, handler = await multiselect.negotiate(muxed_stream) + + net_stream = NetStream(muxed_stream) + net_stream.set_protocol(protocol) # Call notifiers since event occurred for notifee in swarm.notifees: - await notifee.opened_stream(swarm, muxed_stream) + await notifee.opened_stream(swarm, net_stream) # Give to stream handler - asyncio.ensure_future(handler(muxed_stream)) + asyncio.ensure_future(handler(net_stream)) return generic_protocol_handler diff --git a/libp2p/pubsub/__init__.py b/libp2p/pubsub/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py new file mode 100644 index 0000000..68e542c --- /dev/null +++ b/libp2p/pubsub/floodsub.py @@ -0,0 +1,98 @@ +from .pubsub_router_interface import IPubsubRouter +from .message import create_message_talk + +class FloodSub(IPubsubRouter): + + def __init__(self, protocols): + self.protocols = protocols + self.pubsub = None + + def get_protocols(self): + """ + :return: the list of protocols supported by the router + """ + return self.protocols + + def attach(self, pubsub): + """ + Attach is invoked by the PubSub constructor to attach the router to a + freshly initialized PubSub instance. + :param pubsub: pubsub instance to attach to + """ + self.pubsub = pubsub + + def add_peer(self, peer_id, protocol_id): + """ + Notifies the router that a new peer has been connected + :param peer_id: id of peer to add + """ + + def remove_peer(self, peer_id): + """ + Notifies the router that a peer has been disconnected + :param peer_id: id of peer to remove + """ + + def handle_rpc(self, rpc): + """ + Invoked to process control messages in the RPC envelope. + It is invoked after subscriptions and payload messages have been processed + :param rpc: rpc message + """ + + async def publish(self, sender_peer_id, message): + """ + Invoked to forward a new message that has been validated. + This is where the "flooding" part of floodsub happens + + With flooding, routing is almost trivial: for each incoming message, + forward to all known peers in the topic. There is a bit of logic, + as the router maintains a timed cache of previous messages, + so that seen messages are not further forwarded. + It also never forwards a message back to the source + or the peer that forwarded the message. + :param sender_peer_id: peer_id of message sender + :param message: message to forward + """ + + # Encode message + encoded_msg = message.encode() + + # Get message sender, origin, and topics + msg_talk = create_message_talk(message) + msg_sender = str(sender_peer_id) + msg_origin = msg_talk.origin_id + topics = msg_talk.topics + + # Deliver to self if self was origin + # Note: handle_talk checks if self is subscribed to topics in message + if msg_sender == msg_origin and msg_sender == str(self.pubsub.host.get_id()): + await self.pubsub.handle_talk(message) + + # Deliver to self and peers + for topic in topics: + if topic in self.pubsub.peer_topics: + for peer_id_in_topic in self.pubsub.peer_topics[topic]: + # Forward to all known peers in the topic that are not the + # message sender and are not the message origin + if peer_id_in_topic not in (msg_sender, msg_origin): + stream = self.pubsub.peers[peer_id_in_topic] + await stream.write(encoded_msg) + else: + # Implies publish did not write + print("publish did not write") + + def join(self, topic): + """ + Join notifies the router that we want to receive and + forward messages in a topic. It is invoked after the + subscription announcement + :param topic: topic to join + """ + + def leave(self, topic): + """ + Leave notifies the router that we are no longer interested in a topic. + It is invoked after the unsubscription announcement. + :param topic: topic to leave + """ diff --git a/libp2p/pubsub/message.py b/libp2p/pubsub/message.py new file mode 100644 index 0000000..2f839dc --- /dev/null +++ b/libp2p/pubsub/message.py @@ -0,0 +1,118 @@ +import uuid + + +class MessageTalk(): + + """ + Object to make parsing talk messages easier, where talk messages are + defined as custom messages published to a set of topics + """ + # pylint: disable=too-few-public-methods + def __init__(self, from_id, origin_id, topics, data, message_id): + # pylint: disable=too-many-arguments + self.msg_type = "talk" + self.from_id = from_id + self.origin_id = origin_id + self.topics = topics + self.data = data + self.message_id = message_id + + def to_str(self): + """ + Convert to string + :return: MessageTalk object in string representation + """ + out = self.msg_type + '\n' + out += self.from_id + '\n' + out += self.origin_id + '\n' + out += self.message_id + '\n' + for i in range(len(self.topics)): + out += self.topics[i] + if i < len(self.topics) - 1: + out += ',' + out += '\n' + self.data + return out + + +class MessageSub(): + """ + Object to make parsing subscription messages easier, where subscription + messages are defined as indicating the topics a node wishes to subscribe to + or unsubscribe from + """ + # pylint: disable=too-few-public-methods + def __init__(self, from_id, origin_id, subs_map, message_id): + self.msg_type = "subscription" + self.from_id = from_id + self.origin_id = origin_id + self.subs_map = subs_map + self.message_id = message_id + + def to_str(self): + """ + Convert to string + :return: MessageSub object in string representation + """ + out = self.msg_type + '\n' + out += self.from_id + '\n' + out += self.origin_id + '\n' + out += self.message_id + + if self.subs_map: + out += '\n' + + keys = list(self.subs_map) + + for i, topic in enumerate(keys): + sub = self.subs_map[topic] + if sub: + out += "sub:" + else: + out += "unsub:" + out += topic + if i < len(keys) - 1: + out += '\n' + + return out + +def create_message_talk(msg_talk_as_str): + """ + Create a MessageTalk object from a MessageTalk string representation + :param msg_talk_as_str: a MessageTalk object in its string representation + :return: MessageTalk object + """ + msg_comps = msg_talk_as_str.split('\n') + from_id = msg_comps[1] + origin_id = msg_comps[2] + message_id = msg_comps[3] + topics = msg_comps[4].split(',') + data = msg_comps[5] + return MessageTalk(from_id, origin_id, topics, data, message_id) + +def create_message_sub(msg_sub_as_str): + """ + Create a MessageSub object from a MessageSub string representation + :param msg_talk_as_str: a MessageSub object in its string representation + :return: MessageSub object + """ + msg_comps = msg_sub_as_str.split('\n') + from_id = msg_comps[1] + origin_id = msg_comps[2] + message_id = msg_comps[3] + + subs_map = {} + for i in range(4, len(msg_comps)): + sub_comps = msg_comps[i].split(":") + topic = sub_comps[1] + if sub_comps[0] == "sub": + subs_map[topic] = True + else: + subs_map[topic] = False + return MessageSub(from_id, origin_id, subs_map, message_id) + +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py new file mode 100644 index 0000000..9bd072f --- /dev/null +++ b/libp2p/pubsub/pubsub.py @@ -0,0 +1,294 @@ +import asyncio + +from .pubsub_notifee import PubsubNotifee +from .message import MessageSub +from .message import create_message_talk, create_message_sub +from. message import generate_message_id + + +class Pubsub(): + """ + For now, because I'm on a plane and don't have access to the go repo/protobuf stuff, + this is going to be the message format for the two types: subscription and talk + subscription indicates subscribing or unsubscribing from a topic + talk is sending a message on topic(s) + subscription format: + subscription + 'from' + :'topicid' + :'topicid' + ... + Ex. + subscription + msg_sender_peer_id + origin_peer_id + sub:topic1 + sub:topic2 + unsub:fav_topic + talk format: + talk + 'from' + 'origin' + [topic_ids comma-delimited] + 'data' + Ex. + talk + msg_sender_peer_id + origin_peer_id + topic1,topics_are_cool,foo + I like tacos + """ + # pylint: disable=too-many-instance-attributes + + def __init__(self, host, router, my_id): + """ + Construct a new Pubsub object, which is responsible for handling all + Pubsub-related messages and relaying messages as appropriate to the + Pubsub router (which is responsible for choosing who to send messages to). + Since the logic for choosing peers to send pubsub messages to is + in the router, the same Pubsub impl can back floodsub, gossipsub, etc. + """ + self.host = host + self.router = router + self.my_id = my_id + + # Attach this new Pubsub object to the router + self.router.attach(self) + + # Register a notifee + self.peer_queue = asyncio.Queue() + self.host.get_network().notify(PubsubNotifee(self.peer_queue)) + + # Register stream handlers for each pubsub router protocol to handle + # the pubsub streams opened on those protocols + self.protocols = self.router.get_protocols() + for protocol in self.protocols: + self.host.set_stream_handler(protocol, self.stream_handler) + + # TODO: determine if these need to be asyncio queues, or if could possibly + # be ordinary blocking queues + self.incoming_msgs_from_peers = asyncio.Queue() + self.outgoing_messages = asyncio.Queue() + + # TODO: Make seen_messages a cache (LRU cache?) + self.seen_messages = [] + + # Map of topics we are subscribed to to handler functions + # for when the given topic receives a message + self.my_topics = {} + + # Map of topic to peers to keep track of what peers are subscribed to + self.peer_topics = {} + + # Create peers map, which maps peer_id (as string) to stream (to a given peer) + self.peers = {} + + # Call handle peer to keep waiting for updates to peer queue + asyncio.ensure_future(self.handle_peer_queue()) + + def get_hello_packet(self): + """ + Generate subscription message with all topics we are subscribed to + """ + subs_map = {} + for topic in self.my_topics: + subs_map[topic] = True + sub_msg = MessageSub( + str(self.host.get_id()),\ + str(self.host.get_id()), subs_map, generate_message_id()\ + ) + return sub_msg.to_str() + + async def continously_read_stream(self, stream): + """ + Read from input stream in an infinite loop. Process + messages from other nodes, which for now are considered MessageTalk + and MessageSub messages. + TODO: Handle RPC messages instead of my Aspyn's own custom message format + :param stream: stream to continously read from + """ + while True: + incoming = (await stream.read()).decode() + msg_comps = incoming.split('\n') + msg_type = msg_comps[0] + + msg_sender = msg_comps[1] + # msg_origin = msg_comps[2] + msg_id = msg_comps[3] + print("HIT ME1") + if msg_id not in self.seen_messages: + print("HIT ME") + # Do stuff with incoming unseen message + should_publish = True + if msg_type == "subscription": + self.handle_subscription(incoming) + + # We don't need to relay the subscription to our + # peers because a given node only needs its peers + # to know that it is subscribed to the topic (doesn't + # need everyone to know) + should_publish = False + elif msg_type == "talk": + await self.handle_talk(incoming) + + # Add message id to seen + self.seen_messages.append(msg_id) + + # Publish message using router's publish + if should_publish: + msg = create_message_talk(incoming) + + # Adjust raw_msg to that the message sender + # is now our peer_id + msg.from_id = str(self.host.get_id()) + + await self.router.publish(msg_sender, msg.to_str()) + + # Force context switch + await asyncio.sleep(0) + + async def stream_handler(self, stream): + """ + Stream handler for pubsub. Gets invoked whenever a new stream is created + on one of the supported pubsub protocols. + :param stream: newly created stream + """ + # Add peer + # Map peer to stream + peer_id = stream.mplex_conn.peer_id + self.peers[str(peer_id)] = stream + self.router.add_peer(peer_id, stream.get_protocol()) + + # Send hello packet + hello = self.get_hello_packet() + await stream.write(hello.encode()) + # Pass stream off to stream reader + asyncio.ensure_future(self.continously_read_stream(stream)) + + async def handle_peer_queue(self): + """ + Continuously read from peer queue and each time a new peer is found, + open a stream to the peer using a supported pubsub protocol + TODO: Handle failure for when the peer does not support any of the + pubsub protocols we support + """ + while True: + peer_id = await self.peer_queue.get() + + # Open a stream to peer on existing connection + # (we know connection exists since that's the only way + # an element gets added to peer_queue) + stream = await self.host.new_stream(peer_id, self.protocols) + + # Add Peer + # Map peer to stream + self.peers[str(peer_id)] = stream + self.router.add_peer(peer_id, stream.get_protocol()) + + # Send hello packet + hello = self.get_hello_packet() + await stream.write(hello.encode()) + + # Pass stream off to stream reader + asyncio.ensure_future(self.continously_read_stream(stream)) + + # Force context switch + await asyncio.sleep(0) + + def handle_subscription(self, subscription): + """ + Handle an incoming subscription message from a peer. Update internal + mapping to mark the peer as subscribed or unsubscribed to topics as + defined in the subscription message + :param subscription: raw data constituting a subscription message + """ + sub_msg = create_message_sub(subscription) + if sub_msg.subs_map: + print("handle_subscription my_id: " + self.my_id + ", subber: " + sub_msg.origin_id) + for topic_id in sub_msg.subs_map: + # Look at each subscription in the msg individually + if sub_msg.subs_map[topic_id]: + if topic_id not in self.peer_topics: + # Create topic list if it did not yet exist + self.peer_topics[topic_id] = [sub_msg.origin_id] + elif sub_msg.origin_id not in self.peer_topics[topic_id]: + # Add peer to topic + self.peer_topics[topic_id].append(sub_msg.origin_id) + else: + # TODO: Remove peer from topic + pass + + async def handle_talk(self, talk): + """ + Handle incoming Talk message from a peer. A Talk message contains some + custom message that is published on a given topic(s) + :param talk: raw data constituting a talk message + """ + msg = create_message_talk(talk) + + # Check if this message has any topics that we are subscribed to + for topic in msg.topics: + if topic in self.my_topics: + # we are subscribed to a topic this message was sent for, + # so add message to the subscription output queue + # for each topic + await self.my_topics[topic].put(talk) + + async def subscribe(self, topic_id): + """ + Subscribe ourself to a topic + :param topic_id: topic_id to subscribe to + """ + # Map topic_id to blocking queue + self.my_topics[topic_id] = asyncio.Queue() + + # Create subscribe message + sub_msg = MessageSub( + str(self.host.get_id()),\ + str(self.host.get_id()), {topic_id: True}, generate_message_id()\ + ) + + # Send out subscribe message to all peers + await self.message_all_peers(sub_msg.to_str()) + + # Tell router we are joining this topic + self.router.join(topic_id) + + # Return the asyncio queue for messages on this topic + return self.my_topics[topic_id] + + async def unsubscribe(self, topic_id): + """ + Unsubscribe ourself from a topic + :param topic_id: topic_id to unsubscribe from + """ + + # Remove topic_id from map if present + if topic_id in self.my_topics: + del self.my_topics[topic_id] + + # Create unsubscribe message + unsub_msg = MessageSub(str(self.host.get_id()), str(self.host.get_id()),\ + {topic_id: False}, generate_message_id()) + + # Send out unsubscribe message to all peers + await self.message_all_peers(unsub_msg.to_str()) + + # Tell router we are leaving this topic + self.router.leave(topic_id) + + async def message_all_peers(self, raw_msg): + """ + Broadcast a message to peers + :param raw_msg: raw contents of the message to broadcast + """ + + # Encode message for sending + encoded_msg = raw_msg.encode() + + # Broadcast message + for peer in self.peers: + stream = self.peers[peer] + + # Write message to stream + await stream.write(encoded_msg) diff --git a/libp2p/pubsub/pubsub_notifee.py b/libp2p/pubsub/pubsub_notifee.py new file mode 100644 index 0000000..4173bd8 --- /dev/null +++ b/libp2p/pubsub/pubsub_notifee.py @@ -0,0 +1,40 @@ +from libp2p.network.notifee_interface import INotifee + + +class PubsubNotifee(INotifee): + # pylint: disable=too-many-instance-attributes, cell-var-from-loop + + def __init__(self, initiator_peers_queue): + """ + :param initiator_peers_queue: queue to add new peers to so that pubsub + can process new peers after we connect to them + """ + self.initiator_peers_queue = initiator_peers_queue + + async def opened_stream(self, network, stream): + pass + + async def closed_stream(self, network, stream): + pass + + async def connected(self, network, conn): + """ + Add peer_id to initiator_peers_queue, so that this peer_id can be used to + create a stream and we only want to have one pubsub stream with each peer. + :param network: network the connection was opened on + :param conn: connection that was opened + """ + + # Only add peer_id if we are initiator (otherwise we would end up + # with two pubsub streams between us and the peer) + if conn.initiator: + await self.initiator_peers_queue.put(conn.peer_id) + + async def disconnected(self, network, conn): + pass + + async def listen(self, network, multiaddr): + pass + + async def listen_close(self, network, multiaddr): + pass diff --git a/libp2p/pubsub/pubsub_router_interface.py b/libp2p/pubsub/pubsub_router_interface.py new file mode 100644 index 0000000..727b39e --- /dev/null +++ b/libp2p/pubsub/pubsub_router_interface.py @@ -0,0 +1,64 @@ +from abc import ABC, abstractmethod + +class IPubsubRouter(ABC): + + @abstractmethod + def get_protocols(self): + """ + :return: the list of protocols supported by the router + """ + + @abstractmethod + def attach(self, pubsub): + """ + Attach is invoked by the PubSub constructor to attach the router to a + freshly initialized PubSub instance. + :param pubsub: pubsub instance to attach to + """ + + @abstractmethod + def add_peer(self, peer_id, protocol_id): + """ + Notifies the router that a new peer has been connected + :param peer_id: id of peer to add + """ + + @abstractmethod + def remove_peer(self, peer_id): + """ + Notifies the router that a peer has been disconnected + :param peer_id: id of peer to remove + """ + + @abstractmethod + def handle_rpc(self, rpc): + """ + Invoked to process control messages in the RPC envelope. + It is invoked after subscriptions and payload messages have been processed + :param rpc: rpc message + """ + + @abstractmethod + def publish(self, sender_peer_id, message): + """ + Invoked to forward a new message that has been validated + :param sender_peer_id: peer_id of message sender + :param message: message to forward + """ + + @abstractmethod + def join(self, topic): + """ + Join notifies the router that we want to receive and + forward messages in a topic. It is invoked after the + subscription announcement + :param topic: topic to join + """ + + @abstractmethod + def leave(self, topic): + """ + Leave notifies the router that we are no longer interested in a topic. + It is invoked after the unsubscription announcement. + :param topic: topic to leave + """ diff --git a/libp2p/stream_muxer/mplex/mplex.py b/libp2p/stream_muxer/mplex/mplex.py index 135ee1e..0d587b5 100644 --- a/libp2p/stream_muxer/mplex/mplex.py +++ b/libp2p/stream_muxer/mplex/mplex.py @@ -11,14 +11,15 @@ class Mplex(IMuxedConn): reference: https://github.com/libp2p/go-mplex/blob/master/multiplex.go """ - def __init__(self, conn, generic_protocol_handler): + def __init__(self, conn, generic_protocol_handler, peer_id): """ create a new muxed connection :param conn: an instance of raw connection :param generic_protocol_handler: generic protocol handler for new muxed streams + :param peer_id: peer_id of peer the connection is to """ - super(Mplex, self).__init__(conn, generic_protocol_handler) + super(Mplex, self).__init__(conn, generic_protocol_handler, peer_id) self.raw_conn = conn self.initiator = conn.initiator @@ -26,6 +27,9 @@ class Mplex(IMuxedConn): # Store generic protocol handler self.generic_protocol_handler = generic_protocol_handler + # Set peer_id + self.peer_id = peer_id + # Mapping from stream ID -> buffer of messages for that stream self.buffers = {} @@ -56,7 +60,7 @@ class Mplex(IMuxedConn): # TODO: pass down timeout from user and use that if stream_id in self.buffers: try: - data = await asyncio.wait_for(self.buffers[stream_id].get(), timeout=3) + data = await asyncio.wait_for(self.buffers[stream_id].get(), timeout=8) return data except asyncio.TimeoutError: return None diff --git a/libp2p/stream_muxer/muxed_connection_interface.py b/libp2p/stream_muxer/muxed_connection_interface.py index 4017755..541fd64 100644 --- a/libp2p/stream_muxer/muxed_connection_interface.py +++ b/libp2p/stream_muxer/muxed_connection_interface.py @@ -7,12 +7,13 @@ class IMuxedConn(ABC): """ @abstractmethod - def __init__(self, conn, generic_protocol_handler): + def __init__(self, conn, generic_protocol_handler, peer_id): """ create a new muxed connection :param conn: an instance of raw connection :param generic_protocol_handler: generic protocol handler for new muxed streams + :param peer_id: peer_id of peer the connection is to """ @abstractmethod diff --git a/libp2p/transport/upgrader.py b/libp2p/transport/upgrader.py index 5f193a0..9e311e3 100644 --- a/libp2p/transport/upgrader.py +++ b/libp2p/transport/upgrader.py @@ -17,11 +17,11 @@ class TransportUpgrader: def upgrade_security(self): pass - def upgrade_connection(self, conn, generic_protocol_handler): + def upgrade_connection(self, conn, generic_protocol_handler, peer_id): """ upgrade raw connection to muxed connection """ # For PoC, no security, default to mplex # TODO do exchange to determine multiplexer - return Mplex(conn, generic_protocol_handler) + return Mplex(conn, generic_protocol_handler, peer_id) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 71b455b..570ad57 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -49,6 +49,7 @@ class MyNotifee(INotifee): async def listen_close(self, network, _multiaddr): pass + class InvalidNotifee(): # pylint: disable=too-many-instance-attributes, cell-var-from-loop @@ -70,6 +71,36 @@ class InvalidNotifee(): async def listen(self): assert False + +async def perform_two_host_simple_set_up(): + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + async def my_stream_handler(stream): + while True: + read_string = (await stream.read()).decode() + + resp = "ack:" + read_string + await stream.write(resp.encode()) + + node_b.set_stream_handler("/echo/1.0.0", my_stream_handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + return node_a, node_b + + +async def perform_two_host_simple_set_up_custom_handler(handler): + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + node_b.set_stream_handler("/echo/1.0.0", handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + return node_a, node_b + + @pytest.mark.asyncio async def test_one_notifier(): node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py new file mode 100644 index 0000000..f328a6e --- /dev/null +++ b/tests/pubsub/dummy_account_node.py @@ -0,0 +1,134 @@ +import asyncio +import multiaddr + +from libp2p import new_node +from libp2p.pubsub.message import create_message_talk +from libp2p.pubsub.pubsub import Pubsub +from libp2p.pubsub.floodsub import FloodSub +from libp2p.pubsub.message import MessageTalk +from libp2p.pubsub.message import generate_message_id + +SUPPORTED_PUBSUB_PROTOCOLS = ["/floodsub/1.0.0"] +CRYPTO_TOPIC = "ethereum" + +# Message format: +# Sending crypto: ,, +# Ex. send,aspyn,alex,5 +# Set crypto: , +# Ex. set,rob,5 +# Determine message type by looking at first item before first comma + +class DummyAccountNode(): + """ + Node which has an internal balance mapping, meant to serve as + a dummy crypto blockchain. There is no actual blockchain, just a simple + map indicating how much crypto each user in the mappings holds + """ + + def __init__(self): + self.balances = {} + + @classmethod + async def create(cls): + """ + Create a new DummyAccountNode and attach a libp2p node, a floodsub, and a pubsub + instance to this new node + + We use create as this serves as a factory function and allows us + to use async await, unlike the init function + """ + self = DummyAccountNode() + + libp2p_node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + await libp2p_node.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + + self.libp2p_node = libp2p_node + + self.floodsub = FloodSub(SUPPORTED_PUBSUB_PROTOCOLS) + self.pubsub = Pubsub(self.libp2p_node, self.floodsub, "a") + return self + + async def handle_incoming_msgs(self): + """ + Handle all incoming messages on the CRYPTO_TOPIC from peers + """ + while True: + message_raw = await self.q.get() + message = create_message_talk(message_raw) + contents = message.data + + msg_comps = contents.split(",") + + if msg_comps[0] == "send": + self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) + elif msg_comps[0] == "set": + self.handle_set_crypto_for_user(msg_comps[1], int(msg_comps[2])) + + async def setup_crypto_networking(self): + """ + Subscribe to CRYPTO_TOPIC and perform call to function that handles + all incoming messages on said topic + """ + self.q = await self.pubsub.subscribe(CRYPTO_TOPIC) + + asyncio.ensure_future(self.handle_incoming_msgs()) + + async def publish_send_crypto(self, source_user, dest_user, amount): + """ + Create a send crypto message and publish that message to all other nodes + :param source_user: user to send crypto from + :param dest_user: user to send crypto to + :param amount: amount of crypto to send + """ + my_id = str(self.libp2p_node.get_id()) + msg_contents = "send," + source_user + "," + dest_user + "," + str(amount) + msg = MessageTalk(my_id, my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + await self.floodsub.publish(my_id, msg.to_str()) + + async def publish_set_crypto(self, user, amount): + """ + Create a set crypto message and publish that message to all other nodes + :param user: user to set crypto for + :param amount: amount of crypto + """ + my_id = str(self.libp2p_node.get_id()) + msg_contents = "set," + user + "," + str(amount) + msg = MessageTalk(my_id, my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + await self.floodsub.publish(my_id, msg.to_str()) + + def handle_send_crypto(self, source_user, dest_user, amount): + """ + Handle incoming send_crypto message + :param source_user: user to send crypto from + :param dest_user: user to send crypto to + :param amount: amount of crypto to send + """ + if source_user in self.balances: + self.balances[source_user] -= amount + else: + self.balances[source_user] = -amount + + if dest_user in self.balances: + self.balances[dest_user] += amount + else: + self.balances[dest_user] = amount + + def handle_set_crypto_for_user(self, dest_user, amount): + """ + Handle incoming set_crypto message + :param dest_user: user to set crypto for + :param amount: amount of crypto + """ + self.balances[dest_user] = amount + + def get_balance(self, user): + """ + Get balance in crypto for a particular user + :param user: user to get balance for + :return: balance of user + """ + if user in self.balances: + return self.balances[user] + else: + return -1 + diff --git a/tests/pubsub/test_dummyaccount_demo.py b/tests/pubsub/test_dummyaccount_demo.py new file mode 100644 index 0000000..a071fa6 --- /dev/null +++ b/tests/pubsub/test_dummyaccount_demo.py @@ -0,0 +1,189 @@ +import asyncio +import multiaddr +import pytest + +from threading import Thread +from tests.utils import cleanup +from libp2p import new_node +from libp2p.peer.peerinfo import info_from_p2p_addr +from libp2p.pubsub.pubsub import Pubsub +from libp2p.pubsub.floodsub import FloodSub +from libp2p.pubsub.message import MessageTalk +from libp2p.pubsub.message import create_message_talk +from dummy_account_node import DummyAccountNode + +# pylint: disable=too-many-locals + +async def connect(node1, node2): + # node1 connects to node2 + addr = node2.get_addrs()[0] + info = info_from_p2p_addr(addr) + await node1.connect(info) + +def create_setup_in_new_thread_func(dummy_node): + def setup_in_new_thread(): + asyncio.ensure_future(dummy_node.setup_crypto_networking()) + return setup_in_new_thread + +async def perform_test(num_nodes, adjacency_map, action_func, assertion_func): + """ + Helper function to allow for easy construction of custom tests for dummy account nodes + in various network topologies + :param num_nodes: number of nodes in the test + :param adjacency_map: adjacency map defining each node and its list of neighbors + :param action_func: function to execute that includes actions by the nodes, + such as send crypto and set crypto + :param assertion_func: assertions for testing the results of the actions are correct + """ + + # Create nodes + dummy_nodes = [] + for i in range(num_nodes): + dummy_nodes.append(await DummyAccountNode.create()) + + # Create network + for source_num in adjacency_map: + target_nums = adjacency_map[source_num] + for target_num in target_nums: + await connect(dummy_nodes[source_num].libp2p_node, \ + dummy_nodes[target_num].libp2p_node) + + # Allow time for network creation to take place + await asyncio.sleep(0.25) + + # Start a thread for each node so that each node can listen and respond + # to messages on its own thread, which will avoid waiting indefinitely + # on the main thread. On this thread, call the setup func for the node, + # which subscribes the node to the CRYPTO_TOPIC topic + for dummy_node in dummy_nodes: + thread = Thread(target=create_setup_in_new_thread_func(dummy_node)) + thread.run() + + # Allow time for nodes to subscribe to CRYPTO_TOPIC topic + await asyncio.sleep(0.25) + + # Perform action function + await action_func(dummy_nodes) + + # Allow time for action function to be performed (i.e. messages to propogate) + await asyncio.sleep(0.25) + + # Perform assertion function + for dummy_node in dummy_nodes: + assertion_func(dummy_node) + + # Success, terminate pending tasks. + await cleanup() + +@pytest.mark.asyncio +async def test_simple_two_nodes(): + num_nodes = 2 + adj_map = {0: [1]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 10) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 10 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_simple_three_nodes_line_topography(): + num_nodes = 3 + adj_map = {0: [1], 1: [2]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 10) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 10 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_simple_three_nodes_triangle_topography(): + num_nodes = 3 + adj_map = {0: [1, 2], 1: [2]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_simple_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_set_then_send_from_root_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) + await asyncio.sleep(0.25) + await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 15 + assert dummy_node.get_balance("alex") == 5 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} + + async def action_func(dummy_nodes): + await dummy_nodes[6].publish_set_crypto("aspyn", 20) + await asyncio.sleep(0.25) + await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 15 + assert dummy_node.get_balance("alex") == 5 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_simple_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("alex", 20) + await asyncio.sleep(0.25) + await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("alex") == 8 + assert dummy_node.get_balance("rob") == 12 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py new file mode 100644 index 0000000..f4a5826 --- /dev/null +++ b/tests/pubsub/test_floodsub.py @@ -0,0 +1,486 @@ +import asyncio +import multiaddr +import pytest + +from tests.utils import cleanup +from libp2p import new_node +from libp2p.peer.peerinfo import info_from_p2p_addr +from libp2p.pubsub.pubsub import Pubsub +from libp2p.pubsub.floodsub import FloodSub +from libp2p.pubsub.message import MessageTalk +from libp2p.pubsub.message import create_message_talk +from libp2p.pubsub.message import generate_message_id + +# pylint: disable=too-many-locals + +async def connect(node1, node2): + """ + Connect node1 to node2 + """ + addr = node2.get_addrs()[0] + info = info_from_p2p_addr(addr) + await node1.connect(info) + +@pytest.mark.asyncio +async def test_simple_two_nodes(): + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + + supported_protocols = ["/floodsub/1.0.0"] + + floodsub_a = FloodSub(supported_protocols) + pubsub_a = Pubsub(node_a, floodsub_a, "a") + floodsub_b = FloodSub(supported_protocols) + pubsub_b = Pubsub(node_b, floodsub_b, "b") + + await connect(node_a, node_b) + + await asyncio.sleep(0.25) + qb = await pubsub_b.subscribe("my_topic") + + await asyncio.sleep(0.25) + + node_a_id = str(node_a.get_id()) + + msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) + + await floodsub_a.publish(node_a.get_id(), msg.to_str()) + + await asyncio.sleep(0.25) + + res_b = await qb.get() + + # Check that the msg received by node_b is the same + # as the message sent by node_a + assert res_b == msg.to_str() + + # Success, terminate pending tasks. + await cleanup() + +@pytest.mark.asyncio +async def test_simple_three_nodes(): + # Want to pass message from A -> B -> C + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_c = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + await node_c.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + + supported_protocols = ["/floodsub/1.0.0"] + + floodsub_a = FloodSub(supported_protocols) + pubsub_a = Pubsub(node_a, floodsub_a, "a") + floodsub_b = FloodSub(supported_protocols) + pubsub_b = Pubsub(node_b, floodsub_b, "b") + floodsub_c = FloodSub(supported_protocols) + pubsub_c = Pubsub(node_c, floodsub_c, "c") + + await connect(node_a, node_b) + await connect(node_b, node_c) + + await asyncio.sleep(0.25) + qb = await pubsub_b.subscribe("my_topic") + qc = await pubsub_c.subscribe("my_topic") + await asyncio.sleep(0.25) + + node_a_id = str(node_a.get_id()) + + msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) + + await floodsub_a.publish(node_a.get_id(), msg.to_str()) + + await asyncio.sleep(0.25) + res_b = await qb.get() + res_c = await qc.get() + + # Check that the msg received by node_b is the same + # as the message sent by node_a + assert res_b == msg.to_str() + + # res_c should match original msg but with b as sender + node_b_id = str(node_b.get_id()) + msg.from_id = node_b_id + + assert res_c == msg.to_str() + + # Success, terminate pending tasks. + await cleanup() + +async def perform_test_from_obj(obj): + """ + Perform a floodsub test from a test obj. + test obj are composed as follows: + + { + "supported_protocols": ["supported/protocol/1.0.0",...], + "adj_list": { + "node1": ["neighbor1_of_node1", "neighbor2_of_node1", ...], + "node2": ["neighbor1_of_node2", "neighbor2_of_node2", ...], + ... + }, + "topic_map": { + "topic1": ["node1_subscribed_to_topic1", "node2_subscribed_to_topic1", ...] + }, + "messages": [ + { + "topics": ["topic1_for_message", "topic2_for_message", ...], + "data": "some contents of the message (newlines are not supported)", + "node_id": "message sender node id" + }, + ... + ] + } + NOTE: In adj_list, for any neighbors A and B, only list B as a neighbor of A + or B as a neighbor of A once. Do NOT list both A: ["B"] and B:["A"] as the behavior + is undefined (even if it may work) + """ + + # Step 1) Create graph + adj_list = obj["adj_list"] + node_map = {} + floodsub_map = {} + pubsub_map = {} + + supported_protocols = obj["supported_protocols"] + + tasks_connect = [] + for start_node_id in adj_list: + # Create node if node does not yet exist + if start_node_id not in node_map: + node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + await node.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + + node_map[start_node_id] = node + + floodsub = FloodSub(supported_protocols) + floodsub_map[start_node_id] = floodsub + pubsub = Pubsub(node, floodsub, start_node_id) + pubsub_map[start_node_id] = pubsub + + # For each neighbor of start_node, create if does not yet exist, + # then connect start_node to neighbor + for neighbor_id in adj_list[start_node_id]: + # Create neighbor if neighbor does not yet exist + if neighbor_id not in node_map: + neighbor_node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + await neighbor_node.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + + node_map[neighbor_id] = neighbor_node + + floodsub = FloodSub(supported_protocols) + floodsub_map[neighbor_id] = floodsub + pubsub = Pubsub(neighbor_node, floodsub, neighbor_id) + pubsub_map[neighbor_id] = pubsub + + # Connect node and neighbor + # await connect(node_map[start_node_id], node_map[neighbor_id]) + tasks_connect.append(asyncio.ensure_future(connect(node_map[start_node_id], node_map[neighbor_id]))) + tasks_connect.append(asyncio.sleep(2)) + await asyncio.gather(*tasks_connect) + + # Allow time for graph creation before continuing + # await asyncio.sleep(0.25) + + # Step 2) Subscribe to topics + queues_map = {} + topic_map = obj["topic_map"] + + tasks_topic = [] + tasks_topic_data = [] + for topic in topic_map: + for node_id in topic_map[topic]: + """ + # Subscribe node to topic + q = await pubsub_map[node_id].subscribe(topic) + + # Create topic-queue map for node_id if one does not yet exist + if node_id not in queues_map: + queues_map[node_id] = {} + + # Store queue in topic-queue map for node + queues_map[node_id][topic] = q + """ + tasks_topic.append(asyncio.ensure_future(pubsub_map[node_id].subscribe(topic))) + tasks_topic_data.append((node_id, topic)) + tasks_topic.append(asyncio.sleep(2)) + + # Gather is like Promise.all + responses = await asyncio.gather(*tasks_topic, return_exceptions=True) + for i in range(len(responses) - 1): + q = responses[i] + node_id, topic = tasks_topic_data[i] + if node_id not in queues_map: + queues_map[node_id] = {} + + # Store queue in topic-queue map for node + queues_map[node_id][topic] = q + + # Allow time for subscribing before continuing + # await asyncio.sleep(0.01) + + # Step 3) Publish messages + topics_in_msgs_ordered = [] + messages = obj["messages"] + tasks_publish = [] + for msg in messages: + topics = msg["topics"] + + data = msg["data"] + node_id = msg["node_id"] + + # Get actual id for sender node (not the id from the test obj) + actual_node_id = str(node_map[node_id].get_id()) + + # Create correctly formatted message + msg_talk = MessageTalk(actual_node_id, actual_node_id, topics, data, generate_message_id()) + + # Publish message + # await floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()) + tasks_publish.append(asyncio.ensure_future(floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()))) + + # For each topic in topics, add topic, msg_talk tuple to ordered test list + # TODO: Update message sender to be correct message sender before + # adding msg_talk to this list + for topic in topics: + topics_in_msgs_ordered.append((topic, msg_talk)) + + # Allow time for publishing before continuing + # await asyncio.sleep(0.4) + tasks_publish.append(asyncio.sleep(2)) + await asyncio.gather(*tasks_publish) + + # Step 4) Check that all messages were received correctly. + # TODO: Check message sender too + for i in range(len(topics_in_msgs_ordered)): + topic, actual_msg = topics_in_msgs_ordered[i] + for node_id in topic_map[topic]: + # Get message from subscription queue + msg_on_node_str = await queues_map[node_id][topic].get() + msg_on_node = create_message_talk(msg_on_node_str) + + # Perform checks + assert actual_msg.origin_id == msg_on_node.origin_id + assert actual_msg.topics == msg_on_node.topics + assert actual_msg.data == msg_on_node.data + + # Success, terminate pending tasks. + await cleanup() + +@pytest.mark.asyncio +async def test_simple_two_nodes_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "A": ["B"] + }, + "topic_map": { + "topic1": ["B"] + }, + "messages": [ + { + "topics": ["topic1"], + "data": "foo", + "node_id": "A" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_three_nodes_two_topics_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "A": ["B"], + "B": ["C"] + }, + "topic_map": { + "topic1": ["B", "C"], + "topic2": ["B", "C"] + }, + "messages": [ + { + "topics": ["topic1"], + "data": "foo", + "node_id": "A" + }, + { + "topics": ["topic2"], + "data": "Alex is tall", + "node_id": "A" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_two_nodes_one_topic_single_subscriber_is_sender_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "A": ["B"] + }, + "topic_map": { + "topic1": ["B"] + }, + "messages": [ + { + "topics": ["topic1"], + "data": "Alex is tall", + "node_id": "B" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_two_nodes_one_topic_two_msgs_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "A": ["B"] + }, + "topic_map": { + "topic1": ["B"] + }, + "messages": [ + { + "topics": ["topic1"], + "data": "Alex is tall", + "node_id": "B" + }, + { + "topics": ["topic1"], + "data": "foo", + "node_id": "A" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_seven_nodes_tree_one_topics_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2", "3"], + "2": ["4", "5"], + "3": ["6", "7"] + }, + "topic_map": { + "astrophysics": ["2", "3", "4", "5", "6", "7"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_seven_nodes_tree_three_topics_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2", "3"], + "2": ["4", "5"], + "3": ["6", "7"] + }, + "topic_map": { + "astrophysics": ["2", "3", "4", "5", "6", "7"], + "space": ["2", "3", "4", "5", "6", "7"], + "onions": ["2", "3", "4", "5", "6", "7"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["space"], + "data": "foobar", + "node_id": "1" + }, + { + "topics": ["onions"], + "data": "I am allergic", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_seven_nodes_tree_three_topics_diff_origin_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2", "3"], + "2": ["4", "5"], + "3": ["6", "7"] + }, + "topic_map": { + "astrophysics": ["1", "2", "3", "4", "5", "6", "7"], + "space": ["1", "2", "3", "4", "5", "6", "7"], + "onions": ["1", "2", "3", "4", "5", "6", "7"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["space"], + "data": "foobar", + "node_id": "4" + }, + { + "topics": ["onions"], + "data": "I am allergic", + "node_id": "7" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_three_nodes_clique_two_topic_diff_origin_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2", "3"], + "2": ["3"] + }, + "topic_map": { + "astrophysics": ["1", "2", "3"], + "school": ["1", "2", "3"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) From 1c08e9e55ca8f601745ff49050fc5dbf59ae77af Mon Sep 17 00:00:00 2001 From: Alex Haynes Date: Sun, 24 Mar 2019 15:08:36 -0400 Subject: [PATCH 049/131] added rpc.proto from go repo --- rpc.proto | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 rpc.proto diff --git a/rpc.proto b/rpc.proto new file mode 100644 index 0000000..2ae2ef2 --- /dev/null +++ b/rpc.proto @@ -0,0 +1,78 @@ +// Borrowed from https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto + +syntax = "proto2"; + +package pubsub.pb; + +message RPC { + repeated SubOpts subscriptions = 1; + repeated Message publish = 2; + + message SubOpts { + optional bool subscribe = 1; // subscribe or unsubcribe + optional string topicid = 2; + } + + optional ControlMessage control = 3; +} + +message Message { + optional bytes from = 1; + optional bytes data = 2; + optional bytes seqno = 3; + repeated string topicIDs = 4; + optional bytes signature = 5; + optional bytes key = 6; +} + +message ControlMessage { + repeated ControlIHave ihave = 1; + repeated ControlIWant iwant = 2; + repeated ControlGraft graft = 3; + repeated ControlPrune prune = 4; +} + +message ControlIHave { + optional string topicID = 1; + repeated string messageIDs = 2; +} + +message ControlIWant { + repeated string messageIDs = 1; +} + +message ControlGraft { + optional string topicID = 1; +} + +message ControlPrune { + optional string topicID = 1; +} + +message TopicDescriptor { + optional string name = 1; + optional AuthOpts auth = 2; + optional EncOpts enc = 3; + + message AuthOpts { + optional AuthMode mode = 1; + repeated bytes keys = 2; // root keys to trust + + enum AuthMode { + NONE = 0; // no authentication, anyone can publish + KEY = 1; // only messages signed by keys in the topic descriptor are accepted + WOT = 2; // web of trust, certificates can allow publisher set to grow + } + } + + message EncOpts { + optional EncMode mode = 1; + repeated bytes keyHashes = 2; // the hashes of the shared keys used (salted) + + enum EncMode { + NONE = 0; // no encryption, anyone can read + SHAREDKEY = 1; // messages are encrypted with shared key + WOT = 2; // web of trust, certificates can allow publisher set to grow + } + } +} \ No newline at end of file From 2aa7c4a106c6860b42af455ad79edaf281f92627 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 09:41:38 -0400 Subject: [PATCH 050/131] add pubsub proto --- libp2p/pubsub/pb/rpc.proto | 76 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 libp2p/pubsub/pb/rpc.proto diff --git a/libp2p/pubsub/pb/rpc.proto b/libp2p/pubsub/pb/rpc.proto new file mode 100644 index 0000000..764b033 --- /dev/null +++ b/libp2p/pubsub/pb/rpc.proto @@ -0,0 +1,76 @@ +syntax = "proto2"; + +package pubsub.pb; + +message RPC { + repeated SubOpts subscriptions = 1; + repeated Message publish = 2; + + message SubOpts { + optional bool subscribe = 1; // subscribe or unsubcribe + optional string topicid = 2; + } + + optional ControlMessage control = 3; +} + +message Message { + optional bytes from = 1; + optional bytes data = 2; + optional bytes seqno = 3; + repeated string topicIDs = 4; + optional bytes signature = 5; + optional bytes key = 6; +} + +message ControlMessage { + repeated ControlIHave ihave = 1; + repeated ControlIWant iwant = 2; + repeated ControlGraft graft = 3; + repeated ControlPrune prune = 4; +} + +message ControlIHave { + optional string topicID = 1; + repeated string messageIDs = 2; +} + +message ControlIWant { + repeated string messageIDs = 1; +} + +message ControlGraft { + optional string topicID = 1; +} + +message ControlPrune { + optional string topicID = 1; +} + +message TopicDescriptor { + optional string name = 1; + optional AuthOpts auth = 2; + optional EncOpts enc = 3; + + message AuthOpts { + optional AuthMode mode = 1; + repeated bytes keys = 2; // root keys to trust + + enum AuthMode { + NONE = 0; // no authentication, anyone can publish + KEY = 1; // only messages signed by keys in the topic descriptor are accepted + WOT = 2; // web of trust, certificates can allow publisher set to grow + } + } + + message EncOpts { + optional EncMode mode = 1; + repeated bytes keyHashes = 2; // the hashes of the shared keys used (salted) + + enum EncMode { + NONE = 0; // no encryption, anyone can read + SHAREDKEY = 1; // messages are encrypted with shared key + WOT = 2; // web of trust, certificates can allow publisher set to grow + } + } +} \ No newline at end of file From f2de3735ee20ef77ef38da2945a04343aabe71f1 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 09:55:14 -0400 Subject: [PATCH 051/131] add generated rpc code --- libp2p/pubsub/pb/rpc_pb2.py | 638 +++++++++++++++++++++++++++++++ libp2p/pubsub/pb/rpc_pb2_grpc.py | 3 + 2 files changed, 641 insertions(+) create mode 100644 libp2p/pubsub/pb/rpc_pb2.py create mode 100644 libp2p/pubsub/pb/rpc_pb2_grpc.py diff --git a/libp2p/pubsub/pb/rpc_pb2.py b/libp2p/pubsub/pb/rpc_pb2.py new file mode 100644 index 0000000..57035ed --- /dev/null +++ b/libp2p/pubsub/pb/rpc_pb2.py @@ -0,0 +1,638 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: rpc.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='rpc.proto', + package='pubsub.pb', + syntax='proto2', + serialized_options=None, + serialized_pb=_b('\n\trpc.proto\x12\tpubsub.pb\"\xb4\x01\n\x03RPC\x12-\n\rsubscriptions\x18\x01 \x03(\x0b\x32\x16.pubsub.pb.RPC.SubOpts\x12#\n\x07publish\x18\x02 \x03(\x0b\x32\x12.pubsub.pb.Message\x12*\n\x07\x63ontrol\x18\x03 \x01(\x0b\x32\x19.pubsub.pb.ControlMessage\x1a-\n\x07SubOpts\x12\x11\n\tsubscribe\x18\x01 \x01(\x08\x12\x0f\n\x07topicid\x18\x02 \x01(\t\"f\n\x07Message\x12\x0c\n\x04\x66rom\x18\x01 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\r\n\x05seqno\x18\x03 \x01(\x0c\x12\x10\n\x08topicIDs\x18\x04 \x03(\t\x12\x11\n\tsignature\x18\x05 \x01(\x0c\x12\x0b\n\x03key\x18\x06 \x01(\x0c\"\xb0\x01\n\x0e\x43ontrolMessage\x12&\n\x05ihave\x18\x01 \x03(\x0b\x32\x17.pubsub.pb.ControlIHave\x12&\n\x05iwant\x18\x02 \x03(\x0b\x32\x17.pubsub.pb.ControlIWant\x12&\n\x05graft\x18\x03 \x03(\x0b\x32\x17.pubsub.pb.ControlGraft\x12&\n\x05prune\x18\x04 \x03(\x0b\x32\x17.pubsub.pb.ControlPrune\"3\n\x0c\x43ontrolIHave\x12\x0f\n\x07topicID\x18\x01 \x01(\t\x12\x12\n\nmessageIDs\x18\x02 \x03(\t\"\"\n\x0c\x43ontrolIWant\x12\x12\n\nmessageIDs\x18\x01 \x03(\t\"\x1f\n\x0c\x43ontrolGraft\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x1f\n\x0c\x43ontrolPrune\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x87\x03\n\x0fTopicDescriptor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x31\n\x04\x61uth\x18\x02 \x01(\x0b\x32#.pubsub.pb.TopicDescriptor.AuthOpts\x12/\n\x03\x65nc\x18\x03 \x01(\x0b\x32\".pubsub.pb.TopicDescriptor.EncOpts\x1a|\n\x08\x41uthOpts\x12:\n\x04mode\x18\x01 \x01(\x0e\x32,.pubsub.pb.TopicDescriptor.AuthOpts.AuthMode\x12\x0c\n\x04keys\x18\x02 \x03(\x0c\"&\n\x08\x41uthMode\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03KEY\x10\x01\x12\x07\n\x03WOT\x10\x02\x1a\x83\x01\n\x07\x45ncOpts\x12\x38\n\x04mode\x18\x01 \x01(\x0e\x32*.pubsub.pb.TopicDescriptor.EncOpts.EncMode\x12\x11\n\tkeyHashes\x18\x02 \x03(\x0c\"+\n\x07\x45ncMode\x12\x08\n\x04NONE\x10\x00\x12\r\n\tSHAREDKEY\x10\x01\x12\x07\n\x03WOT\x10\x02') +) + + + +_TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE = _descriptor.EnumDescriptor( + name='AuthMode', + full_name='pubsub.pb.TopicDescriptor.AuthOpts.AuthMode', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='NONE', index=0, number=0, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='KEY', index=1, number=1, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='WOT', index=2, number=2, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=865, + serialized_end=903, +) +_sym_db.RegisterEnumDescriptor(_TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE) + +_TOPICDESCRIPTOR_ENCOPTS_ENCMODE = _descriptor.EnumDescriptor( + name='EncMode', + full_name='pubsub.pb.TopicDescriptor.EncOpts.EncMode', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='NONE', index=0, number=0, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='SHAREDKEY', index=1, number=1, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='WOT', index=2, number=2, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=994, + serialized_end=1037, +) +_sym_db.RegisterEnumDescriptor(_TOPICDESCRIPTOR_ENCOPTS_ENCMODE) + + +_RPC_SUBOPTS = _descriptor.Descriptor( + name='SubOpts', + full_name='pubsub.pb.RPC.SubOpts', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='subscribe', full_name='pubsub.pb.RPC.SubOpts.subscribe', index=0, + number=1, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='topicid', full_name='pubsub.pb.RPC.SubOpts.topicid', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=160, + serialized_end=205, +) + +_RPC = _descriptor.Descriptor( + name='RPC', + full_name='pubsub.pb.RPC', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='subscriptions', full_name='pubsub.pb.RPC.subscriptions', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='publish', full_name='pubsub.pb.RPC.publish', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='control', full_name='pubsub.pb.RPC.control', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_RPC_SUBOPTS, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=25, + serialized_end=205, +) + + +_MESSAGE = _descriptor.Descriptor( + name='Message', + full_name='pubsub.pb.Message', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='from', full_name='pubsub.pb.Message.from', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='data', full_name='pubsub.pb.Message.data', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='seqno', full_name='pubsub.pb.Message.seqno', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='topicIDs', full_name='pubsub.pb.Message.topicIDs', index=3, + number=4, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='signature', full_name='pubsub.pb.Message.signature', index=4, + number=5, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='key', full_name='pubsub.pb.Message.key', index=5, + number=6, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=207, + serialized_end=309, +) + + +_CONTROLMESSAGE = _descriptor.Descriptor( + name='ControlMessage', + full_name='pubsub.pb.ControlMessage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='ihave', full_name='pubsub.pb.ControlMessage.ihave', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='iwant', full_name='pubsub.pb.ControlMessage.iwant', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='graft', full_name='pubsub.pb.ControlMessage.graft', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='prune', full_name='pubsub.pb.ControlMessage.prune', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=312, + serialized_end=488, +) + + +_CONTROLIHAVE = _descriptor.Descriptor( + name='ControlIHave', + full_name='pubsub.pb.ControlIHave', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='topicID', full_name='pubsub.pb.ControlIHave.topicID', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='messageIDs', full_name='pubsub.pb.ControlIHave.messageIDs', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=490, + serialized_end=541, +) + + +_CONTROLIWANT = _descriptor.Descriptor( + name='ControlIWant', + full_name='pubsub.pb.ControlIWant', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='messageIDs', full_name='pubsub.pb.ControlIWant.messageIDs', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=543, + serialized_end=577, +) + + +_CONTROLGRAFT = _descriptor.Descriptor( + name='ControlGraft', + full_name='pubsub.pb.ControlGraft', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='topicID', full_name='pubsub.pb.ControlGraft.topicID', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=579, + serialized_end=610, +) + + +_CONTROLPRUNE = _descriptor.Descriptor( + name='ControlPrune', + full_name='pubsub.pb.ControlPrune', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='topicID', full_name='pubsub.pb.ControlPrune.topicID', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=612, + serialized_end=643, +) + + +_TOPICDESCRIPTOR_AUTHOPTS = _descriptor.Descriptor( + name='AuthOpts', + full_name='pubsub.pb.TopicDescriptor.AuthOpts', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='mode', full_name='pubsub.pb.TopicDescriptor.AuthOpts.mode', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='keys', full_name='pubsub.pb.TopicDescriptor.AuthOpts.keys', index=1, + number=2, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE, + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=779, + serialized_end=903, +) + +_TOPICDESCRIPTOR_ENCOPTS = _descriptor.Descriptor( + name='EncOpts', + full_name='pubsub.pb.TopicDescriptor.EncOpts', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='mode', full_name='pubsub.pb.TopicDescriptor.EncOpts.mode', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='keyHashes', full_name='pubsub.pb.TopicDescriptor.EncOpts.keyHashes', index=1, + number=2, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _TOPICDESCRIPTOR_ENCOPTS_ENCMODE, + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=906, + serialized_end=1037, +) + +_TOPICDESCRIPTOR = _descriptor.Descriptor( + name='TopicDescriptor', + full_name='pubsub.pb.TopicDescriptor', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='pubsub.pb.TopicDescriptor.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='auth', full_name='pubsub.pb.TopicDescriptor.auth', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='enc', full_name='pubsub.pb.TopicDescriptor.enc', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_TOPICDESCRIPTOR_AUTHOPTS, _TOPICDESCRIPTOR_ENCOPTS, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=646, + serialized_end=1037, +) + +_RPC_SUBOPTS.containing_type = _RPC +_RPC.fields_by_name['subscriptions'].message_type = _RPC_SUBOPTS +_RPC.fields_by_name['publish'].message_type = _MESSAGE +_RPC.fields_by_name['control'].message_type = _CONTROLMESSAGE +_CONTROLMESSAGE.fields_by_name['ihave'].message_type = _CONTROLIHAVE +_CONTROLMESSAGE.fields_by_name['iwant'].message_type = _CONTROLIWANT +_CONTROLMESSAGE.fields_by_name['graft'].message_type = _CONTROLGRAFT +_CONTROLMESSAGE.fields_by_name['prune'].message_type = _CONTROLPRUNE +_TOPICDESCRIPTOR_AUTHOPTS.fields_by_name['mode'].enum_type = _TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE +_TOPICDESCRIPTOR_AUTHOPTS.containing_type = _TOPICDESCRIPTOR +_TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE.containing_type = _TOPICDESCRIPTOR_AUTHOPTS +_TOPICDESCRIPTOR_ENCOPTS.fields_by_name['mode'].enum_type = _TOPICDESCRIPTOR_ENCOPTS_ENCMODE +_TOPICDESCRIPTOR_ENCOPTS.containing_type = _TOPICDESCRIPTOR +_TOPICDESCRIPTOR_ENCOPTS_ENCMODE.containing_type = _TOPICDESCRIPTOR_ENCOPTS +_TOPICDESCRIPTOR.fields_by_name['auth'].message_type = _TOPICDESCRIPTOR_AUTHOPTS +_TOPICDESCRIPTOR.fields_by_name['enc'].message_type = _TOPICDESCRIPTOR_ENCOPTS +DESCRIPTOR.message_types_by_name['RPC'] = _RPC +DESCRIPTOR.message_types_by_name['Message'] = _MESSAGE +DESCRIPTOR.message_types_by_name['ControlMessage'] = _CONTROLMESSAGE +DESCRIPTOR.message_types_by_name['ControlIHave'] = _CONTROLIHAVE +DESCRIPTOR.message_types_by_name['ControlIWant'] = _CONTROLIWANT +DESCRIPTOR.message_types_by_name['ControlGraft'] = _CONTROLGRAFT +DESCRIPTOR.message_types_by_name['ControlPrune'] = _CONTROLPRUNE +DESCRIPTOR.message_types_by_name['TopicDescriptor'] = _TOPICDESCRIPTOR +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +RPC = _reflection.GeneratedProtocolMessageType('RPC', (_message.Message,), dict( + + SubOpts = _reflection.GeneratedProtocolMessageType('SubOpts', (_message.Message,), dict( + DESCRIPTOR = _RPC_SUBOPTS, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.RPC.SubOpts) + )) + , + DESCRIPTOR = _RPC, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.RPC) + )) +_sym_db.RegisterMessage(RPC) +_sym_db.RegisterMessage(RPC.SubOpts) + +Message = _reflection.GeneratedProtocolMessageType('Message', (_message.Message,), dict( + DESCRIPTOR = _MESSAGE, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.Message) + )) +_sym_db.RegisterMessage(Message) + +ControlMessage = _reflection.GeneratedProtocolMessageType('ControlMessage', (_message.Message,), dict( + DESCRIPTOR = _CONTROLMESSAGE, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlMessage) + )) +_sym_db.RegisterMessage(ControlMessage) + +ControlIHave = _reflection.GeneratedProtocolMessageType('ControlIHave', (_message.Message,), dict( + DESCRIPTOR = _CONTROLIHAVE, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlIHave) + )) +_sym_db.RegisterMessage(ControlIHave) + +ControlIWant = _reflection.GeneratedProtocolMessageType('ControlIWant', (_message.Message,), dict( + DESCRIPTOR = _CONTROLIWANT, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlIWant) + )) +_sym_db.RegisterMessage(ControlIWant) + +ControlGraft = _reflection.GeneratedProtocolMessageType('ControlGraft', (_message.Message,), dict( + DESCRIPTOR = _CONTROLGRAFT, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlGraft) + )) +_sym_db.RegisterMessage(ControlGraft) + +ControlPrune = _reflection.GeneratedProtocolMessageType('ControlPrune', (_message.Message,), dict( + DESCRIPTOR = _CONTROLPRUNE, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlPrune) + )) +_sym_db.RegisterMessage(ControlPrune) + +TopicDescriptor = _reflection.GeneratedProtocolMessageType('TopicDescriptor', (_message.Message,), dict( + + AuthOpts = _reflection.GeneratedProtocolMessageType('AuthOpts', (_message.Message,), dict( + DESCRIPTOR = _TOPICDESCRIPTOR_AUTHOPTS, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.TopicDescriptor.AuthOpts) + )) + , + + EncOpts = _reflection.GeneratedProtocolMessageType('EncOpts', (_message.Message,), dict( + DESCRIPTOR = _TOPICDESCRIPTOR_ENCOPTS, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.TopicDescriptor.EncOpts) + )) + , + DESCRIPTOR = _TOPICDESCRIPTOR, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.TopicDescriptor) + )) +_sym_db.RegisterMessage(TopicDescriptor) +_sym_db.RegisterMessage(TopicDescriptor.AuthOpts) +_sym_db.RegisterMessage(TopicDescriptor.EncOpts) + + +# @@protoc_insertion_point(module_scope) diff --git a/libp2p/pubsub/pb/rpc_pb2_grpc.py b/libp2p/pubsub/pb/rpc_pb2_grpc.py new file mode 100644 index 0000000..a894352 --- /dev/null +++ b/libp2p/pubsub/pb/rpc_pb2_grpc.py @@ -0,0 +1,3 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + From 2b41818561e77f6230289d3829f404bd037a63d9 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 15:23:56 -0400 Subject: [PATCH 052/131] update from to from_id in proto --- libp2p/pubsub/pb/rpc.proto | 2 +- libp2p/pubsub/pb/rpc_pb2.py | 46 ++++++++++++++++++------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/libp2p/pubsub/pb/rpc.proto b/libp2p/pubsub/pb/rpc.proto index 764b033..f07da20 100644 --- a/libp2p/pubsub/pb/rpc.proto +++ b/libp2p/pubsub/pb/rpc.proto @@ -15,7 +15,7 @@ message RPC { } message Message { - optional bytes from = 1; + optional bytes from_id = 1; optional bytes data = 2; optional bytes seqno = 3; repeated string topicIDs = 4; diff --git a/libp2p/pubsub/pb/rpc_pb2.py b/libp2p/pubsub/pb/rpc_pb2.py index 57035ed..d315782 100644 --- a/libp2p/pubsub/pb/rpc_pb2.py +++ b/libp2p/pubsub/pb/rpc_pb2.py @@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( package='pubsub.pb', syntax='proto2', serialized_options=None, - serialized_pb=_b('\n\trpc.proto\x12\tpubsub.pb\"\xb4\x01\n\x03RPC\x12-\n\rsubscriptions\x18\x01 \x03(\x0b\x32\x16.pubsub.pb.RPC.SubOpts\x12#\n\x07publish\x18\x02 \x03(\x0b\x32\x12.pubsub.pb.Message\x12*\n\x07\x63ontrol\x18\x03 \x01(\x0b\x32\x19.pubsub.pb.ControlMessage\x1a-\n\x07SubOpts\x12\x11\n\tsubscribe\x18\x01 \x01(\x08\x12\x0f\n\x07topicid\x18\x02 \x01(\t\"f\n\x07Message\x12\x0c\n\x04\x66rom\x18\x01 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\r\n\x05seqno\x18\x03 \x01(\x0c\x12\x10\n\x08topicIDs\x18\x04 \x03(\t\x12\x11\n\tsignature\x18\x05 \x01(\x0c\x12\x0b\n\x03key\x18\x06 \x01(\x0c\"\xb0\x01\n\x0e\x43ontrolMessage\x12&\n\x05ihave\x18\x01 \x03(\x0b\x32\x17.pubsub.pb.ControlIHave\x12&\n\x05iwant\x18\x02 \x03(\x0b\x32\x17.pubsub.pb.ControlIWant\x12&\n\x05graft\x18\x03 \x03(\x0b\x32\x17.pubsub.pb.ControlGraft\x12&\n\x05prune\x18\x04 \x03(\x0b\x32\x17.pubsub.pb.ControlPrune\"3\n\x0c\x43ontrolIHave\x12\x0f\n\x07topicID\x18\x01 \x01(\t\x12\x12\n\nmessageIDs\x18\x02 \x03(\t\"\"\n\x0c\x43ontrolIWant\x12\x12\n\nmessageIDs\x18\x01 \x03(\t\"\x1f\n\x0c\x43ontrolGraft\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x1f\n\x0c\x43ontrolPrune\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x87\x03\n\x0fTopicDescriptor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x31\n\x04\x61uth\x18\x02 \x01(\x0b\x32#.pubsub.pb.TopicDescriptor.AuthOpts\x12/\n\x03\x65nc\x18\x03 \x01(\x0b\x32\".pubsub.pb.TopicDescriptor.EncOpts\x1a|\n\x08\x41uthOpts\x12:\n\x04mode\x18\x01 \x01(\x0e\x32,.pubsub.pb.TopicDescriptor.AuthOpts.AuthMode\x12\x0c\n\x04keys\x18\x02 \x03(\x0c\"&\n\x08\x41uthMode\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03KEY\x10\x01\x12\x07\n\x03WOT\x10\x02\x1a\x83\x01\n\x07\x45ncOpts\x12\x38\n\x04mode\x18\x01 \x01(\x0e\x32*.pubsub.pb.TopicDescriptor.EncOpts.EncMode\x12\x11\n\tkeyHashes\x18\x02 \x03(\x0c\"+\n\x07\x45ncMode\x12\x08\n\x04NONE\x10\x00\x12\r\n\tSHAREDKEY\x10\x01\x12\x07\n\x03WOT\x10\x02') + serialized_pb=_b('\n\trpc.proto\x12\tpubsub.pb\"\xb4\x01\n\x03RPC\x12-\n\rsubscriptions\x18\x01 \x03(\x0b\x32\x16.pubsub.pb.RPC.SubOpts\x12#\n\x07publish\x18\x02 \x03(\x0b\x32\x12.pubsub.pb.Message\x12*\n\x07\x63ontrol\x18\x03 \x01(\x0b\x32\x19.pubsub.pb.ControlMessage\x1a-\n\x07SubOpts\x12\x11\n\tsubscribe\x18\x01 \x01(\x08\x12\x0f\n\x07topicid\x18\x02 \x01(\t\"i\n\x07Message\x12\x0f\n\x07\x66rom_id\x18\x01 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\r\n\x05seqno\x18\x03 \x01(\x0c\x12\x10\n\x08topicIDs\x18\x04 \x03(\t\x12\x11\n\tsignature\x18\x05 \x01(\x0c\x12\x0b\n\x03key\x18\x06 \x01(\x0c\"\xb0\x01\n\x0e\x43ontrolMessage\x12&\n\x05ihave\x18\x01 \x03(\x0b\x32\x17.pubsub.pb.ControlIHave\x12&\n\x05iwant\x18\x02 \x03(\x0b\x32\x17.pubsub.pb.ControlIWant\x12&\n\x05graft\x18\x03 \x03(\x0b\x32\x17.pubsub.pb.ControlGraft\x12&\n\x05prune\x18\x04 \x03(\x0b\x32\x17.pubsub.pb.ControlPrune\"3\n\x0c\x43ontrolIHave\x12\x0f\n\x07topicID\x18\x01 \x01(\t\x12\x12\n\nmessageIDs\x18\x02 \x03(\t\"\"\n\x0c\x43ontrolIWant\x12\x12\n\nmessageIDs\x18\x01 \x03(\t\"\x1f\n\x0c\x43ontrolGraft\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x1f\n\x0c\x43ontrolPrune\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x87\x03\n\x0fTopicDescriptor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x31\n\x04\x61uth\x18\x02 \x01(\x0b\x32#.pubsub.pb.TopicDescriptor.AuthOpts\x12/\n\x03\x65nc\x18\x03 \x01(\x0b\x32\".pubsub.pb.TopicDescriptor.EncOpts\x1a|\n\x08\x41uthOpts\x12:\n\x04mode\x18\x01 \x01(\x0e\x32,.pubsub.pb.TopicDescriptor.AuthOpts.AuthMode\x12\x0c\n\x04keys\x18\x02 \x03(\x0c\"&\n\x08\x41uthMode\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03KEY\x10\x01\x12\x07\n\x03WOT\x10\x02\x1a\x83\x01\n\x07\x45ncOpts\x12\x38\n\x04mode\x18\x01 \x01(\x0e\x32*.pubsub.pb.TopicDescriptor.EncOpts.EncMode\x12\x11\n\tkeyHashes\x18\x02 \x03(\x0c\"+\n\x07\x45ncMode\x12\x08\n\x04NONE\x10\x00\x12\r\n\tSHAREDKEY\x10\x01\x12\x07\n\x03WOT\x10\x02') ) @@ -45,8 +45,8 @@ _TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE = _descriptor.EnumDescriptor( ], containing_type=None, serialized_options=None, - serialized_start=865, - serialized_end=903, + serialized_start=868, + serialized_end=906, ) _sym_db.RegisterEnumDescriptor(_TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE) @@ -71,8 +71,8 @@ _TOPICDESCRIPTOR_ENCOPTS_ENCMODE = _descriptor.EnumDescriptor( ], containing_type=None, serialized_options=None, - serialized_start=994, - serialized_end=1037, + serialized_start=997, + serialized_end=1040, ) _sym_db.RegisterEnumDescriptor(_TOPICDESCRIPTOR_ENCOPTS_ENCMODE) @@ -167,7 +167,7 @@ _MESSAGE = _descriptor.Descriptor( containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='from', full_name='pubsub.pb.Message.from', index=0, + name='from_id', full_name='pubsub.pb.Message.from_id', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, @@ -221,7 +221,7 @@ _MESSAGE = _descriptor.Descriptor( oneofs=[ ], serialized_start=207, - serialized_end=309, + serialized_end=312, ) @@ -272,8 +272,8 @@ _CONTROLMESSAGE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=312, - serialized_end=488, + serialized_start=315, + serialized_end=491, ) @@ -310,8 +310,8 @@ _CONTROLIHAVE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=490, - serialized_end=541, + serialized_start=493, + serialized_end=544, ) @@ -341,8 +341,8 @@ _CONTROLIWANT = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=543, - serialized_end=577, + serialized_start=546, + serialized_end=580, ) @@ -372,8 +372,8 @@ _CONTROLGRAFT = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=579, - serialized_end=610, + serialized_start=582, + serialized_end=613, ) @@ -403,8 +403,8 @@ _CONTROLPRUNE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=612, - serialized_end=643, + serialized_start=615, + serialized_end=646, ) @@ -442,8 +442,8 @@ _TOPICDESCRIPTOR_AUTHOPTS = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=779, - serialized_end=903, + serialized_start=782, + serialized_end=906, ) _TOPICDESCRIPTOR_ENCOPTS = _descriptor.Descriptor( @@ -480,8 +480,8 @@ _TOPICDESCRIPTOR_ENCOPTS = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=906, - serialized_end=1037, + serialized_start=909, + serialized_end=1040, ) _TOPICDESCRIPTOR = _descriptor.Descriptor( @@ -524,8 +524,8 @@ _TOPICDESCRIPTOR = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=646, - serialized_end=1037, + serialized_start=649, + serialized_end=1040, ) _RPC_SUBOPTS.containing_type = _RPC From b297aa95766f73a5428faf4fccf7e77ab7dbee2a Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 15:25:33 -0400 Subject: [PATCH 053/131] rewrote get_hello_packet --- libp2p/pubsub/pubsub.py | 74 +++++++++++++---------------------------- 1 file changed, 24 insertions(+), 50 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 9bd072f..17ecdd5 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -1,5 +1,7 @@ import asyncio +from .pb import rpc_pb2_grpc +from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee from .message import MessageSub from .message import create_message_talk, create_message_sub @@ -7,37 +9,6 @@ from. message import generate_message_id class Pubsub(): - """ - For now, because I'm on a plane and don't have access to the go repo/protobuf stuff, - this is going to be the message format for the two types: subscription and talk - subscription indicates subscribing or unsubscribing from a topic - talk is sending a message on topic(s) - subscription format: - subscription - 'from' - :'topicid' - :'topicid' - ... - Ex. - subscription - msg_sender_peer_id - origin_peer_id - sub:topic1 - sub:topic2 - unsub:fav_topic - talk format: - talk - 'from' - 'origin' - [topic_ids comma-delimited] - 'data' - Ex. - talk - msg_sender_peer_id - origin_peer_id - topic1,topics_are_cool,foo - I like tacos - """ # pylint: disable=too-many-instance-attributes def __init__(self, host, router, my_id): @@ -90,34 +61,37 @@ class Pubsub(): """ Generate subscription message with all topics we are subscribed to """ - subs_map = {} - for topic in self.my_topics: - subs_map[topic] = True - sub_msg = MessageSub( - str(self.host.get_id()),\ - str(self.host.get_id()), subs_map, generate_message_id()\ - ) - return sub_msg.to_str() + packet = rpc_pb2.RPC() + message = rpc_pb2.Message( + from_id=str(self.host.get_id()).encode('utf-8'), + seqno=str(generate_message_id()).encode('utf-8') + ) + packet.publish.extend([message]) + for topic_id in self.my_topics: + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe=True, topicid=topic_id)]) + + return packet.SerializeToString() async def continously_read_stream(self, stream): """ Read from input stream in an infinite loop. Process messages from other nodes, which for now are considered MessageTalk and MessageSub messages. - TODO: Handle RPC messages instead of my Aspyn's own custom message format :param stream: stream to continously read from """ while True: - incoming = (await stream.read()).decode() - msg_comps = incoming.split('\n') - msg_type = msg_comps[0] + incoming = (await stream.read()) + rpc_incoming = rpc_pb2.RPC() + rpc_incoming.ParseFromString(incoming) + print (rpc_incoming.publish) - msg_sender = msg_comps[1] + # msg_type = msg_comps[0] + + msg_sender = rpc_incoming.publish.from_id # msg_origin = msg_comps[2] - msg_id = msg_comps[3] - print("HIT ME1") + msg_id = rpc_incoming.publish.seqno if msg_id not in self.seen_messages: - print("HIT ME") # Do stuff with incoming unseen message should_publish = True if msg_type == "subscription": @@ -161,7 +135,7 @@ class Pubsub(): # Send hello packet hello = self.get_hello_packet() - await stream.write(hello.encode()) + await stream.write(hello) # Pass stream off to stream reader asyncio.ensure_future(self.continously_read_stream(stream)) @@ -187,7 +161,7 @@ class Pubsub(): # Send hello packet hello = self.get_hello_packet() - await stream.write(hello.encode()) + await stream.write(hello) # Pass stream off to stream reader asyncio.ensure_future(self.continously_read_stream(stream)) @@ -291,4 +265,4 @@ class Pubsub(): stream = self.peers[peer] # Write message to stream - await stream.write(encoded_msg) + await stream.write(encoded_msg) \ No newline at end of file From adce7f0a051f5dac81306ee4b54894ea766055d8 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 15:42:55 -0400 Subject: [PATCH 054/131] update dependencies --- requirements_dev.txt | 2 ++ setup.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/requirements_dev.txt b/requirements_dev.txt index 4bf18f6..e3d8c0e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -3,3 +3,5 @@ codecov pytest-cov pytest-asyncio pylint +grpcio +grpcio-tools diff --git a/setup.py b/setup.py index abaa1c7..c44a276 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,8 @@ setuptools.setup( "base58", "pymultihash", "multiaddr", + "grpcio", + "grpcio-tools" ], packages=["libp2p"], zip_safe=False, From 75c12d4aeda12f515b112ef38d28230d5ef97c07 Mon Sep 17 00:00:00 2001 From: Alex Haynes Date: Fri, 29 Mar 2019 16:23:30 -0400 Subject: [PATCH 055/131] RPC conversion progress --- libp2p/pubsub/floodsub.py | 41 ++++++++++++++----- libp2p/pubsub/pubsub.py | 85 +++++++++++++++++++++++++++------------ 2 files changed, 91 insertions(+), 35 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 68e542c..1cea17e 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -1,4 +1,6 @@ from .pubsub_router_interface import IPubsubRouter +from .pb import rpc_pb2 +from .message import MessageSub, MessageTalk from .message import create_message_talk class FloodSub(IPubsubRouter): @@ -56,28 +58,49 @@ class FloodSub(IPubsubRouter): """ # Encode message - encoded_msg = message.encode() + # encoded_msg = message.encode() + + if isinstance(message, str): + msg_talk = create_message_talk(message) + message = rpc_pb2.Message( + from_id=str(msg_talk.origin_id).encode('utf-8'), + seqno=str(msg_talk.message_id).encode('utf-8'), + topicIDs=msg_talk.topics, + data=msg_talk.data.encode() + + ) + packet = rpc_pb2.RPC() + print("YEET") + print(type(message)) + packet.publish.extend([message]) + + # Get message sender, origin, and topics - msg_talk = create_message_talk(message) + # msg_talk = create_message_talk(message) msg_sender = str(sender_peer_id) - msg_origin = msg_talk.origin_id - topics = msg_talk.topics + # msg_origin = msg_talk.origin_id + # topics = msg_talk.topics # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message - if msg_sender == msg_origin and msg_sender == str(self.pubsub.host.get_id()): - await self.pubsub.handle_talk(message) + if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): + old_format = MessageTalk(sender_peer_id, + message.from_id, + message.topicIDs, + message.data, + message.seqno) + await self.pubsub.handle_talk(old_format) # Deliver to self and peers - for topic in topics: + for topic in message.topicIDs: if topic in self.pubsub.peer_topics: for peer_id_in_topic in self.pubsub.peer_topics[topic]: # Forward to all known peers in the topic that are not the # message sender and are not the message origin - if peer_id_in_topic not in (msg_sender, msg_origin): + if peer_id_in_topic not in (msg_sender, message.from_id): stream = self.pubsub.peers[peer_id_in_topic] - await stream.write(encoded_msg) + await stream.write(packet.SerializeToString()) else: # Implies publish did not write print("publish did not write") diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 17ecdd5..b1ee363 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -3,7 +3,7 @@ import asyncio from .pb import rpc_pb2_grpc from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee -from .message import MessageSub +from .message import MessageSub, MessageTalk from .message import create_message_talk, create_message_sub from. message import generate_message_id @@ -61,6 +61,7 @@ class Pubsub(): """ Generate subscription message with all topics we are subscribed to """ + print("MAKING HELLO PACKET") packet = rpc_pb2.RPC() message = rpc_pb2.Message( from_id=str(self.host.get_id()).encode('utf-8'), @@ -80,43 +81,74 @@ class Pubsub(): and MessageSub messages. :param stream: stream to continously read from """ + + # TODO check on types here + peer_id = stream.mplex_conn.peer_id + while True: incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() + print("JUST GOT STRING") + print(incoming) rpc_incoming.ParseFromString(incoming) - print (rpc_incoming.publish) + print("JUST GOT") + print (rpc_incoming) + + if hasattr(rpc_incoming, 'publish'): + # deal with "talk messages" + for msg in rpc_incoming.publish: + # TODO check what from_id is in go? is it origin or peer + old_format = MessageTalk(peer_id, + msg.from_id, + msg.topicIDs, + msg.data, + msg.seqno) + self.seen_messages.append(msg.seqno) + await self.handle_talk(old_format) + await self.router.publish(peer_id, msg) + + if hasattr(rpc_incoming, 'subscriptions'): + # deal with "subscription messages" + subs_map = {} + for msg in rpc_incoming.subscriptions: + if msg.subscribe: + subs_map[msg.topic_id] = "sub" + else: + subs_map[msg.topic_id] = "unsub" + old_format = MessageSub(peer_id, peer_id, subs_map, generate_message_id()) + self.handle_subscription(old_format) # msg_type = msg_comps[0] - msg_sender = rpc_incoming.publish.from_id - # msg_origin = msg_comps[2] - msg_id = rpc_incoming.publish.seqno - if msg_id not in self.seen_messages: - # Do stuff with incoming unseen message - should_publish = True - if msg_type == "subscription": - self.handle_subscription(incoming) + # msg_sender = rpc_incoming.publish.from_id + # # msg_origin = msg_comps[2] + # msg_id = rpc_incoming.publish.seqno + # if msg_id not in self.seen_messages: + # # Do stuff with incoming unseen message + # should_publish = True + # if msg_type == "subscription": + # self.handle_subscription(incoming) - # We don't need to relay the subscription to our - # peers because a given node only needs its peers - # to know that it is subscribed to the topic (doesn't - # need everyone to know) - should_publish = False - elif msg_type == "talk": - await self.handle_talk(incoming) + # # We don't need to relay the subscription to our + # # peers because a given node only needs its peers + # # to know that it is subscribed to the topic (doesn't + # # need everyone to know) + # should_publish = False + # elif msg_type == "talk": + # await self.handle_talk(incoming) # Add message id to seen - self.seen_messages.append(msg_id) + # self.seen_messages.append(msg_id) # Publish message using router's publish - if should_publish: - msg = create_message_talk(incoming) + # if should_publish: + # msg = create_message_talk(incoming) - # Adjust raw_msg to that the message sender - # is now our peer_id - msg.from_id = str(self.host.get_id()) + # # Adjust raw_msg to that the message sender + # # is now our peer_id + # msg.from_id = str(self.host.get_id()) - await self.router.publish(msg_sender, msg.to_str()) + # await self.router.publish(msg_sender, msg.to_str()) # Force context switch await asyncio.sleep(0) @@ -135,6 +167,7 @@ class Pubsub(): # Send hello packet hello = self.get_hello_packet() + print(hello) await stream.write(hello) # Pass stream off to stream reader asyncio.ensure_future(self.continously_read_stream(stream)) @@ -176,7 +209,7 @@ class Pubsub(): defined in the subscription message :param subscription: raw data constituting a subscription message """ - sub_msg = create_message_sub(subscription) + sub_msg = subscription if sub_msg.subs_map: print("handle_subscription my_id: " + self.my_id + ", subber: " + sub_msg.origin_id) for topic_id in sub_msg.subs_map: @@ -198,7 +231,7 @@ class Pubsub(): custom message that is published on a given topic(s) :param talk: raw data constituting a talk message """ - msg = create_message_talk(talk) + msg = talk # Check if this message has any topics that we are subscribed to for topic in msg.topics: From 598b6b257f8da0db8af3a27f7c4c2bca7d31d85e Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sat, 30 Mar 2019 17:59:08 -0400 Subject: [PATCH 056/131] reworked subscribe unsubsrcibe --- libp2p/pubsub/pubsub.py | 113 +++++++++++++++------------------------- 1 file changed, 43 insertions(+), 70 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index b1ee363..85abacb 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -61,7 +61,7 @@ class Pubsub(): """ Generate subscription message with all topics we are subscribed to """ - print("MAKING HELLO PACKET") + packet = rpc_pb2.RPC() message = rpc_pb2.Message( from_id=str(self.host.get_id()).encode('utf-8'), @@ -74,7 +74,7 @@ class Pubsub(): return packet.SerializeToString() - async def continously_read_stream(self, stream): + async def continuously_read_stream(self, stream): """ Read from input stream in an infinite loop. Process messages from other nodes, which for now are considered MessageTalk @@ -88,16 +88,12 @@ class Pubsub(): while True: incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() - print("JUST GOT STRING") - print(incoming) rpc_incoming.ParseFromString(incoming) - print("JUST GOT") + print ("CONTINUOUSLY") print (rpc_incoming) - - if hasattr(rpc_incoming, 'publish'): + if rpc_incoming.publish: # deal with "talk messages" for msg in rpc_incoming.publish: - # TODO check what from_id is in go? is it origin or peer old_format = MessageTalk(peer_id, msg.from_id, msg.topicIDs, @@ -107,48 +103,16 @@ class Pubsub(): await self.handle_talk(old_format) await self.router.publish(peer_id, msg) - if hasattr(rpc_incoming, 'subscriptions'): + if rpc_incoming.subscriptions: # deal with "subscription messages" subs_map = {} for msg in rpc_incoming.subscriptions: if msg.subscribe: - subs_map[msg.topic_id] = "sub" + subs_map[msg.topicid] = "sub" else: - subs_map[msg.topic_id] = "unsub" - old_format = MessageSub(peer_id, peer_id, subs_map, generate_message_id()) - self.handle_subscription(old_format) + subs_map[msg.topicid] = "unsub" - # msg_type = msg_comps[0] - - # msg_sender = rpc_incoming.publish.from_id - # # msg_origin = msg_comps[2] - # msg_id = rpc_incoming.publish.seqno - # if msg_id not in self.seen_messages: - # # Do stuff with incoming unseen message - # should_publish = True - # if msg_type == "subscription": - # self.handle_subscription(incoming) - - # # We don't need to relay the subscription to our - # # peers because a given node only needs its peers - # # to know that it is subscribed to the topic (doesn't - # # need everyone to know) - # should_publish = False - # elif msg_type == "talk": - # await self.handle_talk(incoming) - - # Add message id to seen - # self.seen_messages.append(msg_id) - - # Publish message using router's publish - # if should_publish: - # msg = create_message_talk(incoming) - - # # Adjust raw_msg to that the message sender - # # is now our peer_id - # msg.from_id = str(self.host.get_id()) - - # await self.router.publish(msg_sender, msg.to_str()) + self.handle_subscription(rpc_incoming) # Force context switch await asyncio.sleep(0) @@ -167,10 +131,10 @@ class Pubsub(): # Send hello packet hello = self.get_hello_packet() - print(hello) + await stream.write(hello) # Pass stream off to stream reader - asyncio.ensure_future(self.continously_read_stream(stream)) + asyncio.ensure_future(self.continuously_read_stream(stream)) async def handle_peer_queue(self): """ @@ -197,30 +161,30 @@ class Pubsub(): await stream.write(hello) # Pass stream off to stream reader - asyncio.ensure_future(self.continously_read_stream(stream)) + asyncio.ensure_future(self.continuously_read_stream(stream)) # Force context switch await asyncio.sleep(0) - def handle_subscription(self, subscription): + def handle_subscription(self, rpc_message): """ Handle an incoming subscription message from a peer. Update internal mapping to mark the peer as subscribed or unsubscribed to topics as defined in the subscription message :param subscription: raw data constituting a subscription message """ - sub_msg = subscription - if sub_msg.subs_map: - print("handle_subscription my_id: " + self.my_id + ", subber: " + sub_msg.origin_id) - for topic_id in sub_msg.subs_map: + for sub_msg in rpc_message.subscriptions: + # Look at each subscription in the msg individually - if sub_msg.subs_map[topic_id]: - if topic_id not in self.peer_topics: + if sub_msg.subscribe: + origin_id = rpc_message.publish[0].from_id + + if sub_msg.topicid not in self.peer_topics: # Create topic list if it did not yet exist - self.peer_topics[topic_id] = [sub_msg.origin_id] - elif sub_msg.origin_id not in self.peer_topics[topic_id]: + self.peer_topics[sub_msg.topicid] = origin_id + elif orgin_id not in self.peer_topics[sub_msg.topicid]: # Add peer to topic - self.peer_topics[topic_id].append(sub_msg.origin_id) + self.peer_topics[sub_msg.topicid].append(origin_id) else: # TODO: Remove peer from topic pass @@ -250,13 +214,18 @@ class Pubsub(): self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message - sub_msg = MessageSub( - str(self.host.get_id()),\ - str(self.host.get_id()), {topic_id: True}, generate_message_id()\ - ) + packet = rpc_pb2.RPC() + packet.publish.extend([rpc_pb2.Message( + from_id=str(self.host.get_id()).encode('utf-8'), + seqno=str(generate_message_id()).encode('utf-8') + )]) + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe = True, + topicid = topic_id.encode('utf-8') + )]) # Send out subscribe message to all peers - await self.message_all_peers(sub_msg.to_str()) + await self.message_all_peers(packet.SerializeToString()) # Tell router we are joining this topic self.router.join(topic_id) @@ -275,27 +244,31 @@ class Pubsub(): del self.my_topics[topic_id] # Create unsubscribe message - unsub_msg = MessageSub(str(self.host.get_id()), str(self.host.get_id()),\ - {topic_id: False}, generate_message_id()) + packet = rpc_pb2.RPC() + packet.publish.extend([rpc_pb2.Message( + from_id=str(self.host.get_id()).encode('utf-8'), + seqno=str(generate_message_id()).encode('utf-8') + )]) + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe = False, + topicid = topic_id.encode('utf-8') + )]) # Send out unsubscribe message to all peers - await self.message_all_peers(unsub_msg.to_str()) + await self.message_all_peers(packet.SerializeToString()) # Tell router we are leaving this topic self.router.leave(topic_id) - async def message_all_peers(self, raw_msg): + async def message_all_peers(self, rpc_msg): """ Broadcast a message to peers :param raw_msg: raw contents of the message to broadcast """ - # Encode message for sending - encoded_msg = raw_msg.encode() - # Broadcast message for peer in self.peers: stream = self.peers[peer] # Write message to stream - await stream.write(encoded_msg) \ No newline at end of file + await stream.write(rpc_msg) From 2183bada41772f8462978fc12e6da4b7e308fb5a Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sat, 30 Mar 2019 18:49:50 -0400 Subject: [PATCH 057/131] remove message.py --- libp2p/pubsub/floodsub.py | 31 +--------- libp2p/pubsub/message.py | 118 -------------------------------------- libp2p/pubsub/pubsub.py | 28 +++++---- 3 files changed, 15 insertions(+), 162 deletions(-) delete mode 100644 libp2p/pubsub/message.py diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 1cea17e..dd4dfc1 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -1,7 +1,6 @@ from .pubsub_router_interface import IPubsubRouter from .pb import rpc_pb2 -from .message import MessageSub, MessageTalk -from .message import create_message_talk + class FloodSub(IPubsubRouter): @@ -57,40 +56,14 @@ class FloodSub(IPubsubRouter): :param message: message to forward """ - # Encode message - # encoded_msg = message.encode() - - if isinstance(message, str): - msg_talk = create_message_talk(message) - message = rpc_pb2.Message( - from_id=str(msg_talk.origin_id).encode('utf-8'), - seqno=str(msg_talk.message_id).encode('utf-8'), - topicIDs=msg_talk.topics, - data=msg_talk.data.encode() - - ) packet = rpc_pb2.RPC() - print("YEET") - print(type(message)) packet.publish.extend([message]) - - - - # Get message sender, origin, and topics - # msg_talk = create_message_talk(message) msg_sender = str(sender_peer_id) - # msg_origin = msg_talk.origin_id - # topics = msg_talk.topics # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): - old_format = MessageTalk(sender_peer_id, - message.from_id, - message.topicIDs, - message.data, - message.seqno) - await self.pubsub.handle_talk(old_format) + await self.pubsub.handle_talk(sender_peer_id, message) # Deliver to self and peers for topic in message.topicIDs: diff --git a/libp2p/pubsub/message.py b/libp2p/pubsub/message.py deleted file mode 100644 index 2f839dc..0000000 --- a/libp2p/pubsub/message.py +++ /dev/null @@ -1,118 +0,0 @@ -import uuid - - -class MessageTalk(): - - """ - Object to make parsing talk messages easier, where talk messages are - defined as custom messages published to a set of topics - """ - # pylint: disable=too-few-public-methods - def __init__(self, from_id, origin_id, topics, data, message_id): - # pylint: disable=too-many-arguments - self.msg_type = "talk" - self.from_id = from_id - self.origin_id = origin_id - self.topics = topics - self.data = data - self.message_id = message_id - - def to_str(self): - """ - Convert to string - :return: MessageTalk object in string representation - """ - out = self.msg_type + '\n' - out += self.from_id + '\n' - out += self.origin_id + '\n' - out += self.message_id + '\n' - for i in range(len(self.topics)): - out += self.topics[i] - if i < len(self.topics) - 1: - out += ',' - out += '\n' + self.data - return out - - -class MessageSub(): - """ - Object to make parsing subscription messages easier, where subscription - messages are defined as indicating the topics a node wishes to subscribe to - or unsubscribe from - """ - # pylint: disable=too-few-public-methods - def __init__(self, from_id, origin_id, subs_map, message_id): - self.msg_type = "subscription" - self.from_id = from_id - self.origin_id = origin_id - self.subs_map = subs_map - self.message_id = message_id - - def to_str(self): - """ - Convert to string - :return: MessageSub object in string representation - """ - out = self.msg_type + '\n' - out += self.from_id + '\n' - out += self.origin_id + '\n' - out += self.message_id - - if self.subs_map: - out += '\n' - - keys = list(self.subs_map) - - for i, topic in enumerate(keys): - sub = self.subs_map[topic] - if sub: - out += "sub:" - else: - out += "unsub:" - out += topic - if i < len(keys) - 1: - out += '\n' - - return out - -def create_message_talk(msg_talk_as_str): - """ - Create a MessageTalk object from a MessageTalk string representation - :param msg_talk_as_str: a MessageTalk object in its string representation - :return: MessageTalk object - """ - msg_comps = msg_talk_as_str.split('\n') - from_id = msg_comps[1] - origin_id = msg_comps[2] - message_id = msg_comps[3] - topics = msg_comps[4].split(',') - data = msg_comps[5] - return MessageTalk(from_id, origin_id, topics, data, message_id) - -def create_message_sub(msg_sub_as_str): - """ - Create a MessageSub object from a MessageSub string representation - :param msg_talk_as_str: a MessageSub object in its string representation - :return: MessageSub object - """ - msg_comps = msg_sub_as_str.split('\n') - from_id = msg_comps[1] - origin_id = msg_comps[2] - message_id = msg_comps[3] - - subs_map = {} - for i in range(4, len(msg_comps)): - sub_comps = msg_comps[i].split(":") - topic = sub_comps[1] - if sub_comps[0] == "sub": - subs_map[topic] = True - else: - subs_map[topic] = False - return MessageSub(from_id, origin_id, subs_map, message_id) - -def generate_message_id(): - """ - Generate a unique message id - :return: messgae id - """ - return str(uuid.uuid1()) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 85abacb..976e6b4 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -1,11 +1,9 @@ import asyncio +import uuid from .pb import rpc_pb2_grpc from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee -from .message import MessageSub, MessageTalk -from .message import create_message_talk, create_message_sub -from. message import generate_message_id class Pubsub(): @@ -77,8 +75,7 @@ class Pubsub(): async def continuously_read_stream(self, stream): """ Read from input stream in an infinite loop. Process - messages from other nodes, which for now are considered MessageTalk - and MessageSub messages. + messages from other nodes :param stream: stream to continously read from """ @@ -94,13 +91,8 @@ class Pubsub(): if rpc_incoming.publish: # deal with "talk messages" for msg in rpc_incoming.publish: - old_format = MessageTalk(peer_id, - msg.from_id, - msg.topicIDs, - msg.data, - msg.seqno) self.seen_messages.append(msg.seqno) - await self.handle_talk(old_format) + await self.handle_talk(peer_id, msg) await self.router.publish(peer_id, msg) if rpc_incoming.subscriptions: @@ -189,21 +181,20 @@ class Pubsub(): # TODO: Remove peer from topic pass - async def handle_talk(self, talk): + async def handle_talk(self, peer_id, publish_message): """ Handle incoming Talk message from a peer. A Talk message contains some custom message that is published on a given topic(s) :param talk: raw data constituting a talk message """ - msg = talk # Check if this message has any topics that we are subscribed to - for topic in msg.topics: + for topic in publish_message.topicIDs: if topic in self.my_topics: # we are subscribed to a topic this message was sent for, # so add message to the subscription output queue # for each topic - await self.my_topics[topic].put(talk) + await self.my_topics[topic].put(publish_message) async def subscribe(self, topic_id): """ @@ -272,3 +263,10 @@ class Pubsub(): # Write message to stream await stream.write(rpc_msg) + +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) \ No newline at end of file From a2e22159b2c29f96c8cf1c13e66919bc431da57b Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sat, 30 Mar 2019 19:12:31 -0400 Subject: [PATCH 058/131] remove Message from dummy account --- tests/pubsub/dummy_account_node.py | 57 +++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index f328a6e..de78211 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -1,12 +1,12 @@ import asyncio +import uuid import multiaddr from libp2p import new_node -from libp2p.pubsub.message import create_message_talk +from libp2p.pubsub.pb import rpc_pb2_grpc +from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub -from libp2p.pubsub.message import MessageTalk -from libp2p.pubsub.message import generate_message_id SUPPORTED_PUBSUB_PROTOCOLS = ["/floodsub/1.0.0"] CRYPTO_TOPIC = "ethereum" @@ -53,16 +53,15 @@ class DummyAccountNode(): Handle all incoming messages on the CRYPTO_TOPIC from peers """ while True: - message_raw = await self.q.get() - message = create_message_talk(message_raw) - contents = message.data - - msg_comps = contents.split(",") - - if msg_comps[0] == "send": - self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) - elif msg_comps[0] == "set": - self.handle_set_crypto_for_user(msg_comps[1], int(msg_comps[2])) + incoming = await self.q.get() + rpc_incoming = rpc_pb2.RPC() + rpc_incoming.ParseFromString(incoming) + for message in rpc_incoming.publish: + msg_comps = message.split(",") + if msg_comps[0] == "send": + self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) + elif msg_comps[0] == "set": + self.handle_set_crypto_for_user(msg_comps[1], int(msg_comps[2])) async def setup_crypto_networking(self): """ @@ -82,8 +81,8 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "send," + source_user + "," + dest_user + "," + str(amount) - msg = MessageTalk(my_id, my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) - await self.floodsub.publish(my_id, msg.to_str()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + await self.floodsub.publish(my_id, msg.SerializeToString()) async def publish_set_crypto(self, user, amount): """ @@ -93,8 +92,9 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "set," + user + "," + str(amount) - msg = MessageTalk(my_id, my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) - await self.floodsub.publish(my_id, msg.to_str()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + + await self.floodsub.publish(my_id, packet.SerializeToString()) def handle_send_crypto(self, source_user, dest_user, amount): """ @@ -132,3 +132,26 @@ class DummyAccountNode(): else: return -1 +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) + +def generate_RPC_packet(origin_id, topics, msg_content, msg_id): + packet = rpc_pb2.RPC() + packet.publish.extend([rpc_pb2.Message( + from_id=origin_id.encode('utf-8'), + seqno=msg_id.encode('utf-8'), + data=msg_content.encode('utf-8'), + topicIDs=topics.encode('utf-8')) + ]) + + for topic in topics: + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe=True, + topicid = topic.encode('utf-8') + )]) + + return packet From 1cbd909dc20c2af2258de696ef02172387f6986f Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sat, 30 Mar 2019 19:30:58 -0400 Subject: [PATCH 059/131] fix encoding issue --- libp2p/pubsub/floodsub.py | 3 +++ tests/pubsub/dummy_account_node.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index dd4dfc1..8b9ba1c 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -57,6 +57,9 @@ class FloodSub(IPubsubRouter): """ packet = rpc_pb2.RPC() + print ("IN FLOOODSUB PUBLISH") + print (message) + print ("++++++++++++++++") packet.publish.extend([message]) msg_sender = str(sender_peer_id) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index de78211..930dc57 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -141,17 +141,21 @@ def generate_message_id(): def generate_RPC_packet(origin_id, topics, msg_content, msg_id): packet = rpc_pb2.RPC() - packet.publish.extend([rpc_pb2.Message( + message = rpc_pb2.Message( from_id=origin_id.encode('utf-8'), seqno=msg_id.encode('utf-8'), - data=msg_content.encode('utf-8'), - topicIDs=topics.encode('utf-8')) - ]) + data=msg_content.encode('utf-8') + ) for topic in topics: + message.topicIDs.extend([topic.encode('utf-8')]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe=True, topicid = topic.encode('utf-8') )]) + packet.publish.extend([message]) + print ("HEYHO") + print (packet) + return packet From 87269a95244b6ec5f554ca8c245581ee7074c12a Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 31 Mar 2019 22:16:28 -0400 Subject: [PATCH 060/131] reworked floodsub logic --- libp2p/pubsub/floodsub.py | 46 +++++++++++++--------- libp2p/pubsub/pubsub.py | 83 ++++++++++++++++++++++----------------- 2 files changed, 74 insertions(+), 55 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 8b9ba1c..d4736d0 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -41,7 +41,7 @@ class FloodSub(IPubsubRouter): :param rpc: rpc message """ - async def publish(self, sender_peer_id, message): + async def publish(self, sender_peer_id, rpc_message): """ Invoked to forward a new message that has been validated. This is where the "flooding" part of floodsub happens @@ -53,33 +53,41 @@ class FloodSub(IPubsubRouter): It also never forwards a message back to the source or the peer that forwarded the message. :param sender_peer_id: peer_id of message sender - :param message: message to forward + :param rpc_message: pubsub message in RPC string format """ packet = rpc_pb2.RPC() + packet.ParseFromString(rpc_message) print ("IN FLOOODSUB PUBLISH") - print (message) + print (packet) print ("++++++++++++++++") - packet.publish.extend([message]) msg_sender = str(sender_peer_id) - # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message - if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): - await self.pubsub.handle_talk(sender_peer_id, message) + for message in packet.publish: + if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): + await self.pubsub.handle_talk(sender_peer_id, message) - # Deliver to self and peers - for topic in message.topicIDs: - if topic in self.pubsub.peer_topics: - for peer_id_in_topic in self.pubsub.peer_topics[topic]: - # Forward to all known peers in the topic that are not the - # message sender and are not the message origin - if peer_id_in_topic not in (msg_sender, message.from_id): - stream = self.pubsub.peers[peer_id_in_topic] - await stream.write(packet.SerializeToString()) - else: - # Implies publish did not write - print("publish did not write") + + print ("OHOHOHOH") + print (self.pubsub.peer_topics) + print ("UUUJUJUJ") + print (self.pubsub.peers) + print ("********") + # Deliver to self and peers + for topic in message.topicIDs: + if topic in self.pubsub.peer_topics: + for peer_id_in_topic in self.pubsub.peer_topics[topic]: + # Forward to all known peers in the topic that are not the + # message sender and are not the message origin + print ("PEERID") + print (peer_id_in_topic) + if peer_id_in_topic not in (msg_sender, message.from_id): + stream = self.pubsub.peers[peer_id_in_topic] + await stream.write(packet.SerializeToString()) + else: + # Implies publish did not write + print("publish did not write") def join(self, topic): """ diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 976e6b4..7705dfb 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -80,31 +80,43 @@ class Pubsub(): """ # TODO check on types here - peer_id = stream.mplex_conn.peer_id + peer_id = str(stream.mplex_conn.peer_id) while True: incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() rpc_incoming.ParseFromString(incoming) - print ("CONTINUOUSLY") + + print ("IN PUBSUB CONTINUOUSLY READ") print (rpc_incoming) + print ("###########################") + + should_publish = True + if rpc_incoming.publish: - # deal with "talk messages" - for msg in rpc_incoming.publish: - self.seen_messages.append(msg.seqno) - await self.handle_talk(peer_id, msg) - await self.router.publish(peer_id, msg) + # deal with RPC.publish + for message in rpc_incoming.publish: + self.seen_messages.append(message.seqno) + await self.handle_talk(peer_id, message) + if rpc_incoming.subscriptions: - # deal with "subscription messages" - subs_map = {} - for msg in rpc_incoming.subscriptions: - if msg.subscribe: - subs_map[msg.topicid] = "sub" - else: - subs_map[msg.topicid] = "unsub" + # deal with RPC.subscriptions + # We don't need to relay the subscription to our + # peers because a given node only needs its peers + # to know that it is subscribed to the topic (doesn't + # need everyone to know) + should_publish = False - self.handle_subscription(rpc_incoming) + # TODO check that peer_id is the same as origin_id + from_id = str(rpc_incoming.publish[0].from_id.decode('utf-8')) + for message in rpc_incoming.subscriptions: + if message.subscribe: + self.handle_subscription(from_id, message) + + if should_publish: + # relay message to peers with router + await self.router.publish(peer_id, incoming) # Force context switch await asyncio.sleep(0) @@ -136,7 +148,10 @@ class Pubsub(): pubsub protocols we support """ while True: + print ("PUBSUB HANDLE PEER QUEUE") peer_id = await self.peer_queue.get() + print (peer_id) + print ("++++++++++++++++++++++++") # Open a stream to peer on existing connection # (we know connection exists since that's the only way @@ -158,34 +173,30 @@ class Pubsub(): # Force context switch await asyncio.sleep(0) - def handle_subscription(self, rpc_message): + def handle_subscription(self, peer_id, sub_message): """ Handle an incoming subscription message from a peer. Update internal mapping to mark the peer as subscribed or unsubscribed to topics as defined in the subscription message - :param subscription: raw data constituting a subscription message + :param origin_id: id of the peer who subscribe to the message + :param sub_message: RPC.SubOpts """ - for sub_msg in rpc_message.subscriptions: - - # Look at each subscription in the msg individually - if sub_msg.subscribe: - origin_id = rpc_message.publish[0].from_id - - if sub_msg.topicid not in self.peer_topics: - # Create topic list if it did not yet exist - self.peer_topics[sub_msg.topicid] = origin_id - elif orgin_id not in self.peer_topics[sub_msg.topicid]: - # Add peer to topic - self.peer_topics[sub_msg.topicid].append(origin_id) - else: - # TODO: Remove peer from topic - pass + # TODO verify logic here + if sub_message.subscribe: + if sub_message.topicid not in self.peer_topics: + self.peer_topics[sub_message.topicid] = [peer_id] + elif peer_id not in self.peer_topics[sub_message.topicid]: + # Add peer to topic + self.peer_topics[sub_message.topicid].append(peer_id) + else: + # TODO: Remove peer from topic + pass async def handle_talk(self, peer_id, publish_message): """ - Handle incoming Talk message from a peer. A Talk message contains some - custom message that is published on a given topic(s) - :param talk: raw data constituting a talk message + Put incoming message from a peer onto my blocking queue + :param peer_id: peer id whom forwarded this message + :param talk: RPC.Message format """ # Check if this message has any topics that we are subscribed to @@ -269,4 +280,4 @@ def generate_message_id(): Generate a unique message id :return: messgae id """ - return str(uuid.uuid1()) \ No newline at end of file + return str(uuid.uuid1()) From 974cb0b06fd9d5bf30909c10a979d7f43006ed97 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 31 Mar 2019 22:16:49 -0400 Subject: [PATCH 061/131] update dummy account node --- tests/pubsub/dummy_account_node.py | 32 +++++++++++++++++++----------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 930dc57..0773533 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -37,6 +37,7 @@ class DummyAccountNode(): We use create as this serves as a factory function and allows us to use async await, unlike the init function """ + print ("**DUMMY** CREATE ACCOUNT NODE") self = DummyAccountNode() libp2p_node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -54,14 +55,17 @@ class DummyAccountNode(): """ while True: incoming = await self.q.get() - rpc_incoming = rpc_pb2.RPC() - rpc_incoming.ParseFromString(incoming) - for message in rpc_incoming.publish: - msg_comps = message.split(",") - if msg_comps[0] == "send": - self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) - elif msg_comps[0] == "set": - self.handle_set_crypto_for_user(msg_comps[1], int(msg_comps[2])) + print ("**DUMMY** HANDLE INCOMING") + print (incoming) + print ("========================") + + msg_comps = incoming.data.decode('utf-8').split(",") + print (msg_comps) + print ("--------") + if msg_comps[0] == "send": + self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) + elif msg_comps[0] == "set": + self.handle_set_crypto(msg_comps[1], int(msg_comps[2])) async def setup_crypto_networking(self): """ @@ -103,6 +107,7 @@ class DummyAccountNode(): :param dest_user: user to send crypto to :param amount: amount of crypto to send """ + print ("**DUMMY** IN HANDLE SEND CRYPTO") if source_user in self.balances: self.balances[source_user] -= amount else: @@ -113,12 +118,15 @@ class DummyAccountNode(): else: self.balances[dest_user] = amount - def handle_set_crypto_for_user(self, dest_user, amount): + def handle_set_crypto(self, dest_user, amount): """ Handle incoming set_crypto message :param dest_user: user to set crypto for :param amount: amount of crypto """ + print ("**DUMMY** IN HANDLE SET CRYPTO") + print (type (dest_user)) + print (amount) self.balances[dest_user] = amount def get_balance(self, user): @@ -127,6 +135,9 @@ class DummyAccountNode(): :param user: user to get balance for :return: balance of user """ + print ("GET BALACNCE") + print (user) + print (self.balances) if user in self.balances: return self.balances[user] else: @@ -155,7 +166,4 @@ def generate_RPC_packet(origin_id, topics, msg_content, msg_id): )]) packet.publish.extend([message]) - print ("HEYHO") - print (packet) - return packet From f287c73236de572ce28b6517362aadb6437eaafa Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Mon, 1 Apr 2019 15:04:20 -0400 Subject: [PATCH 062/131] remove message from test dummy --- tests/pubsub/dummy_account_node.py | 2 +- tests/pubsub/test_dummyaccount_demo.py | 148 ++++++++++++------------- 2 files changed, 74 insertions(+), 76 deletions(-) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 0773533..eaca614 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -125,7 +125,7 @@ class DummyAccountNode(): :param amount: amount of crypto """ print ("**DUMMY** IN HANDLE SET CRYPTO") - print (type (dest_user)) + print (dest_user) print (amount) self.balances[dest_user] = amount diff --git a/tests/pubsub/test_dummyaccount_demo.py b/tests/pubsub/test_dummyaccount_demo.py index a071fa6..9b3a149 100644 --- a/tests/pubsub/test_dummyaccount_demo.py +++ b/tests/pubsub/test_dummyaccount_demo.py @@ -8,8 +8,6 @@ from libp2p import new_node from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub -from libp2p.pubsub.message import MessageTalk -from libp2p.pubsub.message import create_message_talk from dummy_account_node import DummyAccountNode # pylint: disable=too-many-locals @@ -66,7 +64,7 @@ async def perform_test(num_nodes, adjacency_map, action_func, assertion_func): await action_func(dummy_nodes) # Allow time for action function to be performed (i.e. messages to propogate) - await asyncio.sleep(0.25) + await asyncio.sleep(1) # Perform assertion function for dummy_node in dummy_nodes: @@ -88,102 +86,102 @@ async def test_simple_two_nodes(): await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_simple_three_nodes_line_topography(): - num_nodes = 3 - adj_map = {0: [1], 1: [2]} +# @pytest.mark.asyncio +# async def test_simple_three_nodes_line_topography(): +# num_nodes = 3 +# adj_map = {0: [1], 1: [2]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 10) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 10) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 10 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 10 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_simple_three_nodes_triangle_topography(): - num_nodes = 3 - adj_map = {0: [1, 2], 1: [2]} +# @pytest.mark.asyncio +# async def test_simple_three_nodes_triangle_topography(): +# num_nodes = 3 +# adj_map = {0: [1, 2], 1: [2]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 20) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 20) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 20 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 20 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_simple_seven_nodes_tree_topography(): - num_nodes = 7 - adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +# @pytest.mark.asyncio +# async def test_simple_seven_nodes_tree_topography(): +# num_nodes = 7 +# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 20) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 20) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 20 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 20 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_set_then_send_from_root_seven_nodes_tree_topography(): - num_nodes = 7 - adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +# @pytest.mark.asyncio +# async def test_set_then_send_from_root_seven_nodes_tree_topography(): +# num_nodes = 7 +# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 20) - await asyncio.sleep(0.25) - await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 20) +# await asyncio.sleep(0.25) +# await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 15 - assert dummy_node.get_balance("alex") == 5 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 15 +# assert dummy_node.get_balance("alex") == 5 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): - num_nodes = 7 - adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +# @pytest.mark.asyncio +# async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): +# num_nodes = 7 +# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} - async def action_func(dummy_nodes): - await dummy_nodes[6].publish_set_crypto("aspyn", 20) - await asyncio.sleep(0.25) - await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) +# async def action_func(dummy_nodes): +# await dummy_nodes[6].publish_set_crypto("aspyn", 20) +# await asyncio.sleep(0.25) +# await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 15 - assert dummy_node.get_balance("alex") == 5 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 15 +# assert dummy_node.get_balance("alex") == 5 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_simple_five_nodes_ring_topography(): - num_nodes = 5 - adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} +# @pytest.mark.asyncio +# async def test_simple_five_nodes_ring_topography(): +# num_nodes = 5 +# adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 20) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 20) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 20 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 20 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): - num_nodes = 5 - adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} +# @pytest.mark.asyncio +# async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): +# num_nodes = 5 +# adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("alex", 20) - await asyncio.sleep(0.25) - await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("alex", 20) +# await asyncio.sleep(0.25) +# await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) - def assertion_func(dummy_node): - assert dummy_node.get_balance("alex") == 8 - assert dummy_node.get_balance("rob") == 12 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("alex") == 8 +# assert dummy_node.get_balance("rob") == 12 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) From f50ba04178acdb16b8ce267d23554c0df54cf8ba Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Mon, 1 Apr 2019 16:23:20 -0400 Subject: [PATCH 063/131] fix all tests --- libp2p/pubsub/floodsub.py | 18 ++- libp2p/pubsub/pubsub.py | 62 ++++++----- tests/pubsub/dummy_account_node.py | 4 +- tests/pubsub/test_dummyaccount_demo.py | 144 ++++++++++++------------ tests/pubsub/test_floodsub.py | 147 ++++++++++++++++--------- 5 files changed, 221 insertions(+), 154 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index d4736d0..305607e 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -65,7 +65,16 @@ class FloodSub(IPubsubRouter): # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message for message in packet.publish: - if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): + decoded_from_id = message.from_id.decode('utf-8') + + print ("MESSAGE SENDER") + print (msg_sender) + print ("FROM ID") + print (message.from_id) + print (str(self.pubsub.host.get_id())) + + + if msg_sender == decoded_from_id and msg_sender == str(self.pubsub.host.get_id()): await self.pubsub.handle_talk(sender_peer_id, message) @@ -82,9 +91,12 @@ class FloodSub(IPubsubRouter): # message sender and are not the message origin print ("PEERID") print (peer_id_in_topic) - if peer_id_in_topic not in (msg_sender, message.from_id): + if peer_id_in_topic not in (msg_sender, decoded_from_id): stream = self.pubsub.peers[peer_id_in_topic] - await stream.write(packet.SerializeToString()) + # create new packet with just publish message + new_packet = rpc_pb2.RPC() + new_packet.publish.extend([message]) + await stream.write(new_packet.SerializeToString()) else: # Implies publish did not write print("publish did not write") diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 7705dfb..c7bfe06 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -58,17 +58,19 @@ class Pubsub(): def get_hello_packet(self): """ Generate subscription message with all topics we are subscribed to + only send hello packet if we have subscribed topics """ - packet = rpc_pb2.RPC() - message = rpc_pb2.Message( - from_id=str(self.host.get_id()).encode('utf-8'), - seqno=str(generate_message_id()).encode('utf-8') - ) - packet.publish.extend([message]) - for topic_id in self.my_topics: - packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - subscribe=True, topicid=topic_id)]) + if self.my_topics: + for topic_id in self.my_topics: + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe=True, topicid=topic_id)]) + + # message = rpc_pb2.Message( + # from_id=str(self.host.get_id()).encode('utf-8'), + # seqno=str(generate_message_id()).encode('utf-8') + # ) + # packet.publish.extend([message]) return packet.SerializeToString() @@ -80,9 +82,11 @@ class Pubsub(): """ # TODO check on types here + print ("++++++++++ASPYN+++++++++++++++++") peer_id = str(stream.mplex_conn.peer_id) while True: + print ("HIT ME") incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() rpc_incoming.ParseFromString(incoming) @@ -91,13 +95,15 @@ class Pubsub(): print (rpc_incoming) print ("###########################") - should_publish = True + should_publish = False if rpc_incoming.publish: # deal with RPC.publish for message in rpc_incoming.publish: - self.seen_messages.append(message.seqno) - await self.handle_talk(peer_id, message) + if message.seqno not in self.seen_messages: + should_publish = True + self.seen_messages.append(message.seqno) + await self.handle_talk(peer_id, message) if rpc_incoming.subscriptions: @@ -106,13 +112,9 @@ class Pubsub(): # peers because a given node only needs its peers # to know that it is subscribed to the topic (doesn't # need everyone to know) - should_publish = False - - # TODO check that peer_id is the same as origin_id - from_id = str(rpc_incoming.publish[0].from_id.decode('utf-8')) for message in rpc_incoming.subscriptions: if message.subscribe: - self.handle_subscription(from_id, message) + self.handle_subscription(peer_id, message) if should_publish: # relay message to peers with router @@ -182,6 +184,8 @@ class Pubsub(): :param sub_message: RPC.SubOpts """ # TODO verify logic here + + if sub_message.subscribe: if sub_message.topicid not in self.peer_topics: self.peer_topics[sub_message.topicid] = [peer_id] @@ -213,19 +217,23 @@ class Pubsub(): :param topic_id: topic_id to subscribe to """ # Map topic_id to blocking queue + + print ("**PUBSUB** in SUBSCRIBE") self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message packet = rpc_pb2.RPC() - packet.publish.extend([rpc_pb2.Message( - from_id=str(self.host.get_id()).encode('utf-8'), - seqno=str(generate_message_id()).encode('utf-8') - )]) + # packet.publish.extend([rpc_pb2.Message( + # from_id=str(self.host.get_id()).encode('utf-8'), + # seqno=str(generate_message_id()).encode('utf-8') + # )]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe = True, topicid = topic_id.encode('utf-8') )]) - + print (packet) + print ("**PUBSUB** PEEERS") + print (self.peers) # Send out subscribe message to all peers await self.message_all_peers(packet.SerializeToString()) @@ -247,10 +255,10 @@ class Pubsub(): # Create unsubscribe message packet = rpc_pb2.RPC() - packet.publish.extend([rpc_pb2.Message( - from_id=str(self.host.get_id()).encode('utf-8'), - seqno=str(generate_message_id()).encode('utf-8') - )]) + # packet.publish.extend([rpc_pb2.Message( + # from_id=str(self.host.get_id()).encode('utf-8'), + # seqno=str(generate_message_id()).encode('utf-8') + # )]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe = False, topicid = topic_id.encode('utf-8') @@ -267,6 +275,8 @@ class Pubsub(): Broadcast a message to peers :param raw_msg: raw contents of the message to broadcast """ + print ("**PUBSU** IN MESSAGE ALL PEERS") + print (rpc_msg) # Broadcast message for peer in self.peers: diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index eaca614..1951fc6 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -86,7 +86,7 @@ class DummyAccountNode(): my_id = str(self.libp2p_node.get_id()) msg_contents = "send," + source_user + "," + dest_user + "," + str(amount) packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) - await self.floodsub.publish(my_id, msg.SerializeToString()) + await self.floodsub.publish(my_id, packet.SerializeToString()) async def publish_set_crypto(self, user, amount): """ @@ -128,6 +128,8 @@ class DummyAccountNode(): print (dest_user) print (amount) self.balances[dest_user] = amount + print (self.balances) + print ("^^ balance") def get_balance(self, user): """ diff --git a/tests/pubsub/test_dummyaccount_demo.py b/tests/pubsub/test_dummyaccount_demo.py index 9b3a149..1f08c8d 100644 --- a/tests/pubsub/test_dummyaccount_demo.py +++ b/tests/pubsub/test_dummyaccount_demo.py @@ -86,102 +86,102 @@ async def test_simple_two_nodes(): await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_simple_three_nodes_line_topography(): -# num_nodes = 3 -# adj_map = {0: [1], 1: [2]} +@pytest.mark.asyncio +async def test_simple_three_nodes_line_topography(): + num_nodes = 3 + adj_map = {0: [1], 1: [2]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 10) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 10) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 10 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 10 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_simple_three_nodes_triangle_topography(): -# num_nodes = 3 -# adj_map = {0: [1, 2], 1: [2]} +@pytest.mark.asyncio +async def test_simple_three_nodes_triangle_topography(): + num_nodes = 3 + adj_map = {0: [1, 2], 1: [2]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 20) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 20 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_simple_seven_nodes_tree_topography(): -# num_nodes = 7 -# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +@pytest.mark.asyncio +async def test_simple_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 20) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 20 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_set_then_send_from_root_seven_nodes_tree_topography(): -# num_nodes = 7 -# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +@pytest.mark.asyncio +async def test_set_then_send_from_root_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 20) -# await asyncio.sleep(0.25) -# await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) + await asyncio.sleep(0.25) + await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 15 -# assert dummy_node.get_balance("alex") == 5 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 15 + assert dummy_node.get_balance("alex") == 5 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): -# num_nodes = 7 -# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +@pytest.mark.asyncio +async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} -# async def action_func(dummy_nodes): -# await dummy_nodes[6].publish_set_crypto("aspyn", 20) -# await asyncio.sleep(0.25) -# await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) + async def action_func(dummy_nodes): + await dummy_nodes[6].publish_set_crypto("aspyn", 20) + await asyncio.sleep(0.25) + await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 15 -# assert dummy_node.get_balance("alex") == 5 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 15 + assert dummy_node.get_balance("alex") == 5 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_simple_five_nodes_ring_topography(): -# num_nodes = 5 -# adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} +@pytest.mark.asyncio +async def test_simple_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 20) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 20 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): -# num_nodes = 5 -# adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} +@pytest.mark.asyncio +async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("alex", 20) -# await asyncio.sleep(0.25) -# await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("alex", 20) + await asyncio.sleep(0.25) + await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("alex") == 8 -# assert dummy_node.get_balance("rob") == 12 + def assertion_func(dummy_node): + assert dummy_node.get_balance("alex") == 8 + assert dummy_node.get_balance("rob") == 12 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index f4a5826..c4c2681 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -1,15 +1,14 @@ import asyncio +import uuid import multiaddr import pytest from tests.utils import cleanup from libp2p import new_node from libp2p.peer.peerinfo import info_from_p2p_addr +from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub -from libp2p.pubsub.message import MessageTalk -from libp2p.pubsub.message import create_message_talk -from libp2p.pubsub.message import generate_message_id # pylint: disable=too-many-locals @@ -22,7 +21,7 @@ async def connect(node1, node2): await node1.connect(info) @pytest.mark.asyncio -async def test_simple_two_nodes(): +async def test_simple_two_nodes_RPC(): node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -45,71 +44,77 @@ async def test_simple_two_nodes(): node_a_id = str(node_a.get_id()) - msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) - - await floodsub_a.publish(node_a.get_id(), msg.to_str()) - + msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", generate_message_id()) + await floodsub_a.publish(node_a_id, msg.SerializeToString()) + print ("MESSAGE B") + print (msg.SerializeToString()) + print ("=========") await asyncio.sleep(0.25) res_b = await qb.get() + print ("RES B") + print (res_b) + print ("-----") # Check that the msg received by node_b is the same # as the message sent by node_a - assert res_b == msg.to_str() + assert res_b.SerializeToString() == msg.publish[0].SerializeToString() + + # Success, terminate pending tasks. await cleanup() -@pytest.mark.asyncio -async def test_simple_three_nodes(): - # Want to pass message from A -> B -> C - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_c = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) +# @pytest.mark.asyncio +# async def test_simple_three_nodes(): +# # Want to pass message from A -> B -> C +# node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) +# node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) +# node_c = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) - await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) - await node_c.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) +# await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) +# await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) +# await node_c.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) - supported_protocols = ["/floodsub/1.0.0"] +# supported_protocols = ["/floodsub/1.0.0"] - floodsub_a = FloodSub(supported_protocols) - pubsub_a = Pubsub(node_a, floodsub_a, "a") - floodsub_b = FloodSub(supported_protocols) - pubsub_b = Pubsub(node_b, floodsub_b, "b") - floodsub_c = FloodSub(supported_protocols) - pubsub_c = Pubsub(node_c, floodsub_c, "c") +# floodsub_a = FloodSub(supported_protocols) +# pubsub_a = Pubsub(node_a, floodsub_a, "a") +# floodsub_b = FloodSub(supported_protocols) +# pubsub_b = Pubsub(node_b, floodsub_b, "b") +# floodsub_c = FloodSub(supported_protocols) +# pubsub_c = Pubsub(node_c, floodsub_c, "c") - await connect(node_a, node_b) - await connect(node_b, node_c) +# await connect(node_a, node_b) +# await connect(node_b, node_c) - await asyncio.sleep(0.25) - qb = await pubsub_b.subscribe("my_topic") - qc = await pubsub_c.subscribe("my_topic") - await asyncio.sleep(0.25) +# await asyncio.sleep(0.25) +# qb = await pubsub_b.subscribe("my_topic") +# qc = await pubsub_c.subscribe("my_topic") +# await asyncio.sleep(0.25) - node_a_id = str(node_a.get_id()) +# node_a_id = str(node_a.get_id()) - msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) +# msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) - await floodsub_a.publish(node_a.get_id(), msg.to_str()) +# await floodsub_a.publish(node_a.get_id(), msg.to_str()) - await asyncio.sleep(0.25) - res_b = await qb.get() - res_c = await qc.get() +# await asyncio.sleep(0.25) +# res_b = await qb.get() +# res_c = await qc.get() - # Check that the msg received by node_b is the same - # as the message sent by node_a - assert res_b == msg.to_str() +# # Check that the msg received by node_b is the same +# # as the message sent by node_a +# assert res_b == msg.to_str() - # res_c should match original msg but with b as sender - node_b_id = str(node_b.get_id()) - msg.from_id = node_b_id +# # res_c should match original msg but with b as sender +# node_b_id = str(node_b.get_id()) +# msg.from_id = node_b_id - assert res_c == msg.to_str() +# assert res_c == msg.to_str() - # Success, terminate pending tasks. - await cleanup() +# # Success, terminate pending tasks. +# await cleanup() async def perform_test_from_obj(obj): """ @@ -237,11 +242,14 @@ async def perform_test_from_obj(obj): actual_node_id = str(node_map[node_id].get_id()) # Create correctly formatted message - msg_talk = MessageTalk(actual_node_id, actual_node_id, topics, data, generate_message_id()) + msg_talk = generate_RPC_packet(actual_node_id, topics, data, generate_message_id()) + print ("**TEST FLOODSUB** MESSAGE TALK") + print (msg_talk) # Publish message # await floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()) - tasks_publish.append(asyncio.ensure_future(floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()))) + tasks_publish.append(asyncio.ensure_future(floodsub_map[node_id].publish(\ + actual_node_id, msg_talk.SerializeToString()))) # For each topic in topics, add topic, msg_talk tuple to ordered test list # TODO: Update message sender to be correct message sender before @@ -261,12 +269,20 @@ async def perform_test_from_obj(obj): for node_id in topic_map[topic]: # Get message from subscription queue msg_on_node_str = await queues_map[node_id][topic].get() - msg_on_node = create_message_talk(msg_on_node_str) + + print ("MESSAGE ON NODE STR") + print (msg_on_node_str) + + print ("ACTUAL MESSSSAGE") + print (actual_msg) + + assert actual_msg.publish[0].SerializeToString() == msg_on_node_str.SerializeToString() + # msg_on_node = create_message_talk(msg_on_node_str) # Perform checks - assert actual_msg.origin_id == msg_on_node.origin_id - assert actual_msg.topics == msg_on_node.topics - assert actual_msg.data == msg_on_node.data + # assert actual_msg.origin_id == msg_on_node.origin_id + # assert actual_msg.topics == msg_on_node.topics + # assert actual_msg.data == msg_on_node.data # Success, terminate pending tasks. await cleanup() @@ -484,3 +500,30 @@ async def test_three_nodes_clique_two_topic_diff_origin_test_obj(): ] } await perform_test_from_obj(test_obj) + +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) + +def generate_RPC_packet(origin_id, topics, msg_content, msg_id): + packet = rpc_pb2.RPC() + message = rpc_pb2.Message( + from_id=origin_id.encode('utf-8'), + seqno=msg_id.encode('utf-8'), + data=msg_content.encode('utf-8'), + ) + for topic in topics: + message.topicIDs.extend([topic.encode('utf-8')]) + + # for topic in topics: + # message.topicIDs.extend([topic.encode('utf-8')]) + # packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + # subscribe=True, + # topicid = topic.encode('utf-8') + # )]) + + packet.publish.extend([message]) + return packet From 0de028a236f81754f6568ebe7f535b9ba2ab7b9c Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Mon, 1 Apr 2019 16:55:44 -0400 Subject: [PATCH 064/131] clean up --- libp2p/pubsub/floodsub.py | 25 +----- libp2p/pubsub/pubsub.py | 57 +++---------- libp2p/pubsub/pubsub_router_interface.py | 4 +- tests/pubsub/dummy_account_node.py | 47 +---------- tests/pubsub/test_floodsub.py | 103 +---------------------- tests/pubsub/utils.py | 30 +++++++ 6 files changed, 54 insertions(+), 212 deletions(-) create mode 100644 tests/pubsub/utils.py diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 305607e..a4f2e86 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -1,8 +1,9 @@ -from .pubsub_router_interface import IPubsubRouter from .pb import rpc_pb2 +from .pubsub_router_interface import IPubsubRouter class FloodSub(IPubsubRouter): + # pylint: disable=no-member def __init__(self, protocols): self.protocols = protocols @@ -55,42 +56,22 @@ class FloodSub(IPubsubRouter): :param sender_peer_id: peer_id of message sender :param rpc_message: pubsub message in RPC string format """ - packet = rpc_pb2.RPC() packet.ParseFromString(rpc_message) - print ("IN FLOOODSUB PUBLISH") - print (packet) - print ("++++++++++++++++") msg_sender = str(sender_peer_id) # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message for message in packet.publish: decoded_from_id = message.from_id.decode('utf-8') - - print ("MESSAGE SENDER") - print (msg_sender) - print ("FROM ID") - print (message.from_id) - print (str(self.pubsub.host.get_id())) - - if msg_sender == decoded_from_id and msg_sender == str(self.pubsub.host.get_id()): - await self.pubsub.handle_talk(sender_peer_id, message) + await self.pubsub.handle_talk(message) - - print ("OHOHOHOH") - print (self.pubsub.peer_topics) - print ("UUUJUJUJ") - print (self.pubsub.peers) - print ("********") # Deliver to self and peers for topic in message.topicIDs: if topic in self.pubsub.peer_topics: for peer_id_in_topic in self.pubsub.peer_topics[topic]: # Forward to all known peers in the topic that are not the # message sender and are not the message origin - print ("PEERID") - print (peer_id_in_topic) if peer_id_in_topic not in (msg_sender, decoded_from_id): stream = self.pubsub.peers[peer_id_in_topic] # create new packet with just publish message diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index c7bfe06..bfad873 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -1,13 +1,12 @@ import asyncio import uuid -from .pb import rpc_pb2_grpc from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee class Pubsub(): - # pylint: disable=too-many-instance-attributes + # pylint: disable=too-many-instance-attributes, no-member def __init__(self, host, router, my_id): """ @@ -65,12 +64,6 @@ class Pubsub(): for topic_id in self.my_topics: packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe=True, topicid=topic_id)]) - - # message = rpc_pb2.Message( - # from_id=str(self.host.get_id()).encode('utf-8'), - # seqno=str(generate_message_id()).encode('utf-8') - # ) - # packet.publish.extend([message]) return packet.SerializeToString() @@ -82,19 +75,13 @@ class Pubsub(): """ # TODO check on types here - print ("++++++++++ASPYN+++++++++++++++++") peer_id = str(stream.mplex_conn.peer_id) while True: - print ("HIT ME") incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() rpc_incoming.ParseFromString(incoming) - print ("IN PUBSUB CONTINUOUSLY READ") - print (rpc_incoming) - print ("###########################") - should_publish = False if rpc_incoming.publish: @@ -103,8 +90,7 @@ class Pubsub(): if message.seqno not in self.seen_messages: should_publish = True self.seen_messages.append(message.seqno) - await self.handle_talk(peer_id, message) - + await self.handle_talk(message) if rpc_incoming.subscriptions: # deal with RPC.subscriptions @@ -150,10 +136,8 @@ class Pubsub(): pubsub protocols we support """ while True: - print ("PUBSUB HANDLE PEER QUEUE") + peer_id = await self.peer_queue.get() - print (peer_id) - print ("++++++++++++++++++++++++") # Open a stream to peer on existing connection # (we know connection exists since that's the only way @@ -175,7 +159,7 @@ class Pubsub(): # Force context switch await asyncio.sleep(0) - def handle_subscription(self, peer_id, sub_message): + def handle_subscription(self, origin_id, sub_message): """ Handle an incoming subscription message from a peer. Update internal mapping to mark the peer as subscribed or unsubscribed to topics as @@ -183,23 +167,19 @@ class Pubsub(): :param origin_id: id of the peer who subscribe to the message :param sub_message: RPC.SubOpts """ - # TODO verify logic here - - if sub_message.subscribe: if sub_message.topicid not in self.peer_topics: - self.peer_topics[sub_message.topicid] = [peer_id] - elif peer_id not in self.peer_topics[sub_message.topicid]: + self.peer_topics[sub_message.topicid] = [origin_id] + elif origin_id not in self.peer_topics[sub_message.topicid]: # Add peer to topic - self.peer_topics[sub_message.topicid].append(peer_id) + self.peer_topics[sub_message.topicid].append(origin_id) else: # TODO: Remove peer from topic pass - async def handle_talk(self, peer_id, publish_message): + async def handle_talk(self, publish_message): """ Put incoming message from a peer onto my blocking queue - :param peer_id: peer id whom forwarded this message :param talk: RPC.Message format """ @@ -216,9 +196,8 @@ class Pubsub(): Subscribe ourself to a topic :param topic_id: topic_id to subscribe to """ - # Map topic_id to blocking queue - print ("**PUBSUB** in SUBSCRIBE") + # Map topic_id to blocking queue self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message @@ -228,12 +207,10 @@ class Pubsub(): # seqno=str(generate_message_id()).encode('utf-8') # )]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - subscribe = True, - topicid = topic_id.encode('utf-8') + subscribe=True, + topicid=topic_id.encode('utf-8') )]) - print (packet) - print ("**PUBSUB** PEEERS") - print (self.peers) + # Send out subscribe message to all peers await self.message_all_peers(packet.SerializeToString()) @@ -255,13 +232,9 @@ class Pubsub(): # Create unsubscribe message packet = rpc_pb2.RPC() - # packet.publish.extend([rpc_pb2.Message( - # from_id=str(self.host.get_id()).encode('utf-8'), - # seqno=str(generate_message_id()).encode('utf-8') - # )]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - subscribe = False, - topicid = topic_id.encode('utf-8') + subscribe=False, + topicid=topic_id.encode('utf-8') )]) # Send out unsubscribe message to all peers @@ -275,8 +248,6 @@ class Pubsub(): Broadcast a message to peers :param raw_msg: raw contents of the message to broadcast """ - print ("**PUBSU** IN MESSAGE ALL PEERS") - print (rpc_msg) # Broadcast message for peer in self.peers: diff --git a/libp2p/pubsub/pubsub_router_interface.py b/libp2p/pubsub/pubsub_router_interface.py index 727b39e..ec5132e 100644 --- a/libp2p/pubsub/pubsub_router_interface.py +++ b/libp2p/pubsub/pubsub_router_interface.py @@ -39,11 +39,11 @@ class IPubsubRouter(ABC): """ @abstractmethod - def publish(self, sender_peer_id, message): + def publish(self, sender_peer_id, rpc_message): """ Invoked to forward a new message that has been validated :param sender_peer_id: peer_id of message sender - :param message: message to forward + :param rpc_message: message to forward """ @abstractmethod diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 1951fc6..05a02bf 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -1,10 +1,8 @@ import asyncio -import uuid import multiaddr +from utils import generate_message_id, generate_RPC_packet from libp2p import new_node -from libp2p.pubsub.pb import rpc_pb2_grpc -from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub @@ -37,7 +35,6 @@ class DummyAccountNode(): We use create as this serves as a factory function and allows us to use async await, unlike the init function """ - print ("**DUMMY** CREATE ACCOUNT NODE") self = DummyAccountNode() libp2p_node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -54,14 +51,9 @@ class DummyAccountNode(): Handle all incoming messages on the CRYPTO_TOPIC from peers """ while True: - incoming = await self.q.get() - print ("**DUMMY** HANDLE INCOMING") - print (incoming) - print ("========================") - + incoming = await self.q.get() msg_comps = incoming.data.decode('utf-8').split(",") - print (msg_comps) - print ("--------") + if msg_comps[0] == "send": self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) elif msg_comps[0] == "set": @@ -107,7 +99,6 @@ class DummyAccountNode(): :param dest_user: user to send crypto to :param amount: amount of crypto to send """ - print ("**DUMMY** IN HANDLE SEND CRYPTO") if source_user in self.balances: self.balances[source_user] -= amount else: @@ -124,12 +115,7 @@ class DummyAccountNode(): :param dest_user: user to set crypto for :param amount: amount of crypto """ - print ("**DUMMY** IN HANDLE SET CRYPTO") - print (dest_user) - print (amount) self.balances[dest_user] = amount - print (self.balances) - print ("^^ balance") def get_balance(self, user): """ @@ -137,35 +123,8 @@ class DummyAccountNode(): :param user: user to get balance for :return: balance of user """ - print ("GET BALACNCE") - print (user) - print (self.balances) if user in self.balances: return self.balances[user] else: return -1 -def generate_message_id(): - """ - Generate a unique message id - :return: messgae id - """ - return str(uuid.uuid1()) - -def generate_RPC_packet(origin_id, topics, msg_content, msg_id): - packet = rpc_pb2.RPC() - message = rpc_pb2.Message( - from_id=origin_id.encode('utf-8'), - seqno=msg_id.encode('utf-8'), - data=msg_content.encode('utf-8') - ) - - for topic in topics: - message.topicIDs.extend([topic.encode('utf-8')]) - packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - subscribe=True, - topicid = topic.encode('utf-8') - )]) - - packet.publish.extend([message]) - return packet diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index c4c2681..4fc059b 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -1,5 +1,4 @@ import asyncio -import uuid import multiaddr import pytest @@ -9,6 +8,7 @@ from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub +from utils import generate_message_id, generate_RPC_packet # pylint: disable=too-many-locals @@ -46,16 +46,10 @@ async def test_simple_two_nodes_RPC(): msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", generate_message_id()) await floodsub_a.publish(node_a_id, msg.SerializeToString()) - print ("MESSAGE B") - print (msg.SerializeToString()) - print ("=========") await asyncio.sleep(0.25) res_b = await qb.get() - print ("RES B") - print (res_b) - print ("-----") # Check that the msg received by node_b is the same # as the message sent by node_a assert res_b.SerializeToString() == msg.publish[0].SerializeToString() @@ -65,57 +59,6 @@ async def test_simple_two_nodes_RPC(): # Success, terminate pending tasks. await cleanup() -# @pytest.mark.asyncio -# async def test_simple_three_nodes(): -# # Want to pass message from A -> B -> C -# node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) -# node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) -# node_c = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - -# await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) -# await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) -# await node_c.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) - -# supported_protocols = ["/floodsub/1.0.0"] - -# floodsub_a = FloodSub(supported_protocols) -# pubsub_a = Pubsub(node_a, floodsub_a, "a") -# floodsub_b = FloodSub(supported_protocols) -# pubsub_b = Pubsub(node_b, floodsub_b, "b") -# floodsub_c = FloodSub(supported_protocols) -# pubsub_c = Pubsub(node_c, floodsub_c, "c") - -# await connect(node_a, node_b) -# await connect(node_b, node_c) - -# await asyncio.sleep(0.25) -# qb = await pubsub_b.subscribe("my_topic") -# qc = await pubsub_c.subscribe("my_topic") -# await asyncio.sleep(0.25) - -# node_a_id = str(node_a.get_id()) - -# msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) - -# await floodsub_a.publish(node_a.get_id(), msg.to_str()) - -# await asyncio.sleep(0.25) -# res_b = await qb.get() -# res_c = await qc.get() - -# # Check that the msg received by node_b is the same -# # as the message sent by node_a -# assert res_b == msg.to_str() - -# # res_c should match original msg but with b as sender -# node_b_id = str(node_b.get_id()) -# msg.from_id = node_b_id - -# assert res_c == msg.to_str() - -# # Success, terminate pending tasks. -# await cleanup() - async def perform_test_from_obj(obj): """ Perform a floodsub test from a test obj. @@ -243,9 +186,7 @@ async def perform_test_from_obj(obj): # Create correctly formatted message msg_talk = generate_RPC_packet(actual_node_id, topics, data, generate_message_id()) - - print ("**TEST FLOODSUB** MESSAGE TALK") - print (msg_talk) + # Publish message # await floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()) tasks_publish.append(asyncio.ensure_future(floodsub_map[node_id].publish(\ @@ -269,20 +210,7 @@ async def perform_test_from_obj(obj): for node_id in topic_map[topic]: # Get message from subscription queue msg_on_node_str = await queues_map[node_id][topic].get() - - print ("MESSAGE ON NODE STR") - print (msg_on_node_str) - - print ("ACTUAL MESSSSAGE") - print (actual_msg) - assert actual_msg.publish[0].SerializeToString() == msg_on_node_str.SerializeToString() - # msg_on_node = create_message_talk(msg_on_node_str) - - # Perform checks - # assert actual_msg.origin_id == msg_on_node.origin_id - # assert actual_msg.topics == msg_on_node.topics - # assert actual_msg.data == msg_on_node.data # Success, terminate pending tasks. await cleanup() @@ -500,30 +428,3 @@ async def test_three_nodes_clique_two_topic_diff_origin_test_obj(): ] } await perform_test_from_obj(test_obj) - -def generate_message_id(): - """ - Generate a unique message id - :return: messgae id - """ - return str(uuid.uuid1()) - -def generate_RPC_packet(origin_id, topics, msg_content, msg_id): - packet = rpc_pb2.RPC() - message = rpc_pb2.Message( - from_id=origin_id.encode('utf-8'), - seqno=msg_id.encode('utf-8'), - data=msg_content.encode('utf-8'), - ) - for topic in topics: - message.topicIDs.extend([topic.encode('utf-8')]) - - # for topic in topics: - # message.topicIDs.extend([topic.encode('utf-8')]) - # packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - # subscribe=True, - # topicid = topic.encode('utf-8') - # )]) - - packet.publish.extend([message]) - return packet diff --git a/tests/pubsub/utils.py b/tests/pubsub/utils.py new file mode 100644 index 0000000..d4695d7 --- /dev/null +++ b/tests/pubsub/utils.py @@ -0,0 +1,30 @@ +import uuid +from libp2p.pubsub.pb import rpc_pb2 + +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) + +def generate_RPC_packet(origin_id, topics, msg_content, msg_id): + """ + Generate RPC packet to send over wire + :param origin_id: peer id of the message origin + :param topics: list of topics + :param msg_content: string of content in data + :param msg_id: seqno for the message + """ + packet = rpc_pb2.RPC() + message = rpc_pb2.Message( + from_id=origin_id.encode('utf-8'), + seqno=msg_id.encode('utf-8'), + data=msg_content.encode('utf-8'), + ) + + for topic in topics: + message.topicIDs.extend([topic.encode('utf-8')]) + + packet.publish.extend([message]) + return packet From 56fbdf3f2fb5ec057b02331f5ed5d0a413cd2cb1 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Tue, 2 Apr 2019 22:05:14 -0400 Subject: [PATCH 065/131] Add priority queues to handle seqno --- libp2p/pubsub/pubsub.py | 14 ++++++++------ libp2p/pubsub/read_only_queue.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 libp2p/pubsub/read_only_queue.py diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index bfad873..37b8714 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -3,6 +3,7 @@ import uuid from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee +from .read_only_queue import ReadOnlyQueue class Pubsub(): @@ -188,8 +189,9 @@ class Pubsub(): if topic in self.my_topics: # we are subscribed to a topic this message was sent for, # so add message to the subscription output queue - # for each topic - await self.my_topics[topic].put(publish_message) + # for each topic with priority being the message's seqno. + # Note: asyncio.PriorityQueue item format is (priority, data) + await self.my_topics[topic].put((publish_message.seqno, publish_message)) async def subscribe(self, topic_id): """ @@ -197,8 +199,8 @@ class Pubsub(): :param topic_id: topic_id to subscribe to """ - # Map topic_id to blocking queue - self.my_topics[topic_id] = asyncio.Queue() + # Map topic_id to a priority blocking queue + self.my_topics[topic_id] = asyncio.PriorityQueue() # Create subscribe message packet = rpc_pb2.RPC() @@ -217,8 +219,8 @@ class Pubsub(): # Tell router we are joining this topic self.router.join(topic_id) - # Return the asyncio queue for messages on this topic - return self.my_topics[topic_id] + # Return the readonly queue for messages on this topic + return ReadOnlyQueue(self.my_topics[topic_id]) async def unsubscribe(self, topic_id): """ diff --git a/libp2p/pubsub/read_only_queue.py b/libp2p/pubsub/read_only_queue.py new file mode 100644 index 0000000..1656768 --- /dev/null +++ b/libp2p/pubsub/read_only_queue.py @@ -0,0 +1,14 @@ +import asyncio + +class ReadOnlyQueue(): + + def __init__(self, queue): + self.queue = queue + + async def get(self): + """ + Get the next item from queue, which has items in the format (priority, data) + :return: next item from the queue + """ + return (await self.queue.get())[1] + From 2934f93fc9bdd1aa1d4b94e06f4d35da2f68984a Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Tue, 2 Apr 2019 22:05:32 -0400 Subject: [PATCH 066/131] Adjust floodsub tests for new seqno util --- tests/pubsub/test_floodsub.py | 11 ++++++++--- tests/pubsub/utils.py | 22 ++++++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index 4fc059b..ba9d33c 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -8,7 +8,7 @@ from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub -from utils import generate_message_id, generate_RPC_packet +from utils import message_id_generator, generate_RPC_packet # pylint: disable=too-many-locals @@ -44,7 +44,8 @@ async def test_simple_two_nodes_RPC(): node_a_id = str(node_a.get_id()) - msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", generate_message_id()) + next_msg_id_func = message_id_generator(0) + msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", next_msg_id_func()) await floodsub_a.publish(node_a_id, msg.SerializeToString()) await asyncio.sleep(0.25) @@ -175,6 +176,8 @@ async def perform_test_from_obj(obj): topics_in_msgs_ordered = [] messages = obj["messages"] tasks_publish = [] + next_msg_id_func = message_id_generator(0) + for msg in messages: topics = msg["topics"] @@ -185,7 +188,7 @@ async def perform_test_from_obj(obj): actual_node_id = str(node_map[node_id].get_id()) # Create correctly formatted message - msg_talk = generate_RPC_packet(actual_node_id, topics, data, generate_message_id()) + msg_talk = generate_RPC_packet(actual_node_id, topics, data, next_msg_id_func()) # Publish message # await floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()) @@ -207,6 +210,8 @@ async def perform_test_from_obj(obj): # TODO: Check message sender too for i in range(len(topics_in_msgs_ordered)): topic, actual_msg = topics_in_msgs_ordered[i] + + # Look at each node in each topic for node_id in topic_map[topic]: # Get message from subscription queue msg_on_node_str = await queues_map[node_id][topic].get() diff --git a/tests/pubsub/utils.py b/tests/pubsub/utils.py index d4695d7..8aa3f88 100644 --- a/tests/pubsub/utils.py +++ b/tests/pubsub/utils.py @@ -1,12 +1,26 @@ import uuid +import struct from libp2p.pubsub.pb import rpc_pb2 -def generate_message_id(): + +def message_id_generator(start_val): """ Generate a unique message id - :return: messgae id + :param start_val: value to start generating messages at + :return: message id """ - return str(uuid.uuid1()) + val = start_val + def generator(): + # Allow manipulation of val within closure + nonlocal val + + # Increment id + val += 1 + + # Convert val to big endian + return struct.pack('>I', val) + + return generator def generate_RPC_packet(origin_id, topics, msg_content, msg_id): """ @@ -19,7 +33,7 @@ def generate_RPC_packet(origin_id, topics, msg_content, msg_id): packet = rpc_pb2.RPC() message = rpc_pb2.Message( from_id=origin_id.encode('utf-8'), - seqno=msg_id.encode('utf-8'), + seqno=msg_id, data=msg_content.encode('utf-8'), ) From 547e7b3546b95adc53c0211605a08723d507ed45 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Tue, 2 Apr 2019 22:34:01 -0400 Subject: [PATCH 067/131] Modify pubsub to have seen message check incorporate seqno and node id --- libp2p/pubsub/pubsub.py | 16 +++++++--------- libp2p/pubsub/read_only_queue.py | 14 -------------- tests/pubsub/dummy_account_node.py | 13 +++++++++---- 3 files changed, 16 insertions(+), 27 deletions(-) delete mode 100644 libp2p/pubsub/read_only_queue.py diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 37b8714..d5d4b52 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -3,7 +3,6 @@ import uuid from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee -from .read_only_queue import ReadOnlyQueue class Pubsub(): @@ -90,7 +89,7 @@ class Pubsub(): for message in rpc_incoming.publish: if message.seqno not in self.seen_messages: should_publish = True - self.seen_messages.append(message.seqno) + self.seen_messages.append((message.seqno, message.from_id)) await self.handle_talk(message) if rpc_incoming.subscriptions: @@ -189,9 +188,8 @@ class Pubsub(): if topic in self.my_topics: # we are subscribed to a topic this message was sent for, # so add message to the subscription output queue - # for each topic with priority being the message's seqno. - # Note: asyncio.PriorityQueue item format is (priority, data) - await self.my_topics[topic].put((publish_message.seqno, publish_message)) + # for each topic + await self.my_topics[topic].put(publish_message) async def subscribe(self, topic_id): """ @@ -199,8 +197,8 @@ class Pubsub(): :param topic_id: topic_id to subscribe to """ - # Map topic_id to a priority blocking queue - self.my_topics[topic_id] = asyncio.PriorityQueue() + # Map topic_id to blocking queue + self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message packet = rpc_pb2.RPC() @@ -219,8 +217,8 @@ class Pubsub(): # Tell router we are joining this topic self.router.join(topic_id) - # Return the readonly queue for messages on this topic - return ReadOnlyQueue(self.my_topics[topic_id]) + # Return the asyncio queue for messages on this topic + return self.my_topics[topic_id] async def unsubscribe(self, topic_id): """ diff --git a/libp2p/pubsub/read_only_queue.py b/libp2p/pubsub/read_only_queue.py deleted file mode 100644 index 1656768..0000000 --- a/libp2p/pubsub/read_only_queue.py +++ /dev/null @@ -1,14 +0,0 @@ -import asyncio - -class ReadOnlyQueue(): - - def __init__(self, queue): - self.queue = queue - - async def get(self): - """ - Get the next item from queue, which has items in the format (priority, data) - :return: next item from the queue - """ - return (await self.queue.get())[1] - diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 05a02bf..8fb7310 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -1,7 +1,8 @@ import asyncio import multiaddr +import uuid -from utils import generate_message_id, generate_RPC_packet +from utils import message_id_generator, generate_RPC_packet from libp2p import new_node from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub @@ -25,6 +26,8 @@ class DummyAccountNode(): def __init__(self): self.balances = {} + self.next_msg_id_func = message_id_generator(0) + self.node_id = str(uuid.uuid1()) @classmethod async def create(cls): @@ -51,7 +54,7 @@ class DummyAccountNode(): Handle all incoming messages on the CRYPTO_TOPIC from peers """ while True: - incoming = await self.q.get() + incoming = await self.q.get() msg_comps = incoming.data.decode('utf-8').split(",") if msg_comps[0] == "send": @@ -77,7 +80,7 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "send," + source_user + "," + dest_user + "," + str(amount) - packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, self.next_msg_id_func()) await self.floodsub.publish(my_id, packet.SerializeToString()) async def publish_set_crypto(self, user, amount): @@ -88,7 +91,7 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "set," + user + "," + str(amount) - packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, self.next_msg_id_func()) await self.floodsub.publish(my_id, packet.SerializeToString()) @@ -99,6 +102,7 @@ class DummyAccountNode(): :param dest_user: user to send crypto to :param amount: amount of crypto to send """ + print("handle send " + self.node_id) if source_user in self.balances: self.balances[source_user] -= amount else: @@ -115,6 +119,7 @@ class DummyAccountNode(): :param dest_user: user to set crypto for :param amount: amount of crypto """ + print("handle set " + self.node_id) self.balances[dest_user] = amount def get_balance(self, user): From 60fe42bf60ea55c64bd95d39420ba688b9d7a950 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 00:34:39 -0400 Subject: [PATCH 068/131] Fix seen messages bug --- libp2p/pubsub/pubsub.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index d5d4b52..d8fb3c0 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -87,9 +87,10 @@ class Pubsub(): if rpc_incoming.publish: # deal with RPC.publish for message in rpc_incoming.publish: - if message.seqno not in self.seen_messages: + id_in_seen_msgs = (message.seqno, message.from_id) + if id_in_seen_msgs not in self.seen_messages: should_publish = True - self.seen_messages.append((message.seqno, message.from_id)) + self.seen_messages.append(id_in_seen_msgs) await self.handle_talk(message) if rpc_incoming.subscriptions: From 62d1bff888fc2f5c7dae4caa7866077cba331eaf Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 00:35:39 -0400 Subject: [PATCH 069/131] Add dummy node test --- tests/pubsub/test_dummyaccount_demo.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/pubsub/test_dummyaccount_demo.py b/tests/pubsub/test_dummyaccount_demo.py index 1f08c8d..9fa2aa7 100644 --- a/tests/pubsub/test_dummyaccount_demo.py +++ b/tests/pubsub/test_dummyaccount_demo.py @@ -185,3 +185,28 @@ async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): assert dummy_node.get_balance("rob") == 12 await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_set_then_send_from_five_diff_nodes_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("alex", 20) + await asyncio.sleep(1) + await dummy_nodes[1].publish_send_crypto("alex", "rob", 3) + await asyncio.sleep(1) + await dummy_nodes[2].publish_send_crypto("rob", "aspyn", 2) + await asyncio.sleep(1) + await dummy_nodes[3].publish_send_crypto("aspyn", "zx", 1) + await asyncio.sleep(1) + await dummy_nodes[4].publish_send_crypto("zx", "raul", 1) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("alex") == 17 + assert dummy_node.get_balance("rob") == 1 + assert dummy_node.get_balance("aspyn") == 1 + assert dummy_node.get_balance("zx") == 0 + assert dummy_node.get_balance("raul") == 1 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) From db53976a6a80f59cf7eb5f2427f7681ac3a15a62 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 14:17:33 -0400 Subject: [PATCH 070/131] Add test for multiple messages from two origins --- tests/pubsub/test_floodsub.py | 54 +++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index ba9d33c..00ac956 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -433,3 +433,57 @@ async def test_three_nodes_clique_two_topic_diff_origin_test_obj(): ] } await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_four_nodes_clique_two_topic_diff_origin_many_msgs_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2", "3", "4"], + "2": ["1", "3", "4"], + "3": ["1", "2", "4"], + "4": ["1", "2", "3"] + }, + "topic_map": { + "astrophysics": ["1", "2", "3", "4"], + "school": ["1", "2", "3", "4"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar2", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar3", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic3", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) From 0713cddbd15b51caee8cef3a26f8dd59ddb5040b Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 14:20:50 -0400 Subject: [PATCH 071/131] Add test for ring topology multiple messages from two origins --- tests/pubsub/test_floodsub.py | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index 00ac956..2d4ed8d 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -487,3 +487,58 @@ async def test_four_nodes_clique_two_topic_diff_origin_many_msgs_test_obj(): ] } await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_five_nodes_ring_two_topic_diff_origin_many_msgs_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2"], + "2": ["3"], + "3": ["4"], + "4": ["5"], + "5": ["1"] + }, + "topic_map": { + "astrophysics": ["1", "2", "3", "4", "5"], + "school": ["1", "2", "3", "4", "5"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar2", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar3", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic3", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) \ No newline at end of file From 204c103d92592161aab6497a92e0ead7a3aa14d9 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 18:12:09 -0500 Subject: [PATCH 072/131] Add notifee interface --- libp2p/network/notifee_interface.py | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 libp2p/network/notifee_interface.py diff --git a/libp2p/network/notifee_interface.py b/libp2p/network/notifee_interface.py new file mode 100644 index 0000000..fbc61f4 --- /dev/null +++ b/libp2p/network/notifee_interface.py @@ -0,0 +1,46 @@ +from abc import ABC, abstractmethod + + +class INotifee(ABC): + + @abstractmethod + def opened_stream(self, network, stream): + """ + :param network: network the stream was opened on + :param stream: stream that was opened + """ + + @abstractmethod + def closed_stream(self, network, stream): + """ + :param network: network the stream was closed on + :param stream: stream that was closed + """ + + @abstractmethod + def connected(self, network, conn): + """ + :param network: network the connection was opened on + :param conn: connection that was opened + """ + + @abstractmethod + def disconnected(self, network, conn): + """ + :param network: network the connection was closed on + :param conn: connection that was closed + """ + + @abstractmethod + def listen(self, network, multiaddr): + """ + :param network: network the listener is listening on + :param multiaddr: multiaddress listener is listening on + """ + + @abstractmethod + def listen_close(self, network, multiaddr): + """ + :param network: network the connection was opened on + :param multiaddr: multiaddress listener is no longer listening on + """ From 1701fba42e3af0bac11e3fe556dc1d6a8520230f Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 18:12:33 -0500 Subject: [PATCH 073/131] Add notify function to network interface --- libp2p/network/network_interface.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libp2p/network/network_interface.py b/libp2p/network/network_interface.py index 1867efc..acd20b1 100644 --- a/libp2p/network/network_interface.py +++ b/libp2p/network/network_interface.py @@ -41,3 +41,9 @@ class INetwork(ABC): :param *args: one or many multiaddrs to start listening on :return: True if at least one success """ + + @abstractmethod + def notify(self, notifee): + """ + :param notifee: object implementing Notifee interface + """ From c465141331819701bb5efeaf647fea15d3fad02c Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 18:18:58 -0500 Subject: [PATCH 074/131] Implement notify feature --- libp2p/network/swarm.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 705ee31..1ecc0df 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -24,6 +24,9 @@ class Swarm(INetwork): self.multiselect = Multiselect() self.multiselect_client = MultiselectClient() + # Create Notifee array + self.notifees = [] + # Create generic protocol handler self.generic_protocol_handler = create_generic_protocol_handler(self) @@ -61,6 +64,7 @@ class Swarm(INetwork): # set muxed connection equal to existing muxed connection muxed_conn = self.connections[peer_id] else: + # Dial peer (connection to peer does not yet exist) # Transport dials peer (gets back a raw conn) raw_conn = await self.transport.dial(multiaddr, self.self_id) @@ -70,6 +74,10 @@ class Swarm(INetwork): # Store muxed connection in connections self.connections[peer_id] = muxed_conn + # Call notifiers since event occurred + for notifee in self.notifees: + notifee.connected(self, muxed_conn) + return muxed_conn async def new_stream(self, peer_id, protocol_ids): @@ -100,6 +108,10 @@ class Swarm(INetwork): net_stream = NetStream(muxed_stream) net_stream.set_protocol(selected_protocol) + # Call notifiers since event occurred + for notifee in self.notifees: + notifee.opened_stream(self, net_stream) + return net_stream async def listen(self, *args): @@ -137,11 +149,20 @@ class Swarm(INetwork): # Store muxed_conn with peer id self.connections[peer_id] = muxed_conn + # Call notifiers since event occurred + for notifee in self.notifees: + notifee.connected(self, muxed_conn) + try: # Success listener = self.transport.create_listener(conn_handler) self.listeners[str(multiaddr)] = listener await listener.listen(multiaddr) + + # Call notifiers since event occurred + for notifee in self.notifees: + notifee.listen(self, multiaddr) + return True except IOError: # Failed. Continue looping. @@ -150,6 +171,13 @@ class Swarm(INetwork): # No multiaddr succeeded return False + def notify(self, notifee): + """ + :param notifee: object implementing Notifee interface + """ + # TODO: Add check to ensure notifee conforms to Notifee interface + self.notifees.append(notifee) + def add_transport(self, transport): # TODO: Support more than one transport self.transport = transport From 3beec9d4ae86053d6dd810d109891f089f00affd Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 19:08:13 -0500 Subject: [PATCH 075/131] Add tests for notify --- libp2p/network/notifee_interface.py | 1 - tests/libp2p/test_notify.py | 176 ++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 tests/libp2p/test_notify.py diff --git a/libp2p/network/notifee_interface.py b/libp2p/network/notifee_interface.py index fbc61f4..4ce9ed5 100644 --- a/libp2p/network/notifee_interface.py +++ b/libp2p/network/notifee_interface.py @@ -1,6 +1,5 @@ from abc import ABC, abstractmethod - class INotifee(ABC): @abstractmethod diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py new file mode 100644 index 0000000..1df267c --- /dev/null +++ b/tests/libp2p/test_notify.py @@ -0,0 +1,176 @@ +import multiaddr +import pytest, asyncio + +from tests.utils import cleanup +from libp2p import new_node +from libp2p.peer.peerinfo import info_from_p2p_addr +from libp2p.network.notifee_interface import INotifee + +# pylint: disable=too-many-locals + +""" +Test Notify and Notifee by ensuring that the proper events get +called at the proper time, and that the stream passed into +opened_stream is correct + +Note: Listen event does not get hit because MyNotifee is passed +into network after network has already started listening + +TODO: Add tests to ensure conn is the correct connection +TODO: Add tests for closed_stream disconnected, listen_close as those +features are implemented in network +passed into connected +""" +class MyNotifee(INotifee): + # pylint: disable=too-many-instance-attributes, cell-var-from-loop + + def __init__(self, events, val_to_append_to_event): + self.events = events + self.val_to_append_to_event = val_to_append_to_event + + def opened_stream(self, network, stream): + self.events.append(["opened_stream" + \ + self.val_to_append_to_event, stream]) + + def closed_stream(self, network, stream): + pass + + def connected(self, network, conn): + self.events.append("connected" + self.val_to_append_to_event) + + def disconnected(self, network, conn): + pass + + def listen(self, network, multiaddr): + pass + + def listen_close(self, network, multiaddr): + pass + +@pytest.mark.asyncio +async def test_one_notifier(): + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + async def stream_handler(stream): + while True: + read_string = (await stream.read()).decode() + + response = "ack:" + read_string + await stream.write(response.encode()) + + node_b.set_stream_handler("/echo/1.0.0", stream_handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + + # Add notifee for node_a + events = [] + node_a.get_network().notify(MyNotifee(events, "0")) + + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # Ensure the connected and opened_stream events were hit in MyNotifee obj + # and that stream passed into opened_stream matches the stream created on + # node_a + assert events == ["connected0", ["opened_stream0", stream]] + + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() + +@pytest.mark.asyncio +async def test_two_notifiers(): + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + async def stream_handler(stream): + while True: + read_string = (await stream.read()).decode() + + response = "ack:" + read_string + await stream.write(response.encode()) + + node_b.set_stream_handler("/echo/1.0.0", stream_handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + + # Add notifee for node_a + events0 = [] + node_a.get_network().notify(MyNotifee(events0, "0")) + + events1 = [] + node_a.get_network().notify(MyNotifee(events1, "1")) + + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # Ensure the connected and opened_stream events were hit in both Notifee objs + # and that the stream passed into opened_stream matches the stream created on + # node_a + assert events0 == ["connected0", ["opened_stream0", stream]] + assert events1 == ["connected1", ["opened_stream1", stream]] + + + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() + +@pytest.mark.asyncio +async def test_ten_notifiers(): + num_notifiers = 10 + + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + async def stream_handler(stream): + while True: + read_string = (await stream.read()).decode() + + response = "ack:" + read_string + await stream.write(response.encode()) + + node_b.set_stream_handler("/echo/1.0.0", stream_handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + + # Add notifee for node_a + events_lst = [] + for i in range(num_notifiers): + events_lst.append([]) + node_a.get_network().notify(MyNotifee(events_lst[i], str(i))) + + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # Ensure the connected and opened_stream events were hit in both Notifee objs + # and that the stream passed into opened_stream matches the stream created on + # node_a + for i in range(num_notifiers): + assert events_lst[i] == ["connected" + str(i), \ + ["opened_stream" + str(i), stream]] + + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() \ No newline at end of file From 1ecdcf7df46f4a0bd97f2ef78bfa0a1832650a35 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 19:11:04 -0500 Subject: [PATCH 076/131] Make notifee functions all async --- libp2p/network/notifee_interface.py | 13 +++++++------ libp2p/network/swarm.py | 8 ++++---- tests/libp2p/test_notify.py | 12 ++++++------ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/libp2p/network/notifee_interface.py b/libp2p/network/notifee_interface.py index 4ce9ed5..837ecd1 100644 --- a/libp2p/network/notifee_interface.py +++ b/libp2p/network/notifee_interface.py @@ -1,44 +1,45 @@ +import asyncio from abc import ABC, abstractmethod class INotifee(ABC): @abstractmethod - def opened_stream(self, network, stream): + async def opened_stream(self, network, stream): """ :param network: network the stream was opened on :param stream: stream that was opened """ @abstractmethod - def closed_stream(self, network, stream): + async def closed_stream(self, network, stream): """ :param network: network the stream was closed on :param stream: stream that was closed """ @abstractmethod - def connected(self, network, conn): + async def connected(self, network, conn): """ :param network: network the connection was opened on :param conn: connection that was opened """ @abstractmethod - def disconnected(self, network, conn): + async def disconnected(self, network, conn): """ :param network: network the connection was closed on :param conn: connection that was closed """ @abstractmethod - def listen(self, network, multiaddr): + async def listen(self, network, multiaddr): """ :param network: network the listener is listening on :param multiaddr: multiaddress listener is listening on """ @abstractmethod - def listen_close(self, network, multiaddr): + async def listen_close(self, network, multiaddr): """ :param network: network the connection was opened on :param multiaddr: multiaddress listener is no longer listening on diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 1ecc0df..e6bf288 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -76,7 +76,7 @@ class Swarm(INetwork): # Call notifiers since event occurred for notifee in self.notifees: - notifee.connected(self, muxed_conn) + await notifee.connected(self, muxed_conn) return muxed_conn @@ -110,7 +110,7 @@ class Swarm(INetwork): # Call notifiers since event occurred for notifee in self.notifees: - notifee.opened_stream(self, net_stream) + await notifee.opened_stream(self, net_stream) return net_stream @@ -151,7 +151,7 @@ class Swarm(INetwork): # Call notifiers since event occurred for notifee in self.notifees: - notifee.connected(self, muxed_conn) + await notifee.connected(self, muxed_conn) try: # Success @@ -161,7 +161,7 @@ class Swarm(INetwork): # Call notifiers since event occurred for notifee in self.notifees: - notifee.listen(self, multiaddr) + await notifee.listen(self, multiaddr) return True except IOError: diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 1df267c..05b2c4f 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -28,23 +28,23 @@ class MyNotifee(INotifee): self.events = events self.val_to_append_to_event = val_to_append_to_event - def opened_stream(self, network, stream): + async def opened_stream(self, network, stream): self.events.append(["opened_stream" + \ self.val_to_append_to_event, stream]) - def closed_stream(self, network, stream): + async def closed_stream(self, network, stream): pass - def connected(self, network, conn): + async def connected(self, network, conn): self.events.append("connected" + self.val_to_append_to_event) - def disconnected(self, network, conn): + async def disconnected(self, network, conn): pass - def listen(self, network, multiaddr): + async def listen(self, network, multiaddr): pass - def listen_close(self, network, multiaddr): + async def listen_close(self, network, multiaddr): pass @pytest.mark.asyncio From 4298b0631fbaf10b483f7f4d6ab21efb7f75c46a Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 19:13:26 -0500 Subject: [PATCH 077/131] Fix linting issue --- libp2p/network/notifee_interface.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libp2p/network/notifee_interface.py b/libp2p/network/notifee_interface.py index 837ecd1..17ba1fe 100644 --- a/libp2p/network/notifee_interface.py +++ b/libp2p/network/notifee_interface.py @@ -1,4 +1,3 @@ -import asyncio from abc import ABC, abstractmethod class INotifee(ABC): From 66703bd4148a0f5d91c5d12f00767f1b5aaa1a66 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 28 Feb 2019 19:17:11 -0500 Subject: [PATCH 078/131] Fix linting issue --- tests/libp2p/test_notify.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 05b2c4f..e9a62cd 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -1,26 +1,23 @@ -import multiaddr -import pytest, asyncio +import pytest from tests.utils import cleanup from libp2p import new_node -from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.network.notifee_interface import INotifee # pylint: disable=too-many-locals """ Test Notify and Notifee by ensuring that the proper events get -called at the proper time, and that the stream passed into -opened_stream is correct +called, and that the stream passed into opened_stream is correct Note: Listen event does not get hit because MyNotifee is passed into network after network has already started listening TODO: Add tests to ensure conn is the correct connection -TODO: Add tests for closed_stream disconnected, listen_close as those -features are implemented in network -passed into connected +TODO: Add tests for closed_stream disconnected, listen_close when those +features are implemented in swarm """ + class MyNotifee(INotifee): # pylint: disable=too-many-instance-attributes, cell-var-from-loop @@ -173,4 +170,4 @@ async def test_ten_notifiers(): assert response == ("ack:" + message) # Success, terminate pending tasks. - await cleanup() \ No newline at end of file + await cleanup() From 86ce7530d59edb97033c8ed90679710b6eab2953 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Sun, 3 Mar 2019 23:34:49 +1100 Subject: [PATCH 079/131] Add conn to net stream and conn tests --- libp2p/network/stream/net_stream.py | 1 + tests/libp2p/test_notify.py | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/libp2p/network/stream/net_stream.py b/libp2p/network/stream/net_stream.py index 8cfb635..f708962 100644 --- a/libp2p/network/stream/net_stream.py +++ b/libp2p/network/stream/net_stream.py @@ -5,6 +5,7 @@ class NetStream(INetStream): def __init__(self, muxed_stream): self.muxed_stream = muxed_stream + self.mplex_conn = muxed_stream.mplex_conn self.protocol_id = None def get_protocol(self): diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index e9a62cd..3021cf1 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -33,7 +33,8 @@ class MyNotifee(INotifee): pass async def connected(self, network, conn): - self.events.append("connected" + self.val_to_append_to_event) + self.events.append(["connected" + self.val_to_append_to_event,\ + conn]) async def disconnected(self, network, conn): pass @@ -70,7 +71,8 @@ async def test_one_notifier(): # Ensure the connected and opened_stream events were hit in MyNotifee obj # and that stream passed into opened_stream matches the stream created on # node_a - assert events == ["connected0", ["opened_stream0", stream]] + assert events == [["connected0", stream.mplex_conn], \ + ["opened_stream0", stream]] messages = ["hello", "hello"] for message in messages: @@ -112,8 +114,8 @@ async def test_two_notifiers(): # Ensure the connected and opened_stream events were hit in both Notifee objs # and that the stream passed into opened_stream matches the stream created on # node_a - assert events0 == ["connected0", ["opened_stream0", stream]] - assert events1 == ["connected1", ["opened_stream1", stream]] + assert events0 == [["connected0", stream.mplex_conn], ["opened_stream0", stream]] + assert events1 == [["connected1", stream.mplex_conn], ["opened_stream1", stream]] messages = ["hello", "hello"] @@ -158,7 +160,7 @@ async def test_ten_notifiers(): # and that the stream passed into opened_stream matches the stream created on # node_a for i in range(num_notifiers): - assert events_lst[i] == ["connected" + str(i), \ + assert events_lst[i] == [["connected" + str(i), stream.mplex_conn], \ ["opened_stream" + str(i), stream]] messages = ["hello", "hello"] From fa97b977482ce1ad433001a34cb9b84f6194becf Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Sun, 3 Mar 2019 23:44:15 +1100 Subject: [PATCH 080/131] Refactor test setup to remove duplicate code --- tests/libp2p/test_notify.py | 38 ++++++++----------------------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 3021cf1..80dd6c1 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -45,8 +45,7 @@ class MyNotifee(INotifee): async def listen_close(self, network, multiaddr): pass -@pytest.mark.asyncio -async def test_one_notifier(): +async def perform_two_host_simple_set_up(): node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -61,6 +60,11 @@ async def test_one_notifier(): # Associate the peer with local ip address (see default parameters of Libp2p()) node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + return node_a, node_b + +@pytest.mark.asyncio +async def test_one_notifier(): + node_a, node_b = await perform_two_host_simple_set_up() # Add notifee for node_a events = [] @@ -87,20 +91,7 @@ async def test_one_notifier(): @pytest.mark.asyncio async def test_two_notifiers(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - - async def stream_handler(stream): - while True: - read_string = (await stream.read()).decode() - - response = "ack:" + read_string - await stream.write(response.encode()) - - node_b.set_stream_handler("/echo/1.0.0", stream_handler) - - # Associate the peer with local ip address (see default parameters of Libp2p()) - node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + node_a, node_b = await perform_two_host_simple_set_up() # Add notifee for node_a events0 = [] @@ -133,20 +124,7 @@ async def test_two_notifiers(): async def test_ten_notifiers(): num_notifiers = 10 - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - - async def stream_handler(stream): - while True: - read_string = (await stream.read()).decode() - - response = "ack:" + read_string - await stream.write(response.encode()) - - node_b.set_stream_handler("/echo/1.0.0", stream_handler) - - # Associate the peer with local ip address (see default parameters of Libp2p()) - node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + node_a, node_b = await perform_two_host_simple_set_up() # Add notifee for node_a events_lst = [] From f58afc0eac59b53705bce4469b694771e79aa867 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Sun, 3 Mar 2019 23:49:28 +1100 Subject: [PATCH 081/131] Fix linting issues --- tests/libp2p/test_notify.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 80dd6c1..5ebed1b 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -49,14 +49,14 @@ async def perform_two_host_simple_set_up(): node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - async def stream_handler(stream): + async def my_stream_handler(stream): while True: read_string = (await stream.read()).decode() - response = "ack:" + read_string - await stream.write(response.encode()) + resp = "ack:" + read_string + await stream.write(resp.encode()) - node_b.set_stream_handler("/echo/1.0.0", stream_handler) + node_b.set_stream_handler("/echo/1.0.0", my_stream_handler) # Associate the peer with local ip address (see default parameters of Libp2p()) node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) From 5cabc1168783b0f10a1ddb5d221846827a7988c4 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Sun, 3 Mar 2019 23:54:04 +1100 Subject: [PATCH 082/131] Fix linting issue --- tests/libp2p/test_notify.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 5ebed1b..fdf7368 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -1,11 +1,3 @@ -import pytest - -from tests.utils import cleanup -from libp2p import new_node -from libp2p.network.notifee_interface import INotifee - -# pylint: disable=too-many-locals - """ Test Notify and Notifee by ensuring that the proper events get called, and that the stream passed into opened_stream is correct @@ -13,11 +5,18 @@ called, and that the stream passed into opened_stream is correct Note: Listen event does not get hit because MyNotifee is passed into network after network has already started listening -TODO: Add tests to ensure conn is the correct connection TODO: Add tests for closed_stream disconnected, listen_close when those features are implemented in swarm """ +import pytest + +from tests.utils import cleanup +from libp2p import new_node +from libp2p.network.notifee_interface import INotifee + +# pylint: disable=too-many-locals + class MyNotifee(INotifee): # pylint: disable=too-many-instance-attributes, cell-var-from-loop From 92bfd816b9f8e63b10d6e4a7b8dd704dfff6f039 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Sun, 3 Mar 2019 23:57:20 +1100 Subject: [PATCH 083/131] Fix linting issue --- libp2p/protocol_muxer/multiselect_client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libp2p/protocol_muxer/multiselect_client.py b/libp2p/protocol_muxer/multiselect_client.py index 8e5463a..4cc07d9 100644 --- a/libp2p/protocol_muxer/multiselect_client.py +++ b/libp2p/protocol_muxer/multiselect_client.py @@ -104,9 +104,8 @@ class MultiselectClient(IMultiselectClient): if response == protocol: return protocol if response == PROTOCOL_NOT_FOUND_MSG: - raise MultiselectClientError("protocol not supported") - else: - raise MultiselectClientError("unrecognized response: " + response) + raise MultiselectClientError("protocol not supported") + raise MultiselectClientError("unrecognized response: " + response) def validate_handshake(handshake_contents): From a0ef64fff70c8d64d9d9e4ed5e5a90f238d47909 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Sun, 3 Mar 2019 23:59:38 +1100 Subject: [PATCH 084/131] Fix outstanding unrelated lint issue in multiselect_client --- libp2p/protocol_muxer/multiselect_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libp2p/protocol_muxer/multiselect_client.py b/libp2p/protocol_muxer/multiselect_client.py index 4cc07d9..dec5493 100644 --- a/libp2p/protocol_muxer/multiselect_client.py +++ b/libp2p/protocol_muxer/multiselect_client.py @@ -104,7 +104,7 @@ class MultiselectClient(IMultiselectClient): if response == protocol: return protocol if response == PROTOCOL_NOT_FOUND_MSG: - raise MultiselectClientError("protocol not supported") + raise MultiselectClientError("protocol not supported") raise MultiselectClientError("unrecognized response: " + response) From 4b9327c5a36a6b77f4e2b61bbd6307c77fae37d2 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 14 Mar 2019 12:47:40 -0400 Subject: [PATCH 085/131] Add opened_stream call for non-initiator --- libp2p/network/swarm.py | 9 +++- tests/libp2p/test_notify.py | 100 ++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index e6bf288..daabb64 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -5,6 +5,7 @@ from libp2p.protocol_muxer.multiselect import Multiselect from libp2p.peer.id import id_b58_decode from .network_interface import INetwork +from .notifee_interface import INotifee from .stream.net_stream import NetStream from .connection.raw_connection import RawConnection @@ -175,8 +176,8 @@ class Swarm(INetwork): """ :param notifee: object implementing Notifee interface """ - # TODO: Add check to ensure notifee conforms to Notifee interface - self.notifees.append(notifee) + if isinstance(notifee, INotifee): + self.notifees.append(notifee) def add_transport(self, transport): # TODO: Support more than one transport @@ -195,6 +196,10 @@ def create_generic_protocol_handler(swarm): # Perform protocol muxing to determine protocol to use _, handler = await multiselect.negotiate(muxed_stream) + # Call notifiers since event occurred + for notifee in swarm.notifees: + await notifee.opened_stream(swarm, muxed_stream) + # Give to stream handler asyncio.ensure_future(handler(muxed_stream)) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index fdf7368..37b287a 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -44,6 +44,27 @@ class MyNotifee(INotifee): async def listen_close(self, network, multiaddr): pass +class InvalidNotifee(): + # pylint: disable=too-many-instance-attributes, cell-var-from-loop + + def __init__(self): + pass + + async def opened_stream(self, network, stream): + assert False + + async def closed_stream(self, network, stream): + assert False + + async def connected(self, network, conn): + assert False + + async def disconnected(self, network, conn): + assert False + + async def listen(self, network, multiaddr): + assert False + async def perform_two_host_simple_set_up(): node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -61,6 +82,16 @@ async def perform_two_host_simple_set_up(): node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) return node_a, node_b +async def perform_two_host_simple_set_up_custom_handler(handler): + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + node_b.set_stream_handler("/echo/1.0.0", handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + return node_a, node_b + @pytest.mark.asyncio async def test_one_notifier(): node_a, node_b = await perform_two_host_simple_set_up() @@ -88,6 +119,47 @@ async def test_one_notifier(): # Success, terminate pending tasks. await cleanup() +@pytest.mark.asyncio +async def test_one_notifier_on_two_nodes(): + events_b = [] + + async def my_stream_handler(stream): + assert events_b == [["connectedb", stream.mplex_conn], \ + ["opened_streamb", stream]] + while True: + read_string = (await stream.read()).decode() + + resp = "ack:" + read_string + await stream.write(resp.encode()) + + node_a, node_b = await perform_two_host_simple_set_up_custom_handler(my_stream_handler) + + # Add notifee for node_a + events_a = [] + node_a.get_network().notify(MyNotifee(events_a, "a")) + + # Add notifee for node_b + node_b.get_network().notify(MyNotifee(events_b, "b")) + + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # Ensure the connected and opened_stream events were hit in MyNotifee obj + # and that stream passed into opened_stream matches the stream created on + # node_a + assert events_a == [["connecteda", stream.mplex_conn], \ + ["opened_streama", stream]] + + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() + @pytest.mark.asyncio async def test_two_notifiers(): node_a, node_b = await perform_two_host_simple_set_up() @@ -150,3 +222,31 @@ async def test_ten_notifiers(): # Success, terminate pending tasks. await cleanup() + +@pytest.mark.asyncio +async def test_invalid_notifee(): + num_notifiers = 10 + + node_a, node_b = await perform_two_host_simple_set_up() + + # Add notifee for node_a + events_lst = [] + for i in range(num_notifiers): + events_lst.append([]) + node_a.get_network().notify(InvalidNotifee()) + + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # If this point is reached, this implies that the InvalidNotifee instance + # did not assert false, i.e. no functions of InvalidNotifee were called (which is correct + # given that InvalidNotifee should not have been added as a notifee) + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() From dfe2b0e7e72935530c93919d17a51fc3d48fc210 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 14 Mar 2019 14:01:37 -0400 Subject: [PATCH 086/131] Add return value to Notify --- libp2p/network/network_interface.py | 1 + libp2p/network/swarm.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/libp2p/network/network_interface.py b/libp2p/network/network_interface.py index acd20b1..71e813c 100644 --- a/libp2p/network/network_interface.py +++ b/libp2p/network/network_interface.py @@ -46,4 +46,5 @@ class INetwork(ABC): def notify(self, notifee): """ :param notifee: object implementing Notifee interface + :return: true if notifee registered successfully, false otherwise """ diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index daabb64..f2943ba 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -175,9 +175,12 @@ class Swarm(INetwork): def notify(self, notifee): """ :param notifee: object implementing Notifee interface + :return: true if notifee registered successfully, false otherwise """ if isinstance(notifee, INotifee): self.notifees.append(notifee) + return True + return False def add_transport(self, transport): # TODO: Support more than one transport From ed3c5f4b51b9f2ac9ea18d7afdb74214342e9bf3 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 14 Mar 2019 14:01:59 -0400 Subject: [PATCH 087/131] Add additional initiator and non-initiator notifee tests --- tests/libp2p/test_notify.py | 79 ++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 37b287a..108f04a 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -50,19 +50,19 @@ class InvalidNotifee(): def __init__(self): pass - async def opened_stream(self, network, stream): + async def opened_stream(self): assert False - async def closed_stream(self, network, stream): + async def closed_stream(self): assert False - async def connected(self, network, conn): + async def connected(self): assert False - async def disconnected(self, network, conn): + async def disconnected(self): assert False - async def listen(self, network, multiaddr): + async def listen(self): assert False async def perform_two_host_simple_set_up(): @@ -98,7 +98,7 @@ async def test_one_notifier(): # Add notifee for node_a events = [] - node_a.get_network().notify(MyNotifee(events, "0")) + assert node_a.get_network().notify(MyNotifee(events, "0")) stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) @@ -122,8 +122,11 @@ async def test_one_notifier(): @pytest.mark.asyncio async def test_one_notifier_on_two_nodes(): events_b = [] - + async def my_stream_handler(stream): + # Ensure the connected and opened_stream events were hit in Notifee obj + # and that the stream passed into opened_stream matches the stream created on + # node_b assert events_b == [["connectedb", stream.mplex_conn], \ ["opened_streamb", stream]] while True: @@ -136,10 +139,10 @@ async def test_one_notifier_on_two_nodes(): # Add notifee for node_a events_a = [] - node_a.get_network().notify(MyNotifee(events_a, "a")) + assert node_a.get_network().notify(MyNotifee(events_a, "a")) # Add notifee for node_b - node_b.get_network().notify(MyNotifee(events_b, "b")) + assert node_b.get_network().notify(MyNotifee(events_b, "b")) stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) @@ -166,10 +169,10 @@ async def test_two_notifiers(): # Add notifee for node_a events0 = [] - node_a.get_network().notify(MyNotifee(events0, "0")) + assert node_a.get_network().notify(MyNotifee(events0, "0")) events1 = [] - node_a.get_network().notify(MyNotifee(events1, "1")) + assert node_a.get_network().notify(MyNotifee(events1, "1")) stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) @@ -201,7 +204,7 @@ async def test_ten_notifiers(): events_lst = [] for i in range(num_notifiers): events_lst.append([]) - node_a.get_network().notify(MyNotifee(events_lst[i], str(i))) + assert node_a.get_network().notify(MyNotifee(events_lst[i], str(i))) stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) @@ -223,6 +226,54 @@ async def test_ten_notifiers(): # Success, terminate pending tasks. await cleanup() +@pytest.mark.asyncio +async def test_ten_notifiers_on_two_nodes(): + num_notifiers = 10 + events_lst_b = [] + + async def my_stream_handler(stream): + # Ensure the connected and opened_stream events were hit in all Notifee objs + # and that the stream passed into opened_stream matches the stream created on + # node_b + for i in range(num_notifiers): + assert events_lst_b[i] == [["connectedb" + str(i), stream.mplex_conn], \ + ["opened_streamb" + str(i), stream]] + while True: + read_string = (await stream.read()).decode() + + resp = "ack:" + read_string + await stream.write(resp.encode()) + + node_a, node_b = await perform_two_host_simple_set_up_custom_handler(my_stream_handler) + + # Add notifee for node_a and node_b + events_lst_a = [] + for i in range(num_notifiers): + events_lst_a.append([]) + events_lst_b.append([]) + assert node_a.get_network().notify(MyNotifee(events_lst_a[i], "a" + str(i))) + assert node_b.get_network().notify(MyNotifee(events_lst_b[i], "b" + str(i))) + + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # Ensure the connected and opened_stream events were hit in all Notifee objs + # and that the stream passed into opened_stream matches the stream created on + # node_a + for i in range(num_notifiers): + assert events_lst_a[i] == [["connecteda" + str(i), stream.mplex_conn], \ + ["opened_streama" + str(i), stream]] + + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() + @pytest.mark.asyncio async def test_invalid_notifee(): num_notifiers = 10 @@ -231,9 +282,9 @@ async def test_invalid_notifee(): # Add notifee for node_a events_lst = [] - for i in range(num_notifiers): + for _ in range(num_notifiers): events_lst.append([]) - node_a.get_network().notify(InvalidNotifee()) + assert not node_a.get_network().notify(InvalidNotifee()) stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) From fca5782d01a5277a399ba1e6a68ef7364869183a Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 17 Mar 2019 19:20:46 -0400 Subject: [PATCH 088/131] refactor new_node --- libp2p/__init__.py | 60 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/libp2p/__init__.py b/libp2p/__init__.py index 0fa2fdb..d83ec3c 100644 --- a/libp2p/__init__.py +++ b/libp2p/__init__.py @@ -11,6 +11,9 @@ from .transport.tcp.tcp import TCP async def cleanup_done_tasks(): + """ + clean up asyncio done tasks to free up resources + """ while True: for task in asyncio.all_tasks(): if task.done(): @@ -20,30 +23,59 @@ async def cleanup_done_tasks(): # Some sleep necessary to context switch await asyncio.sleep(3) -async def new_node( +def initialize_default_swarm( id_opt=None, transport_opt=None, - muxer_opt=None, sec_opt=None, peerstore=None): - - if id_opt is None: + muxer_opt=None, sec_opt=None, peerstore_opt=None): + """ + initialize swarm when no swarm is passed in + :param id_opt: optional id for host + :param transport_opt: optional choice of transport upgrade + :param muxer_opt: optional choice of stream muxer + :param sec_opt: optional choice of security upgrade + :param peerstore_opt: optional peerstore + :return: return a default swarm instance + """ + # pylint: disable=too-many-arguments + if not id_opt: new_key = RSA.generate(2048, e=65537) id_opt = id_from_public_key(new_key.publickey()) # private_key = new_key.exportKey("PEM") transport_opt = transport_opt or ["/ip4/127.0.0.1/tcp/8001"] - transport_opt = [multiaddr.Multiaddr(t) for t in transport_opt] - muxer_opt = muxer_opt or ["mplex/6.7.0"] - sec_opt = sec_opt or ["secio"] - peerstore = peerstore or PeerStore() - - upgrader = TransportUpgrader(sec_opt, transport_opt) - swarm = Swarm(id_opt, peerstore, upgrader) + transport = [multiaddr.Multiaddr(t) for t in transport_opt] + # TODO wire muxer up with swarm + # muxer = muxer_opt or ["mplex/6.7.0"] + sec = sec_opt or ["secio"] + peerstore = peerstore_opt or PeerStore() + upgrader = TransportUpgrader(sec, transport) + swarm_opt = Swarm(id_opt, peerstore, upgrader) tcp = TCP() - swarm.add_transport(tcp) - await swarm.listen(transport_opt[0]) + swarm_opt.add_transport(tcp) + + return swarm_opt + +async def new_node( + swarm_opt=None, id_opt=None, transport_opt=None, + muxer_opt=None, sec_opt=None, peerstore_opt=None): + """ + create new libp2p node + :param id_opt: optional id for host + :param transport_opt: optional choice of transport upgrade + :param muxer_opt: optional choice of stream muxer + :param sec_opt: optional choice of security upgrade + :param peerstore_opt: optional peerstore + :return: return a default swarm instance + """ + # pylint: disable=too-many-arguments + if not swarm_opt: + swarm_opt = initialize_default_swarm( + id_opt=id_opt, transport_opt=transport_opt, + muxer_opt=muxer_opt, sec_opt=sec_opt, + peerstore_opt=peerstore_opt) # TODO enable support for other host type # TODO routing unimplemented - host = BasicHost(swarm) + host = BasicHost(swarm_opt) # Kick off cleanup job asyncio.ensure_future(cleanup_done_tasks()) From 4c89a59406f5721f7c0d521a9c9645f39da22058 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 17 Mar 2019 19:33:10 -0400 Subject: [PATCH 089/131] refactor host setup helper --- tests/utils.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index c995412..e7f9b85 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,8 @@ -import asyncio - from contextlib import suppress +import asyncio +import multiaddr + +from libp2p import new_node async def cleanup(): @@ -12,3 +14,18 @@ async def cleanup(): # Cancelled task raises asyncio.CancelledError that we can suppress: with suppress(asyncio.CancelledError): await task + +async def set_up_nodes_by_transport_opt(transport_opt_list): + nodes_list = [] + for transport_opt in transport_opt_list: + node = await new_node(transport_opt=transport_opt) + await node.get_network().listen(multiaddr.Multiaddr(transport_opt[0])) + nodes_list.append(node) + return tuple(nodes_list) + +async def echo_stream_handler(stream): + while True: + read_string = (await stream.read()).decode() + + resp = "ack:" + read_string + await stream.write(resp.encode()) From 2cb0279b57c873bdb687c515bf5dfbb2c54f1b26 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 17 Mar 2019 19:33:40 -0400 Subject: [PATCH 090/131] fix existing tests --- tests/examples/test_chat.py | 8 ++--- tests/libp2p/test_libp2p.py | 35 ++++++++++--------- tests/libp2p/test_notify.py | 37 ++++++--------------- tests/protocol_muxer/test_protocol_muxer.py | 10 ++---- 4 files changed, 34 insertions(+), 56 deletions(-) diff --git a/tests/examples/test_chat.py b/tests/examples/test_chat.py index 0ea1979..ee192cc 100644 --- a/tests/examples/test_chat.py +++ b/tests/examples/test_chat.py @@ -1,7 +1,8 @@ import pytest import asyncio +import multiaddr -from tests.utils import cleanup +from tests.utils import cleanup, set_up_nodes_by_transport_opt from libp2p import new_node from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.protocol_muxer.multiselect_client import MultiselectClientError @@ -9,7 +10,6 @@ from libp2p.protocol_muxer.multiselect_client import MultiselectClientError PROTOCOL_ID = '/chat/1.0.0' - async def hello_world(host_a, host_b): async def stream_handler(stream): read = await stream.read() @@ -100,8 +100,8 @@ async def no_common_protocol(host_a, host_b): (no_common_protocol), ]) async def test_chat(test): - host_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - host_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (host_a, host_b) = await set_up_nodes_by_transport_opt(transport_opt_list) addr = host_a.get_addrs()[0] info = info_from_p2p_addr(addr) diff --git a/tests/libp2p/test_libp2p.py b/tests/libp2p/test_libp2p.py index 4b7e59c..3d74516 100644 --- a/tests/libp2p/test_libp2p.py +++ b/tests/libp2p/test_libp2p.py @@ -1,17 +1,16 @@ import multiaddr import pytest -from tests.utils import cleanup +from tests.utils import cleanup, set_up_nodes_by_transport_opt from libp2p import new_node from libp2p.peer.peerinfo import info_from_p2p_addr + # pylint: disable=too-many-locals - - @pytest.mark.asyncio async def test_simple_messages(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler(stream): while True: @@ -41,8 +40,8 @@ async def test_simple_messages(): @pytest.mark.asyncio async def test_double_response(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler(stream): while True: @@ -78,8 +77,8 @@ async def test_double_response(): async def test_multiple_streams(): # Node A should be able to open a stream with node B and then vice versa. # Stream IDs should be generated uniquely so that the stream state is not overwritten - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler_a(stream): while True: @@ -124,8 +123,8 @@ async def test_multiple_streams(): @pytest.mark.asyncio async def test_multiple_streams_same_initiator_different_protocols(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler_a1(stream): while True: @@ -184,8 +183,8 @@ async def test_multiple_streams_same_initiator_different_protocols(): @pytest.mark.asyncio async def test_multiple_streams_two_initiators(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler_a1(stream): while True: @@ -262,9 +261,9 @@ async def test_multiple_streams_two_initiators(): @pytest.mark.asyncio async def test_triangle_nodes_connection(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_c = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"],\ + ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b, node_c) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler(stream): while True: @@ -315,8 +314,8 @@ async def test_triangle_nodes_connection(): @pytest.mark.asyncio async def test_host_connect(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) assert not node_a.get_peerstore().peers() diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 108f04a..a2fecaf 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -11,7 +11,7 @@ features are implemented in swarm import pytest -from tests.utils import cleanup +from tests.utils import * from libp2p import new_node from libp2p.network.notifee_interface import INotifee @@ -65,26 +65,9 @@ class InvalidNotifee(): async def listen(self): assert False -async def perform_two_host_simple_set_up(): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - - async def my_stream_handler(stream): - while True: - read_string = (await stream.read()).decode() - - resp = "ack:" + read_string - await stream.write(resp.encode()) - - node_b.set_stream_handler("/echo/1.0.0", my_stream_handler) - - # Associate the peer with local ip address (see default parameters of Libp2p()) - node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) - return node_a, node_b - -async def perform_two_host_simple_set_up_custom_handler(handler): - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) +async def perform_two_host_set_up_custom_handler(handler): + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) node_b.set_stream_handler("/echo/1.0.0", handler) @@ -94,7 +77,7 @@ async def perform_two_host_simple_set_up_custom_handler(handler): @pytest.mark.asyncio async def test_one_notifier(): - node_a, node_b = await perform_two_host_simple_set_up() + node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) # Add notifee for node_a events = [] @@ -135,7 +118,7 @@ async def test_one_notifier_on_two_nodes(): resp = "ack:" + read_string await stream.write(resp.encode()) - node_a, node_b = await perform_two_host_simple_set_up_custom_handler(my_stream_handler) + node_a, node_b = await perform_two_host_set_up_custom_handler(my_stream_handler) # Add notifee for node_a events_a = [] @@ -165,7 +148,7 @@ async def test_one_notifier_on_two_nodes(): @pytest.mark.asyncio async def test_two_notifiers(): - node_a, node_b = await perform_two_host_simple_set_up() + node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) # Add notifee for node_a events0 = [] @@ -198,7 +181,7 @@ async def test_two_notifiers(): async def test_ten_notifiers(): num_notifiers = 10 - node_a, node_b = await perform_two_host_simple_set_up() + node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) # Add notifee for node_a events_lst = [] @@ -244,7 +227,7 @@ async def test_ten_notifiers_on_two_nodes(): resp = "ack:" + read_string await stream.write(resp.encode()) - node_a, node_b = await perform_two_host_simple_set_up_custom_handler(my_stream_handler) + node_a, node_b = await perform_two_host_set_up_custom_handler(my_stream_handler) # Add notifee for node_a and node_b events_lst_a = [] @@ -278,7 +261,7 @@ async def test_ten_notifiers_on_two_nodes(): async def test_invalid_notifee(): num_notifiers = 10 - node_a, node_b = await perform_two_host_simple_set_up() + node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) # Add notifee for node_a events_lst = [] diff --git a/tests/protocol_muxer/test_protocol_muxer.py b/tests/protocol_muxer/test_protocol_muxer.py index a7e19af..1eff211 100644 --- a/tests/protocol_muxer/test_protocol_muxer.py +++ b/tests/protocol_muxer/test_protocol_muxer.py @@ -1,6 +1,6 @@ import pytest -from tests.utils import cleanup +from tests.utils import cleanup, set_up_nodes_by_transport_opt from libp2p import new_node from libp2p.protocol_muxer.multiselect_client import MultiselectClientError @@ -15,12 +15,8 @@ from libp2p.protocol_muxer.multiselect_client import MultiselectClientError async def perform_simple_test(expected_selected_protocol, protocols_for_client, protocols_with_handlers): - transport_opt_a = ["/ip4/127.0.0.1/tcp/0"] - transport_opt_b = ["/ip4/127.0.0.1/tcp/0"] - node_a = await new_node( - transport_opt=transport_opt_a) - node_b = await new_node( - transport_opt=transport_opt_b) + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler(stream): while True: From 2e437e5b8b00b5acbb896827617c1cb162f1acd5 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 17 Mar 2019 21:15:14 -0400 Subject: [PATCH 091/131] add test for listen event --- tests/libp2p/test_notify.py | 79 +++++++++++++++++++++++++++++++------ tests/utils.py | 10 +++++ 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index a2fecaf..edaba2f 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -12,8 +12,9 @@ features are implemented in swarm import pytest from tests.utils import * -from libp2p import new_node +from libp2p import new_node, initialize_default_swarm from libp2p.network.notifee_interface import INotifee +from libp2p.host.basic_host import BasicHost # pylint: disable=too-many-locals @@ -39,7 +40,8 @@ class MyNotifee(INotifee): pass async def listen(self, network, multiaddr): - pass + self.events.append(["listened" + self.val_to_append_to_event,\ + multiaddr]) async def listen_close(self, network, multiaddr): pass @@ -65,16 +67,6 @@ class InvalidNotifee(): async def listen(self): assert False -async def perform_two_host_set_up_custom_handler(handler): - transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] - (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) - - node_b.set_stream_handler("/echo/1.0.0", handler) - - # Associate the peer with local ip address (see default parameters of Libp2p()) - node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) - return node_a, node_b - @pytest.mark.asyncio async def test_one_notifier(): node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) @@ -146,6 +138,69 @@ async def test_one_notifier_on_two_nodes(): # Success, terminate pending tasks. await cleanup() +@pytest.mark.asyncio +async def test_one_notifier_on_two_nodes_with_listen(): + events_b = [] + + node_a_transport_opt = ["/ip4/127.0.0.1/tcp/0"] + node_a = await new_node(transport_opt=node_a_transport_opt) + await node_a.get_network().listen(multiaddr.Multiaddr(node_a_transport_opt[0])) + + # Set up node_b swarm to pass into host + node_b_transport_opt = ["/ip4/127.0.0.1/tcp/0"] + node_b_multiaddr = multiaddr.Multiaddr(node_b_transport_opt[0]) + node_b_swarm = initialize_default_swarm(transport_opt=node_b_transport_opt) + node_b = BasicHost(node_b_swarm) + + async def my_stream_handler(stream): + # Ensure the listened, connected and opened_stream events were hit in Notifee obj + # and that the stream passed into opened_stream matches the stream created on + # node_b + assert events_b == [ + ["listenedb", node_b_multiaddr], \ + ["connectedb", stream.mplex_conn], \ + ["opened_streamb", stream] + ] + while True: + read_string = (await stream.read()).decode() + + resp = "ack:" + read_string + await stream.write(resp.encode()) + + # Add notifee for node_a + events_a = [] + assert node_a.get_network().notify(MyNotifee(events_a, "a")) + + # Add notifee for node_b + assert node_b.get_network().notify(MyNotifee(events_b, "b")) + + # start listen on node_b_swarm + await node_b.get_network().listen(node_b_multiaddr) + + node_b.set_stream_handler("/echo/1.0.0", my_stream_handler) + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"]) + + # Ensure the connected and opened_stream events were hit in MyNotifee obj + # and that stream passed into opened_stream matches the stream created on + # node_a + assert events_a == [ + ["connecteda", stream.mplex_conn], \ + ["opened_streama", stream] + ] + + messages = ["hello", "hello"] + for message in messages: + await stream.write(message.encode()) + + response = (await stream.read()).decode() + + assert response == ("ack:" + message) + + # Success, terminate pending tasks. + await cleanup() + @pytest.mark.asyncio async def test_two_notifiers(): node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) diff --git a/tests/utils.py b/tests/utils.py index e7f9b85..28f217e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -29,3 +29,13 @@ async def echo_stream_handler(stream): resp = "ack:" + read_string await stream.write(resp.encode()) + +async def perform_two_host_set_up_custom_handler(handler): + transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) + + node_b.set_stream_handler("/echo/1.0.0", handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + return node_a, node_b \ No newline at end of file From fee905ace2e119eb2996ff2f2e25faa88aeafdb8 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 17 Mar 2019 21:30:56 -0400 Subject: [PATCH 092/131] fix linting issues --- libp2p/__init__.py | 2 +- tests/libp2p/test_libp2p.py | 3 +-- tests/libp2p/test_notify.py | 11 +++++++---- tests/protocol_muxer/test_protocol_muxer.py | 3 +-- tests/utils.py | 4 ++-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/libp2p/__init__.py b/libp2p/__init__.py index d83ec3c..62e014b 100644 --- a/libp2p/__init__.py +++ b/libp2p/__init__.py @@ -35,7 +35,7 @@ def initialize_default_swarm( :param peerstore_opt: optional peerstore :return: return a default swarm instance """ - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments, unused-argument if not id_opt: new_key = RSA.generate(2048, e=65537) id_opt = id_from_public_key(new_key.publickey()) diff --git a/tests/libp2p/test_libp2p.py b/tests/libp2p/test_libp2p.py index 3d74516..e9bfb83 100644 --- a/tests/libp2p/test_libp2p.py +++ b/tests/libp2p/test_libp2p.py @@ -2,7 +2,6 @@ import multiaddr import pytest from tests.utils import cleanup, set_up_nodes_by_transport_opt -from libp2p import new_node from libp2p.peer.peerinfo import info_from_p2p_addr @@ -10,7 +9,7 @@ from libp2p.peer.peerinfo import info_from_p2p_addr @pytest.mark.asyncio async def test_simple_messages(): transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] - (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler(stream): while True: diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index edaba2f..71b455b 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -10,8 +10,11 @@ features are implemented in swarm """ import pytest +import multiaddr -from tests.utils import * + +from tests.utils import cleanup, echo_stream_handler, \ + perform_two_host_set_up_custom_handler from libp2p import new_node, initialize_default_swarm from libp2p.network.notifee_interface import INotifee from libp2p.host.basic_host import BasicHost @@ -39,11 +42,11 @@ class MyNotifee(INotifee): async def disconnected(self, network, conn): pass - async def listen(self, network, multiaddr): + async def listen(self, network, _multiaddr): self.events.append(["listened" + self.val_to_append_to_event,\ - multiaddr]) + _multiaddr]) - async def listen_close(self, network, multiaddr): + async def listen_close(self, network, _multiaddr): pass class InvalidNotifee(): diff --git a/tests/protocol_muxer/test_protocol_muxer.py b/tests/protocol_muxer/test_protocol_muxer.py index 1eff211..00949fa 100644 --- a/tests/protocol_muxer/test_protocol_muxer.py +++ b/tests/protocol_muxer/test_protocol_muxer.py @@ -1,7 +1,6 @@ import pytest from tests.utils import cleanup, set_up_nodes_by_transport_opt -from libp2p import new_node from libp2p.protocol_muxer.multiselect_client import MultiselectClientError # TODO: Add tests for multiple streams being opened on different @@ -16,7 +15,7 @@ from libp2p.protocol_muxer.multiselect_client import MultiselectClientError async def perform_simple_test(expected_selected_protocol, protocols_for_client, protocols_with_handlers): transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] - (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) async def stream_handler(stream): while True: diff --git a/tests/utils.py b/tests/utils.py index 28f217e..4efde83 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -32,10 +32,10 @@ async def echo_stream_handler(stream): async def perform_two_host_set_up_custom_handler(handler): transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] - (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) + (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) node_b.set_stream_handler("/echo/1.0.0", handler) # Associate the peer with local ip address (see default parameters of Libp2p()) node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) - return node_a, node_b \ No newline at end of file + return node_a, node_b From 9c6d441f9fdf912c9d2df06d149650a86c8c53c7 Mon Sep 17 00:00:00 2001 From: stuckinaboot Date: Sat, 23 Mar 2019 13:52:02 -0400 Subject: [PATCH 093/131] [WIP] PubSub and FloodSub development (#133) * Add notifee interface * Add notify function to network interface * Implement notify feature * Add tests for notify * Make notifee functions all async * Fix linting issue * Fix linting issue * Scaffold pubsub router interface * Scaffold pubsub directory * Store peer_id in muxed connection * Implement pubsub notifee * Remove outdated files * Implement pubsub first attempt * Prepare pubsub for floodsub * Add mplex conn to net stream and add conn in notify tests * Implement floodsub * Use NetStream in generic protocol handler * Debugging async issues * Modify test to perform proper assert. Test passes * Remove callbacks. Reduce sleep time * Add simple three node test * Clean up code. Add message classes * Add test for two topics * Add conn to net stream and conn tests * Refactor test setup to remove duplicate code * Fix linting issues * Fix linting issue * Fix linting issue * Fix outstanding unrelated lint issue in multiselect_client * Add connect function * Remove debug prints * Remove debug prints from floodsub * Use MessageTalk in place of direct message breakdown * Remove extra prints * Remove outdated function * Add message to queues for all topics in message * Debugging * Add message self delivery * Increase read timeout to 5 to get pubsub tests passing * Refactor testing helper func. Add tests * Add tests and increase timeout to get tests passing * Add dummy account demo scaffolding * Attempt to use threads. Test fails * Implement basic dummy node tests using threads * Add generic testing function * Add simple seven node tree test * Add more complex seven node tree tests * Add five node ring tests * Remove unnecessary get_message_type func * Add documentation to classes * Add message id to messages * Add documentation to test helper func * Add docs to dummy account node helper func * Add more docs to dummy account node test helper func * fixed linting errors in floodsub * small notify bugfix * move pubsub into libp2p * fixed pubsub linting * fixing pubsub test failures * linting --- libp2p/network/swarm.py | 14 +- libp2p/pubsub/__init__.py | 0 libp2p/pubsub/floodsub.py | 98 ++++ libp2p/pubsub/message.py | 118 +++++ libp2p/pubsub/pubsub.py | 294 +++++++++++ libp2p/pubsub/pubsub_notifee.py | 40 ++ libp2p/pubsub/pubsub_router_interface.py | 64 +++ libp2p/stream_muxer/mplex/mplex.py | 10 +- .../muxed_connection_interface.py | 3 +- libp2p/transport/upgrader.py | 4 +- tests/libp2p/test_notify.py | 31 ++ tests/pubsub/dummy_account_node.py | 134 +++++ tests/pubsub/test_dummyaccount_demo.py | 189 +++++++ tests/pubsub/test_floodsub.py | 486 ++++++++++++++++++ 14 files changed, 1474 insertions(+), 11 deletions(-) create mode 100644 libp2p/pubsub/__init__.py create mode 100644 libp2p/pubsub/floodsub.py create mode 100644 libp2p/pubsub/message.py create mode 100644 libp2p/pubsub/pubsub.py create mode 100644 libp2p/pubsub/pubsub_notifee.py create mode 100644 libp2p/pubsub/pubsub_router_interface.py create mode 100644 tests/pubsub/dummy_account_node.py create mode 100644 tests/pubsub/test_dummyaccount_demo.py create mode 100644 tests/pubsub/test_floodsub.py diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index f2943ba..b30567b 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -70,7 +70,8 @@ class Swarm(INetwork): raw_conn = await self.transport.dial(multiaddr, self.self_id) # Use upgrader to upgrade raw conn to muxed conn - muxed_conn = self.upgrader.upgrade_connection(raw_conn, self.generic_protocol_handler) + muxed_conn = self.upgrader.upgrade_connection(raw_conn, \ + self.generic_protocol_handler, peer_id) # Store muxed connection in connections self.connections[peer_id] = muxed_conn @@ -145,7 +146,7 @@ class Swarm(INetwork): raw_conn = RawConnection(multiaddr.value_for_protocol('ip4'), multiaddr.value_for_protocol('tcp'), reader, writer, False) muxed_conn = self.upgrader.upgrade_connection(raw_conn, \ - self.generic_protocol_handler) + self.generic_protocol_handler, peer_id) # Store muxed_conn with peer id self.connections[peer_id] = muxed_conn @@ -197,14 +198,17 @@ def create_generic_protocol_handler(swarm): async def generic_protocol_handler(muxed_stream): # Perform protocol muxing to determine protocol to use - _, handler = await multiselect.negotiate(muxed_stream) + protocol, handler = await multiselect.negotiate(muxed_stream) + + net_stream = NetStream(muxed_stream) + net_stream.set_protocol(protocol) # Call notifiers since event occurred for notifee in swarm.notifees: - await notifee.opened_stream(swarm, muxed_stream) + await notifee.opened_stream(swarm, net_stream) # Give to stream handler - asyncio.ensure_future(handler(muxed_stream)) + asyncio.ensure_future(handler(net_stream)) return generic_protocol_handler diff --git a/libp2p/pubsub/__init__.py b/libp2p/pubsub/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py new file mode 100644 index 0000000..68e542c --- /dev/null +++ b/libp2p/pubsub/floodsub.py @@ -0,0 +1,98 @@ +from .pubsub_router_interface import IPubsubRouter +from .message import create_message_talk + +class FloodSub(IPubsubRouter): + + def __init__(self, protocols): + self.protocols = protocols + self.pubsub = None + + def get_protocols(self): + """ + :return: the list of protocols supported by the router + """ + return self.protocols + + def attach(self, pubsub): + """ + Attach is invoked by the PubSub constructor to attach the router to a + freshly initialized PubSub instance. + :param pubsub: pubsub instance to attach to + """ + self.pubsub = pubsub + + def add_peer(self, peer_id, protocol_id): + """ + Notifies the router that a new peer has been connected + :param peer_id: id of peer to add + """ + + def remove_peer(self, peer_id): + """ + Notifies the router that a peer has been disconnected + :param peer_id: id of peer to remove + """ + + def handle_rpc(self, rpc): + """ + Invoked to process control messages in the RPC envelope. + It is invoked after subscriptions and payload messages have been processed + :param rpc: rpc message + """ + + async def publish(self, sender_peer_id, message): + """ + Invoked to forward a new message that has been validated. + This is where the "flooding" part of floodsub happens + + With flooding, routing is almost trivial: for each incoming message, + forward to all known peers in the topic. There is a bit of logic, + as the router maintains a timed cache of previous messages, + so that seen messages are not further forwarded. + It also never forwards a message back to the source + or the peer that forwarded the message. + :param sender_peer_id: peer_id of message sender + :param message: message to forward + """ + + # Encode message + encoded_msg = message.encode() + + # Get message sender, origin, and topics + msg_talk = create_message_talk(message) + msg_sender = str(sender_peer_id) + msg_origin = msg_talk.origin_id + topics = msg_talk.topics + + # Deliver to self if self was origin + # Note: handle_talk checks if self is subscribed to topics in message + if msg_sender == msg_origin and msg_sender == str(self.pubsub.host.get_id()): + await self.pubsub.handle_talk(message) + + # Deliver to self and peers + for topic in topics: + if topic in self.pubsub.peer_topics: + for peer_id_in_topic in self.pubsub.peer_topics[topic]: + # Forward to all known peers in the topic that are not the + # message sender and are not the message origin + if peer_id_in_topic not in (msg_sender, msg_origin): + stream = self.pubsub.peers[peer_id_in_topic] + await stream.write(encoded_msg) + else: + # Implies publish did not write + print("publish did not write") + + def join(self, topic): + """ + Join notifies the router that we want to receive and + forward messages in a topic. It is invoked after the + subscription announcement + :param topic: topic to join + """ + + def leave(self, topic): + """ + Leave notifies the router that we are no longer interested in a topic. + It is invoked after the unsubscription announcement. + :param topic: topic to leave + """ diff --git a/libp2p/pubsub/message.py b/libp2p/pubsub/message.py new file mode 100644 index 0000000..2f839dc --- /dev/null +++ b/libp2p/pubsub/message.py @@ -0,0 +1,118 @@ +import uuid + + +class MessageTalk(): + + """ + Object to make parsing talk messages easier, where talk messages are + defined as custom messages published to a set of topics + """ + # pylint: disable=too-few-public-methods + def __init__(self, from_id, origin_id, topics, data, message_id): + # pylint: disable=too-many-arguments + self.msg_type = "talk" + self.from_id = from_id + self.origin_id = origin_id + self.topics = topics + self.data = data + self.message_id = message_id + + def to_str(self): + """ + Convert to string + :return: MessageTalk object in string representation + """ + out = self.msg_type + '\n' + out += self.from_id + '\n' + out += self.origin_id + '\n' + out += self.message_id + '\n' + for i in range(len(self.topics)): + out += self.topics[i] + if i < len(self.topics) - 1: + out += ',' + out += '\n' + self.data + return out + + +class MessageSub(): + """ + Object to make parsing subscription messages easier, where subscription + messages are defined as indicating the topics a node wishes to subscribe to + or unsubscribe from + """ + # pylint: disable=too-few-public-methods + def __init__(self, from_id, origin_id, subs_map, message_id): + self.msg_type = "subscription" + self.from_id = from_id + self.origin_id = origin_id + self.subs_map = subs_map + self.message_id = message_id + + def to_str(self): + """ + Convert to string + :return: MessageSub object in string representation + """ + out = self.msg_type + '\n' + out += self.from_id + '\n' + out += self.origin_id + '\n' + out += self.message_id + + if self.subs_map: + out += '\n' + + keys = list(self.subs_map) + + for i, topic in enumerate(keys): + sub = self.subs_map[topic] + if sub: + out += "sub:" + else: + out += "unsub:" + out += topic + if i < len(keys) - 1: + out += '\n' + + return out + +def create_message_talk(msg_talk_as_str): + """ + Create a MessageTalk object from a MessageTalk string representation + :param msg_talk_as_str: a MessageTalk object in its string representation + :return: MessageTalk object + """ + msg_comps = msg_talk_as_str.split('\n') + from_id = msg_comps[1] + origin_id = msg_comps[2] + message_id = msg_comps[3] + topics = msg_comps[4].split(',') + data = msg_comps[5] + return MessageTalk(from_id, origin_id, topics, data, message_id) + +def create_message_sub(msg_sub_as_str): + """ + Create a MessageSub object from a MessageSub string representation + :param msg_talk_as_str: a MessageSub object in its string representation + :return: MessageSub object + """ + msg_comps = msg_sub_as_str.split('\n') + from_id = msg_comps[1] + origin_id = msg_comps[2] + message_id = msg_comps[3] + + subs_map = {} + for i in range(4, len(msg_comps)): + sub_comps = msg_comps[i].split(":") + topic = sub_comps[1] + if sub_comps[0] == "sub": + subs_map[topic] = True + else: + subs_map[topic] = False + return MessageSub(from_id, origin_id, subs_map, message_id) + +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py new file mode 100644 index 0000000..9bd072f --- /dev/null +++ b/libp2p/pubsub/pubsub.py @@ -0,0 +1,294 @@ +import asyncio + +from .pubsub_notifee import PubsubNotifee +from .message import MessageSub +from .message import create_message_talk, create_message_sub +from. message import generate_message_id + + +class Pubsub(): + """ + For now, because I'm on a plane and don't have access to the go repo/protobuf stuff, + this is going to be the message format for the two types: subscription and talk + subscription indicates subscribing or unsubscribing from a topic + talk is sending a message on topic(s) + subscription format: + subscription + 'from' + :'topicid' + :'topicid' + ... + Ex. + subscription + msg_sender_peer_id + origin_peer_id + sub:topic1 + sub:topic2 + unsub:fav_topic + talk format: + talk + 'from' + 'origin' + [topic_ids comma-delimited] + 'data' + Ex. + talk + msg_sender_peer_id + origin_peer_id + topic1,topics_are_cool,foo + I like tacos + """ + # pylint: disable=too-many-instance-attributes + + def __init__(self, host, router, my_id): + """ + Construct a new Pubsub object, which is responsible for handling all + Pubsub-related messages and relaying messages as appropriate to the + Pubsub router (which is responsible for choosing who to send messages to). + Since the logic for choosing peers to send pubsub messages to is + in the router, the same Pubsub impl can back floodsub, gossipsub, etc. + """ + self.host = host + self.router = router + self.my_id = my_id + + # Attach this new Pubsub object to the router + self.router.attach(self) + + # Register a notifee + self.peer_queue = asyncio.Queue() + self.host.get_network().notify(PubsubNotifee(self.peer_queue)) + + # Register stream handlers for each pubsub router protocol to handle + # the pubsub streams opened on those protocols + self.protocols = self.router.get_protocols() + for protocol in self.protocols: + self.host.set_stream_handler(protocol, self.stream_handler) + + # TODO: determine if these need to be asyncio queues, or if could possibly + # be ordinary blocking queues + self.incoming_msgs_from_peers = asyncio.Queue() + self.outgoing_messages = asyncio.Queue() + + # TODO: Make seen_messages a cache (LRU cache?) + self.seen_messages = [] + + # Map of topics we are subscribed to to handler functions + # for when the given topic receives a message + self.my_topics = {} + + # Map of topic to peers to keep track of what peers are subscribed to + self.peer_topics = {} + + # Create peers map, which maps peer_id (as string) to stream (to a given peer) + self.peers = {} + + # Call handle peer to keep waiting for updates to peer queue + asyncio.ensure_future(self.handle_peer_queue()) + + def get_hello_packet(self): + """ + Generate subscription message with all topics we are subscribed to + """ + subs_map = {} + for topic in self.my_topics: + subs_map[topic] = True + sub_msg = MessageSub( + str(self.host.get_id()),\ + str(self.host.get_id()), subs_map, generate_message_id()\ + ) + return sub_msg.to_str() + + async def continously_read_stream(self, stream): + """ + Read from input stream in an infinite loop. Process + messages from other nodes, which for now are considered MessageTalk + and MessageSub messages. + TODO: Handle RPC messages instead of my Aspyn's own custom message format + :param stream: stream to continously read from + """ + while True: + incoming = (await stream.read()).decode() + msg_comps = incoming.split('\n') + msg_type = msg_comps[0] + + msg_sender = msg_comps[1] + # msg_origin = msg_comps[2] + msg_id = msg_comps[3] + print("HIT ME1") + if msg_id not in self.seen_messages: + print("HIT ME") + # Do stuff with incoming unseen message + should_publish = True + if msg_type == "subscription": + self.handle_subscription(incoming) + + # We don't need to relay the subscription to our + # peers because a given node only needs its peers + # to know that it is subscribed to the topic (doesn't + # need everyone to know) + should_publish = False + elif msg_type == "talk": + await self.handle_talk(incoming) + + # Add message id to seen + self.seen_messages.append(msg_id) + + # Publish message using router's publish + if should_publish: + msg = create_message_talk(incoming) + + # Adjust raw_msg to that the message sender + # is now our peer_id + msg.from_id = str(self.host.get_id()) + + await self.router.publish(msg_sender, msg.to_str()) + + # Force context switch + await asyncio.sleep(0) + + async def stream_handler(self, stream): + """ + Stream handler for pubsub. Gets invoked whenever a new stream is created + on one of the supported pubsub protocols. + :param stream: newly created stream + """ + # Add peer + # Map peer to stream + peer_id = stream.mplex_conn.peer_id + self.peers[str(peer_id)] = stream + self.router.add_peer(peer_id, stream.get_protocol()) + + # Send hello packet + hello = self.get_hello_packet() + await stream.write(hello.encode()) + # Pass stream off to stream reader + asyncio.ensure_future(self.continously_read_stream(stream)) + + async def handle_peer_queue(self): + """ + Continuously read from peer queue and each time a new peer is found, + open a stream to the peer using a supported pubsub protocol + TODO: Handle failure for when the peer does not support any of the + pubsub protocols we support + """ + while True: + peer_id = await self.peer_queue.get() + + # Open a stream to peer on existing connection + # (we know connection exists since that's the only way + # an element gets added to peer_queue) + stream = await self.host.new_stream(peer_id, self.protocols) + + # Add Peer + # Map peer to stream + self.peers[str(peer_id)] = stream + self.router.add_peer(peer_id, stream.get_protocol()) + + # Send hello packet + hello = self.get_hello_packet() + await stream.write(hello.encode()) + + # Pass stream off to stream reader + asyncio.ensure_future(self.continously_read_stream(stream)) + + # Force context switch + await asyncio.sleep(0) + + def handle_subscription(self, subscription): + """ + Handle an incoming subscription message from a peer. Update internal + mapping to mark the peer as subscribed or unsubscribed to topics as + defined in the subscription message + :param subscription: raw data constituting a subscription message + """ + sub_msg = create_message_sub(subscription) + if sub_msg.subs_map: + print("handle_subscription my_id: " + self.my_id + ", subber: " + sub_msg.origin_id) + for topic_id in sub_msg.subs_map: + # Look at each subscription in the msg individually + if sub_msg.subs_map[topic_id]: + if topic_id not in self.peer_topics: + # Create topic list if it did not yet exist + self.peer_topics[topic_id] = [sub_msg.origin_id] + elif sub_msg.origin_id not in self.peer_topics[topic_id]: + # Add peer to topic + self.peer_topics[topic_id].append(sub_msg.origin_id) + else: + # TODO: Remove peer from topic + pass + + async def handle_talk(self, talk): + """ + Handle incoming Talk message from a peer. A Talk message contains some + custom message that is published on a given topic(s) + :param talk: raw data constituting a talk message + """ + msg = create_message_talk(talk) + + # Check if this message has any topics that we are subscribed to + for topic in msg.topics: + if topic in self.my_topics: + # we are subscribed to a topic this message was sent for, + # so add message to the subscription output queue + # for each topic + await self.my_topics[topic].put(talk) + + async def subscribe(self, topic_id): + """ + Subscribe ourself to a topic + :param topic_id: topic_id to subscribe to + """ + # Map topic_id to blocking queue + self.my_topics[topic_id] = asyncio.Queue() + + # Create subscribe message + sub_msg = MessageSub( + str(self.host.get_id()),\ + str(self.host.get_id()), {topic_id: True}, generate_message_id()\ + ) + + # Send out subscribe message to all peers + await self.message_all_peers(sub_msg.to_str()) + + # Tell router we are joining this topic + self.router.join(topic_id) + + # Return the asyncio queue for messages on this topic + return self.my_topics[topic_id] + + async def unsubscribe(self, topic_id): + """ + Unsubscribe ourself from a topic + :param topic_id: topic_id to unsubscribe from + """ + + # Remove topic_id from map if present + if topic_id in self.my_topics: + del self.my_topics[topic_id] + + # Create unsubscribe message + unsub_msg = MessageSub(str(self.host.get_id()), str(self.host.get_id()),\ + {topic_id: False}, generate_message_id()) + + # Send out unsubscribe message to all peers + await self.message_all_peers(unsub_msg.to_str()) + + # Tell router we are leaving this topic + self.router.leave(topic_id) + + async def message_all_peers(self, raw_msg): + """ + Broadcast a message to peers + :param raw_msg: raw contents of the message to broadcast + """ + + # Encode message for sending + encoded_msg = raw_msg.encode() + + # Broadcast message + for peer in self.peers: + stream = self.peers[peer] + + # Write message to stream + await stream.write(encoded_msg) diff --git a/libp2p/pubsub/pubsub_notifee.py b/libp2p/pubsub/pubsub_notifee.py new file mode 100644 index 0000000..4173bd8 --- /dev/null +++ b/libp2p/pubsub/pubsub_notifee.py @@ -0,0 +1,40 @@ +from libp2p.network.notifee_interface import INotifee + + +class PubsubNotifee(INotifee): + # pylint: disable=too-many-instance-attributes, cell-var-from-loop + + def __init__(self, initiator_peers_queue): + """ + :param initiator_peers_queue: queue to add new peers to so that pubsub + can process new peers after we connect to them + """ + self.initiator_peers_queue = initiator_peers_queue + + async def opened_stream(self, network, stream): + pass + + async def closed_stream(self, network, stream): + pass + + async def connected(self, network, conn): + """ + Add peer_id to initiator_peers_queue, so that this peer_id can be used to + create a stream and we only want to have one pubsub stream with each peer. + :param network: network the connection was opened on + :param conn: connection that was opened + """ + + # Only add peer_id if we are initiator (otherwise we would end up + # with two pubsub streams between us and the peer) + if conn.initiator: + await self.initiator_peers_queue.put(conn.peer_id) + + async def disconnected(self, network, conn): + pass + + async def listen(self, network, multiaddr): + pass + + async def listen_close(self, network, multiaddr): + pass diff --git a/libp2p/pubsub/pubsub_router_interface.py b/libp2p/pubsub/pubsub_router_interface.py new file mode 100644 index 0000000..727b39e --- /dev/null +++ b/libp2p/pubsub/pubsub_router_interface.py @@ -0,0 +1,64 @@ +from abc import ABC, abstractmethod + +class IPubsubRouter(ABC): + + @abstractmethod + def get_protocols(self): + """ + :return: the list of protocols supported by the router + """ + + @abstractmethod + def attach(self, pubsub): + """ + Attach is invoked by the PubSub constructor to attach the router to a + freshly initialized PubSub instance. + :param pubsub: pubsub instance to attach to + """ + + @abstractmethod + def add_peer(self, peer_id, protocol_id): + """ + Notifies the router that a new peer has been connected + :param peer_id: id of peer to add + """ + + @abstractmethod + def remove_peer(self, peer_id): + """ + Notifies the router that a peer has been disconnected + :param peer_id: id of peer to remove + """ + + @abstractmethod + def handle_rpc(self, rpc): + """ + Invoked to process control messages in the RPC envelope. + It is invoked after subscriptions and payload messages have been processed + :param rpc: rpc message + """ + + @abstractmethod + def publish(self, sender_peer_id, message): + """ + Invoked to forward a new message that has been validated + :param sender_peer_id: peer_id of message sender + :param message: message to forward + """ + + @abstractmethod + def join(self, topic): + """ + Join notifies the router that we want to receive and + forward messages in a topic. It is invoked after the + subscription announcement + :param topic: topic to join + """ + + @abstractmethod + def leave(self, topic): + """ + Leave notifies the router that we are no longer interested in a topic. + It is invoked after the unsubscription announcement. + :param topic: topic to leave + """ diff --git a/libp2p/stream_muxer/mplex/mplex.py b/libp2p/stream_muxer/mplex/mplex.py index 135ee1e..0d587b5 100644 --- a/libp2p/stream_muxer/mplex/mplex.py +++ b/libp2p/stream_muxer/mplex/mplex.py @@ -11,14 +11,15 @@ class Mplex(IMuxedConn): reference: https://github.com/libp2p/go-mplex/blob/master/multiplex.go """ - def __init__(self, conn, generic_protocol_handler): + def __init__(self, conn, generic_protocol_handler, peer_id): """ create a new muxed connection :param conn: an instance of raw connection :param generic_protocol_handler: generic protocol handler for new muxed streams + :param peer_id: peer_id of peer the connection is to """ - super(Mplex, self).__init__(conn, generic_protocol_handler) + super(Mplex, self).__init__(conn, generic_protocol_handler, peer_id) self.raw_conn = conn self.initiator = conn.initiator @@ -26,6 +27,9 @@ class Mplex(IMuxedConn): # Store generic protocol handler self.generic_protocol_handler = generic_protocol_handler + # Set peer_id + self.peer_id = peer_id + # Mapping from stream ID -> buffer of messages for that stream self.buffers = {} @@ -56,7 +60,7 @@ class Mplex(IMuxedConn): # TODO: pass down timeout from user and use that if stream_id in self.buffers: try: - data = await asyncio.wait_for(self.buffers[stream_id].get(), timeout=3) + data = await asyncio.wait_for(self.buffers[stream_id].get(), timeout=8) return data except asyncio.TimeoutError: return None diff --git a/libp2p/stream_muxer/muxed_connection_interface.py b/libp2p/stream_muxer/muxed_connection_interface.py index 4017755..541fd64 100644 --- a/libp2p/stream_muxer/muxed_connection_interface.py +++ b/libp2p/stream_muxer/muxed_connection_interface.py @@ -7,12 +7,13 @@ class IMuxedConn(ABC): """ @abstractmethod - def __init__(self, conn, generic_protocol_handler): + def __init__(self, conn, generic_protocol_handler, peer_id): """ create a new muxed connection :param conn: an instance of raw connection :param generic_protocol_handler: generic protocol handler for new muxed streams + :param peer_id: peer_id of peer the connection is to """ @abstractmethod diff --git a/libp2p/transport/upgrader.py b/libp2p/transport/upgrader.py index 5f193a0..9e311e3 100644 --- a/libp2p/transport/upgrader.py +++ b/libp2p/transport/upgrader.py @@ -17,11 +17,11 @@ class TransportUpgrader: def upgrade_security(self): pass - def upgrade_connection(self, conn, generic_protocol_handler): + def upgrade_connection(self, conn, generic_protocol_handler, peer_id): """ upgrade raw connection to muxed connection """ # For PoC, no security, default to mplex # TODO do exchange to determine multiplexer - return Mplex(conn, generic_protocol_handler) + return Mplex(conn, generic_protocol_handler, peer_id) diff --git a/tests/libp2p/test_notify.py b/tests/libp2p/test_notify.py index 71b455b..570ad57 100644 --- a/tests/libp2p/test_notify.py +++ b/tests/libp2p/test_notify.py @@ -49,6 +49,7 @@ class MyNotifee(INotifee): async def listen_close(self, network, _multiaddr): pass + class InvalidNotifee(): # pylint: disable=too-many-instance-attributes, cell-var-from-loop @@ -70,6 +71,36 @@ class InvalidNotifee(): async def listen(self): assert False + +async def perform_two_host_simple_set_up(): + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + async def my_stream_handler(stream): + while True: + read_string = (await stream.read()).decode() + + resp = "ack:" + read_string + await stream.write(resp.encode()) + + node_b.set_stream_handler("/echo/1.0.0", my_stream_handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + return node_a, node_b + + +async def perform_two_host_simple_set_up_custom_handler(handler): + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + node_b.set_stream_handler("/echo/1.0.0", handler) + + # Associate the peer with local ip address (see default parameters of Libp2p()) + node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) + return node_a, node_b + + @pytest.mark.asyncio async def test_one_notifier(): node_a, node_b = await perform_two_host_set_up_custom_handler(echo_stream_handler) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py new file mode 100644 index 0000000..f328a6e --- /dev/null +++ b/tests/pubsub/dummy_account_node.py @@ -0,0 +1,134 @@ +import asyncio +import multiaddr + +from libp2p import new_node +from libp2p.pubsub.message import create_message_talk +from libp2p.pubsub.pubsub import Pubsub +from libp2p.pubsub.floodsub import FloodSub +from libp2p.pubsub.message import MessageTalk +from libp2p.pubsub.message import generate_message_id + +SUPPORTED_PUBSUB_PROTOCOLS = ["/floodsub/1.0.0"] +CRYPTO_TOPIC = "ethereum" + +# Message format: +# Sending crypto: ,, +# Ex. send,aspyn,alex,5 +# Set crypto: , +# Ex. set,rob,5 +# Determine message type by looking at first item before first comma + +class DummyAccountNode(): + """ + Node which has an internal balance mapping, meant to serve as + a dummy crypto blockchain. There is no actual blockchain, just a simple + map indicating how much crypto each user in the mappings holds + """ + + def __init__(self): + self.balances = {} + + @classmethod + async def create(cls): + """ + Create a new DummyAccountNode and attach a libp2p node, a floodsub, and a pubsub + instance to this new node + + We use create as this serves as a factory function and allows us + to use async await, unlike the init function + """ + self = DummyAccountNode() + + libp2p_node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + await libp2p_node.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + + self.libp2p_node = libp2p_node + + self.floodsub = FloodSub(SUPPORTED_PUBSUB_PROTOCOLS) + self.pubsub = Pubsub(self.libp2p_node, self.floodsub, "a") + return self + + async def handle_incoming_msgs(self): + """ + Handle all incoming messages on the CRYPTO_TOPIC from peers + """ + while True: + message_raw = await self.q.get() + message = create_message_talk(message_raw) + contents = message.data + + msg_comps = contents.split(",") + + if msg_comps[0] == "send": + self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) + elif msg_comps[0] == "set": + self.handle_set_crypto_for_user(msg_comps[1], int(msg_comps[2])) + + async def setup_crypto_networking(self): + """ + Subscribe to CRYPTO_TOPIC and perform call to function that handles + all incoming messages on said topic + """ + self.q = await self.pubsub.subscribe(CRYPTO_TOPIC) + + asyncio.ensure_future(self.handle_incoming_msgs()) + + async def publish_send_crypto(self, source_user, dest_user, amount): + """ + Create a send crypto message and publish that message to all other nodes + :param source_user: user to send crypto from + :param dest_user: user to send crypto to + :param amount: amount of crypto to send + """ + my_id = str(self.libp2p_node.get_id()) + msg_contents = "send," + source_user + "," + dest_user + "," + str(amount) + msg = MessageTalk(my_id, my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + await self.floodsub.publish(my_id, msg.to_str()) + + async def publish_set_crypto(self, user, amount): + """ + Create a set crypto message and publish that message to all other nodes + :param user: user to set crypto for + :param amount: amount of crypto + """ + my_id = str(self.libp2p_node.get_id()) + msg_contents = "set," + user + "," + str(amount) + msg = MessageTalk(my_id, my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + await self.floodsub.publish(my_id, msg.to_str()) + + def handle_send_crypto(self, source_user, dest_user, amount): + """ + Handle incoming send_crypto message + :param source_user: user to send crypto from + :param dest_user: user to send crypto to + :param amount: amount of crypto to send + """ + if source_user in self.balances: + self.balances[source_user] -= amount + else: + self.balances[source_user] = -amount + + if dest_user in self.balances: + self.balances[dest_user] += amount + else: + self.balances[dest_user] = amount + + def handle_set_crypto_for_user(self, dest_user, amount): + """ + Handle incoming set_crypto message + :param dest_user: user to set crypto for + :param amount: amount of crypto + """ + self.balances[dest_user] = amount + + def get_balance(self, user): + """ + Get balance in crypto for a particular user + :param user: user to get balance for + :return: balance of user + """ + if user in self.balances: + return self.balances[user] + else: + return -1 + diff --git a/tests/pubsub/test_dummyaccount_demo.py b/tests/pubsub/test_dummyaccount_demo.py new file mode 100644 index 0000000..a071fa6 --- /dev/null +++ b/tests/pubsub/test_dummyaccount_demo.py @@ -0,0 +1,189 @@ +import asyncio +import multiaddr +import pytest + +from threading import Thread +from tests.utils import cleanup +from libp2p import new_node +from libp2p.peer.peerinfo import info_from_p2p_addr +from libp2p.pubsub.pubsub import Pubsub +from libp2p.pubsub.floodsub import FloodSub +from libp2p.pubsub.message import MessageTalk +from libp2p.pubsub.message import create_message_talk +from dummy_account_node import DummyAccountNode + +# pylint: disable=too-many-locals + +async def connect(node1, node2): + # node1 connects to node2 + addr = node2.get_addrs()[0] + info = info_from_p2p_addr(addr) + await node1.connect(info) + +def create_setup_in_new_thread_func(dummy_node): + def setup_in_new_thread(): + asyncio.ensure_future(dummy_node.setup_crypto_networking()) + return setup_in_new_thread + +async def perform_test(num_nodes, adjacency_map, action_func, assertion_func): + """ + Helper function to allow for easy construction of custom tests for dummy account nodes + in various network topologies + :param num_nodes: number of nodes in the test + :param adjacency_map: adjacency map defining each node and its list of neighbors + :param action_func: function to execute that includes actions by the nodes, + such as send crypto and set crypto + :param assertion_func: assertions for testing the results of the actions are correct + """ + + # Create nodes + dummy_nodes = [] + for i in range(num_nodes): + dummy_nodes.append(await DummyAccountNode.create()) + + # Create network + for source_num in adjacency_map: + target_nums = adjacency_map[source_num] + for target_num in target_nums: + await connect(dummy_nodes[source_num].libp2p_node, \ + dummy_nodes[target_num].libp2p_node) + + # Allow time for network creation to take place + await asyncio.sleep(0.25) + + # Start a thread for each node so that each node can listen and respond + # to messages on its own thread, which will avoid waiting indefinitely + # on the main thread. On this thread, call the setup func for the node, + # which subscribes the node to the CRYPTO_TOPIC topic + for dummy_node in dummy_nodes: + thread = Thread(target=create_setup_in_new_thread_func(dummy_node)) + thread.run() + + # Allow time for nodes to subscribe to CRYPTO_TOPIC topic + await asyncio.sleep(0.25) + + # Perform action function + await action_func(dummy_nodes) + + # Allow time for action function to be performed (i.e. messages to propogate) + await asyncio.sleep(0.25) + + # Perform assertion function + for dummy_node in dummy_nodes: + assertion_func(dummy_node) + + # Success, terminate pending tasks. + await cleanup() + +@pytest.mark.asyncio +async def test_simple_two_nodes(): + num_nodes = 2 + adj_map = {0: [1]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 10) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 10 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_simple_three_nodes_line_topography(): + num_nodes = 3 + adj_map = {0: [1], 1: [2]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 10) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 10 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_simple_three_nodes_triangle_topography(): + num_nodes = 3 + adj_map = {0: [1, 2], 1: [2]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_simple_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_set_then_send_from_root_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) + await asyncio.sleep(0.25) + await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 15 + assert dummy_node.get_balance("alex") == 5 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} + + async def action_func(dummy_nodes): + await dummy_nodes[6].publish_set_crypto("aspyn", 20) + await asyncio.sleep(0.25) + await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 15 + assert dummy_node.get_balance("alex") == 5 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_simple_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("alex", 20) + await asyncio.sleep(0.25) + await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("alex") == 8 + assert dummy_node.get_balance("rob") == 12 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py new file mode 100644 index 0000000..f4a5826 --- /dev/null +++ b/tests/pubsub/test_floodsub.py @@ -0,0 +1,486 @@ +import asyncio +import multiaddr +import pytest + +from tests.utils import cleanup +from libp2p import new_node +from libp2p.peer.peerinfo import info_from_p2p_addr +from libp2p.pubsub.pubsub import Pubsub +from libp2p.pubsub.floodsub import FloodSub +from libp2p.pubsub.message import MessageTalk +from libp2p.pubsub.message import create_message_talk +from libp2p.pubsub.message import generate_message_id + +# pylint: disable=too-many-locals + +async def connect(node1, node2): + """ + Connect node1 to node2 + """ + addr = node2.get_addrs()[0] + info = info_from_p2p_addr(addr) + await node1.connect(info) + +@pytest.mark.asyncio +async def test_simple_two_nodes(): + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + + supported_protocols = ["/floodsub/1.0.0"] + + floodsub_a = FloodSub(supported_protocols) + pubsub_a = Pubsub(node_a, floodsub_a, "a") + floodsub_b = FloodSub(supported_protocols) + pubsub_b = Pubsub(node_b, floodsub_b, "b") + + await connect(node_a, node_b) + + await asyncio.sleep(0.25) + qb = await pubsub_b.subscribe("my_topic") + + await asyncio.sleep(0.25) + + node_a_id = str(node_a.get_id()) + + msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) + + await floodsub_a.publish(node_a.get_id(), msg.to_str()) + + await asyncio.sleep(0.25) + + res_b = await qb.get() + + # Check that the msg received by node_b is the same + # as the message sent by node_a + assert res_b == msg.to_str() + + # Success, terminate pending tasks. + await cleanup() + +@pytest.mark.asyncio +async def test_simple_three_nodes(): + # Want to pass message from A -> B -> C + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_c = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + await node_c.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + + supported_protocols = ["/floodsub/1.0.0"] + + floodsub_a = FloodSub(supported_protocols) + pubsub_a = Pubsub(node_a, floodsub_a, "a") + floodsub_b = FloodSub(supported_protocols) + pubsub_b = Pubsub(node_b, floodsub_b, "b") + floodsub_c = FloodSub(supported_protocols) + pubsub_c = Pubsub(node_c, floodsub_c, "c") + + await connect(node_a, node_b) + await connect(node_b, node_c) + + await asyncio.sleep(0.25) + qb = await pubsub_b.subscribe("my_topic") + qc = await pubsub_c.subscribe("my_topic") + await asyncio.sleep(0.25) + + node_a_id = str(node_a.get_id()) + + msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) + + await floodsub_a.publish(node_a.get_id(), msg.to_str()) + + await asyncio.sleep(0.25) + res_b = await qb.get() + res_c = await qc.get() + + # Check that the msg received by node_b is the same + # as the message sent by node_a + assert res_b == msg.to_str() + + # res_c should match original msg but with b as sender + node_b_id = str(node_b.get_id()) + msg.from_id = node_b_id + + assert res_c == msg.to_str() + + # Success, terminate pending tasks. + await cleanup() + +async def perform_test_from_obj(obj): + """ + Perform a floodsub test from a test obj. + test obj are composed as follows: + + { + "supported_protocols": ["supported/protocol/1.0.0",...], + "adj_list": { + "node1": ["neighbor1_of_node1", "neighbor2_of_node1", ...], + "node2": ["neighbor1_of_node2", "neighbor2_of_node2", ...], + ... + }, + "topic_map": { + "topic1": ["node1_subscribed_to_topic1", "node2_subscribed_to_topic1", ...] + }, + "messages": [ + { + "topics": ["topic1_for_message", "topic2_for_message", ...], + "data": "some contents of the message (newlines are not supported)", + "node_id": "message sender node id" + }, + ... + ] + } + NOTE: In adj_list, for any neighbors A and B, only list B as a neighbor of A + or B as a neighbor of A once. Do NOT list both A: ["B"] and B:["A"] as the behavior + is undefined (even if it may work) + """ + + # Step 1) Create graph + adj_list = obj["adj_list"] + node_map = {} + floodsub_map = {} + pubsub_map = {} + + supported_protocols = obj["supported_protocols"] + + tasks_connect = [] + for start_node_id in adj_list: + # Create node if node does not yet exist + if start_node_id not in node_map: + node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + await node.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + + node_map[start_node_id] = node + + floodsub = FloodSub(supported_protocols) + floodsub_map[start_node_id] = floodsub + pubsub = Pubsub(node, floodsub, start_node_id) + pubsub_map[start_node_id] = pubsub + + # For each neighbor of start_node, create if does not yet exist, + # then connect start_node to neighbor + for neighbor_id in adj_list[start_node_id]: + # Create neighbor if neighbor does not yet exist + if neighbor_id not in node_map: + neighbor_node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + await neighbor_node.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + + node_map[neighbor_id] = neighbor_node + + floodsub = FloodSub(supported_protocols) + floodsub_map[neighbor_id] = floodsub + pubsub = Pubsub(neighbor_node, floodsub, neighbor_id) + pubsub_map[neighbor_id] = pubsub + + # Connect node and neighbor + # await connect(node_map[start_node_id], node_map[neighbor_id]) + tasks_connect.append(asyncio.ensure_future(connect(node_map[start_node_id], node_map[neighbor_id]))) + tasks_connect.append(asyncio.sleep(2)) + await asyncio.gather(*tasks_connect) + + # Allow time for graph creation before continuing + # await asyncio.sleep(0.25) + + # Step 2) Subscribe to topics + queues_map = {} + topic_map = obj["topic_map"] + + tasks_topic = [] + tasks_topic_data = [] + for topic in topic_map: + for node_id in topic_map[topic]: + """ + # Subscribe node to topic + q = await pubsub_map[node_id].subscribe(topic) + + # Create topic-queue map for node_id if one does not yet exist + if node_id not in queues_map: + queues_map[node_id] = {} + + # Store queue in topic-queue map for node + queues_map[node_id][topic] = q + """ + tasks_topic.append(asyncio.ensure_future(pubsub_map[node_id].subscribe(topic))) + tasks_topic_data.append((node_id, topic)) + tasks_topic.append(asyncio.sleep(2)) + + # Gather is like Promise.all + responses = await asyncio.gather(*tasks_topic, return_exceptions=True) + for i in range(len(responses) - 1): + q = responses[i] + node_id, topic = tasks_topic_data[i] + if node_id not in queues_map: + queues_map[node_id] = {} + + # Store queue in topic-queue map for node + queues_map[node_id][topic] = q + + # Allow time for subscribing before continuing + # await asyncio.sleep(0.01) + + # Step 3) Publish messages + topics_in_msgs_ordered = [] + messages = obj["messages"] + tasks_publish = [] + for msg in messages: + topics = msg["topics"] + + data = msg["data"] + node_id = msg["node_id"] + + # Get actual id for sender node (not the id from the test obj) + actual_node_id = str(node_map[node_id].get_id()) + + # Create correctly formatted message + msg_talk = MessageTalk(actual_node_id, actual_node_id, topics, data, generate_message_id()) + + # Publish message + # await floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()) + tasks_publish.append(asyncio.ensure_future(floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()))) + + # For each topic in topics, add topic, msg_talk tuple to ordered test list + # TODO: Update message sender to be correct message sender before + # adding msg_talk to this list + for topic in topics: + topics_in_msgs_ordered.append((topic, msg_talk)) + + # Allow time for publishing before continuing + # await asyncio.sleep(0.4) + tasks_publish.append(asyncio.sleep(2)) + await asyncio.gather(*tasks_publish) + + # Step 4) Check that all messages were received correctly. + # TODO: Check message sender too + for i in range(len(topics_in_msgs_ordered)): + topic, actual_msg = topics_in_msgs_ordered[i] + for node_id in topic_map[topic]: + # Get message from subscription queue + msg_on_node_str = await queues_map[node_id][topic].get() + msg_on_node = create_message_talk(msg_on_node_str) + + # Perform checks + assert actual_msg.origin_id == msg_on_node.origin_id + assert actual_msg.topics == msg_on_node.topics + assert actual_msg.data == msg_on_node.data + + # Success, terminate pending tasks. + await cleanup() + +@pytest.mark.asyncio +async def test_simple_two_nodes_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "A": ["B"] + }, + "topic_map": { + "topic1": ["B"] + }, + "messages": [ + { + "topics": ["topic1"], + "data": "foo", + "node_id": "A" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_three_nodes_two_topics_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "A": ["B"], + "B": ["C"] + }, + "topic_map": { + "topic1": ["B", "C"], + "topic2": ["B", "C"] + }, + "messages": [ + { + "topics": ["topic1"], + "data": "foo", + "node_id": "A" + }, + { + "topics": ["topic2"], + "data": "Alex is tall", + "node_id": "A" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_two_nodes_one_topic_single_subscriber_is_sender_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "A": ["B"] + }, + "topic_map": { + "topic1": ["B"] + }, + "messages": [ + { + "topics": ["topic1"], + "data": "Alex is tall", + "node_id": "B" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_two_nodes_one_topic_two_msgs_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "A": ["B"] + }, + "topic_map": { + "topic1": ["B"] + }, + "messages": [ + { + "topics": ["topic1"], + "data": "Alex is tall", + "node_id": "B" + }, + { + "topics": ["topic1"], + "data": "foo", + "node_id": "A" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_seven_nodes_tree_one_topics_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2", "3"], + "2": ["4", "5"], + "3": ["6", "7"] + }, + "topic_map": { + "astrophysics": ["2", "3", "4", "5", "6", "7"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_seven_nodes_tree_three_topics_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2", "3"], + "2": ["4", "5"], + "3": ["6", "7"] + }, + "topic_map": { + "astrophysics": ["2", "3", "4", "5", "6", "7"], + "space": ["2", "3", "4", "5", "6", "7"], + "onions": ["2", "3", "4", "5", "6", "7"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["space"], + "data": "foobar", + "node_id": "1" + }, + { + "topics": ["onions"], + "data": "I am allergic", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_seven_nodes_tree_three_topics_diff_origin_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2", "3"], + "2": ["4", "5"], + "3": ["6", "7"] + }, + "topic_map": { + "astrophysics": ["1", "2", "3", "4", "5", "6", "7"], + "space": ["1", "2", "3", "4", "5", "6", "7"], + "onions": ["1", "2", "3", "4", "5", "6", "7"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["space"], + "data": "foobar", + "node_id": "4" + }, + { + "topics": ["onions"], + "data": "I am allergic", + "node_id": "7" + } + ] + } + await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_three_nodes_clique_two_topic_diff_origin_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2", "3"], + "2": ["3"] + }, + "topic_map": { + "astrophysics": ["1", "2", "3"], + "school": ["1", "2", "3"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) From 0d7b8c3e5d435b1ae2babea86d5f1a8d51908cd1 Mon Sep 17 00:00:00 2001 From: Alex Haynes Date: Sun, 24 Mar 2019 15:08:36 -0400 Subject: [PATCH 094/131] added rpc.proto from go repo --- rpc.proto | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 rpc.proto diff --git a/rpc.proto b/rpc.proto new file mode 100644 index 0000000..2ae2ef2 --- /dev/null +++ b/rpc.proto @@ -0,0 +1,78 @@ +// Borrowed from https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto + +syntax = "proto2"; + +package pubsub.pb; + +message RPC { + repeated SubOpts subscriptions = 1; + repeated Message publish = 2; + + message SubOpts { + optional bool subscribe = 1; // subscribe or unsubcribe + optional string topicid = 2; + } + + optional ControlMessage control = 3; +} + +message Message { + optional bytes from = 1; + optional bytes data = 2; + optional bytes seqno = 3; + repeated string topicIDs = 4; + optional bytes signature = 5; + optional bytes key = 6; +} + +message ControlMessage { + repeated ControlIHave ihave = 1; + repeated ControlIWant iwant = 2; + repeated ControlGraft graft = 3; + repeated ControlPrune prune = 4; +} + +message ControlIHave { + optional string topicID = 1; + repeated string messageIDs = 2; +} + +message ControlIWant { + repeated string messageIDs = 1; +} + +message ControlGraft { + optional string topicID = 1; +} + +message ControlPrune { + optional string topicID = 1; +} + +message TopicDescriptor { + optional string name = 1; + optional AuthOpts auth = 2; + optional EncOpts enc = 3; + + message AuthOpts { + optional AuthMode mode = 1; + repeated bytes keys = 2; // root keys to trust + + enum AuthMode { + NONE = 0; // no authentication, anyone can publish + KEY = 1; // only messages signed by keys in the topic descriptor are accepted + WOT = 2; // web of trust, certificates can allow publisher set to grow + } + } + + message EncOpts { + optional EncMode mode = 1; + repeated bytes keyHashes = 2; // the hashes of the shared keys used (salted) + + enum EncMode { + NONE = 0; // no encryption, anyone can read + SHAREDKEY = 1; // messages are encrypted with shared key + WOT = 2; // web of trust, certificates can allow publisher set to grow + } + } +} \ No newline at end of file From f655148d9ce99d1ccd346bcd97c10cb68bcd65b0 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 09:41:38 -0400 Subject: [PATCH 095/131] add pubsub proto --- libp2p/pubsub/pb/rpc.proto | 76 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 libp2p/pubsub/pb/rpc.proto diff --git a/libp2p/pubsub/pb/rpc.proto b/libp2p/pubsub/pb/rpc.proto new file mode 100644 index 0000000..764b033 --- /dev/null +++ b/libp2p/pubsub/pb/rpc.proto @@ -0,0 +1,76 @@ +syntax = "proto2"; + +package pubsub.pb; + +message RPC { + repeated SubOpts subscriptions = 1; + repeated Message publish = 2; + + message SubOpts { + optional bool subscribe = 1; // subscribe or unsubcribe + optional string topicid = 2; + } + + optional ControlMessage control = 3; +} + +message Message { + optional bytes from = 1; + optional bytes data = 2; + optional bytes seqno = 3; + repeated string topicIDs = 4; + optional bytes signature = 5; + optional bytes key = 6; +} + +message ControlMessage { + repeated ControlIHave ihave = 1; + repeated ControlIWant iwant = 2; + repeated ControlGraft graft = 3; + repeated ControlPrune prune = 4; +} + +message ControlIHave { + optional string topicID = 1; + repeated string messageIDs = 2; +} + +message ControlIWant { + repeated string messageIDs = 1; +} + +message ControlGraft { + optional string topicID = 1; +} + +message ControlPrune { + optional string topicID = 1; +} + +message TopicDescriptor { + optional string name = 1; + optional AuthOpts auth = 2; + optional EncOpts enc = 3; + + message AuthOpts { + optional AuthMode mode = 1; + repeated bytes keys = 2; // root keys to trust + + enum AuthMode { + NONE = 0; // no authentication, anyone can publish + KEY = 1; // only messages signed by keys in the topic descriptor are accepted + WOT = 2; // web of trust, certificates can allow publisher set to grow + } + } + + message EncOpts { + optional EncMode mode = 1; + repeated bytes keyHashes = 2; // the hashes of the shared keys used (salted) + + enum EncMode { + NONE = 0; // no encryption, anyone can read + SHAREDKEY = 1; // messages are encrypted with shared key + WOT = 2; // web of trust, certificates can allow publisher set to grow + } + } +} \ No newline at end of file From 2aa24cbfe88171b181df14b218b16a3587882f58 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 09:55:14 -0400 Subject: [PATCH 096/131] add generated rpc code --- libp2p/pubsub/pb/rpc_pb2.py | 638 +++++++++++++++++++++++++++++++ libp2p/pubsub/pb/rpc_pb2_grpc.py | 3 + 2 files changed, 641 insertions(+) create mode 100644 libp2p/pubsub/pb/rpc_pb2.py create mode 100644 libp2p/pubsub/pb/rpc_pb2_grpc.py diff --git a/libp2p/pubsub/pb/rpc_pb2.py b/libp2p/pubsub/pb/rpc_pb2.py new file mode 100644 index 0000000..57035ed --- /dev/null +++ b/libp2p/pubsub/pb/rpc_pb2.py @@ -0,0 +1,638 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: rpc.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='rpc.proto', + package='pubsub.pb', + syntax='proto2', + serialized_options=None, + serialized_pb=_b('\n\trpc.proto\x12\tpubsub.pb\"\xb4\x01\n\x03RPC\x12-\n\rsubscriptions\x18\x01 \x03(\x0b\x32\x16.pubsub.pb.RPC.SubOpts\x12#\n\x07publish\x18\x02 \x03(\x0b\x32\x12.pubsub.pb.Message\x12*\n\x07\x63ontrol\x18\x03 \x01(\x0b\x32\x19.pubsub.pb.ControlMessage\x1a-\n\x07SubOpts\x12\x11\n\tsubscribe\x18\x01 \x01(\x08\x12\x0f\n\x07topicid\x18\x02 \x01(\t\"f\n\x07Message\x12\x0c\n\x04\x66rom\x18\x01 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\r\n\x05seqno\x18\x03 \x01(\x0c\x12\x10\n\x08topicIDs\x18\x04 \x03(\t\x12\x11\n\tsignature\x18\x05 \x01(\x0c\x12\x0b\n\x03key\x18\x06 \x01(\x0c\"\xb0\x01\n\x0e\x43ontrolMessage\x12&\n\x05ihave\x18\x01 \x03(\x0b\x32\x17.pubsub.pb.ControlIHave\x12&\n\x05iwant\x18\x02 \x03(\x0b\x32\x17.pubsub.pb.ControlIWant\x12&\n\x05graft\x18\x03 \x03(\x0b\x32\x17.pubsub.pb.ControlGraft\x12&\n\x05prune\x18\x04 \x03(\x0b\x32\x17.pubsub.pb.ControlPrune\"3\n\x0c\x43ontrolIHave\x12\x0f\n\x07topicID\x18\x01 \x01(\t\x12\x12\n\nmessageIDs\x18\x02 \x03(\t\"\"\n\x0c\x43ontrolIWant\x12\x12\n\nmessageIDs\x18\x01 \x03(\t\"\x1f\n\x0c\x43ontrolGraft\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x1f\n\x0c\x43ontrolPrune\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x87\x03\n\x0fTopicDescriptor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x31\n\x04\x61uth\x18\x02 \x01(\x0b\x32#.pubsub.pb.TopicDescriptor.AuthOpts\x12/\n\x03\x65nc\x18\x03 \x01(\x0b\x32\".pubsub.pb.TopicDescriptor.EncOpts\x1a|\n\x08\x41uthOpts\x12:\n\x04mode\x18\x01 \x01(\x0e\x32,.pubsub.pb.TopicDescriptor.AuthOpts.AuthMode\x12\x0c\n\x04keys\x18\x02 \x03(\x0c\"&\n\x08\x41uthMode\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03KEY\x10\x01\x12\x07\n\x03WOT\x10\x02\x1a\x83\x01\n\x07\x45ncOpts\x12\x38\n\x04mode\x18\x01 \x01(\x0e\x32*.pubsub.pb.TopicDescriptor.EncOpts.EncMode\x12\x11\n\tkeyHashes\x18\x02 \x03(\x0c\"+\n\x07\x45ncMode\x12\x08\n\x04NONE\x10\x00\x12\r\n\tSHAREDKEY\x10\x01\x12\x07\n\x03WOT\x10\x02') +) + + + +_TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE = _descriptor.EnumDescriptor( + name='AuthMode', + full_name='pubsub.pb.TopicDescriptor.AuthOpts.AuthMode', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='NONE', index=0, number=0, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='KEY', index=1, number=1, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='WOT', index=2, number=2, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=865, + serialized_end=903, +) +_sym_db.RegisterEnumDescriptor(_TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE) + +_TOPICDESCRIPTOR_ENCOPTS_ENCMODE = _descriptor.EnumDescriptor( + name='EncMode', + full_name='pubsub.pb.TopicDescriptor.EncOpts.EncMode', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='NONE', index=0, number=0, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='SHAREDKEY', index=1, number=1, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='WOT', index=2, number=2, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=994, + serialized_end=1037, +) +_sym_db.RegisterEnumDescriptor(_TOPICDESCRIPTOR_ENCOPTS_ENCMODE) + + +_RPC_SUBOPTS = _descriptor.Descriptor( + name='SubOpts', + full_name='pubsub.pb.RPC.SubOpts', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='subscribe', full_name='pubsub.pb.RPC.SubOpts.subscribe', index=0, + number=1, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='topicid', full_name='pubsub.pb.RPC.SubOpts.topicid', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=160, + serialized_end=205, +) + +_RPC = _descriptor.Descriptor( + name='RPC', + full_name='pubsub.pb.RPC', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='subscriptions', full_name='pubsub.pb.RPC.subscriptions', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='publish', full_name='pubsub.pb.RPC.publish', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='control', full_name='pubsub.pb.RPC.control', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_RPC_SUBOPTS, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=25, + serialized_end=205, +) + + +_MESSAGE = _descriptor.Descriptor( + name='Message', + full_name='pubsub.pb.Message', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='from', full_name='pubsub.pb.Message.from', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='data', full_name='pubsub.pb.Message.data', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='seqno', full_name='pubsub.pb.Message.seqno', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='topicIDs', full_name='pubsub.pb.Message.topicIDs', index=3, + number=4, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='signature', full_name='pubsub.pb.Message.signature', index=4, + number=5, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='key', full_name='pubsub.pb.Message.key', index=5, + number=6, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=207, + serialized_end=309, +) + + +_CONTROLMESSAGE = _descriptor.Descriptor( + name='ControlMessage', + full_name='pubsub.pb.ControlMessage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='ihave', full_name='pubsub.pb.ControlMessage.ihave', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='iwant', full_name='pubsub.pb.ControlMessage.iwant', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='graft', full_name='pubsub.pb.ControlMessage.graft', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='prune', full_name='pubsub.pb.ControlMessage.prune', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=312, + serialized_end=488, +) + + +_CONTROLIHAVE = _descriptor.Descriptor( + name='ControlIHave', + full_name='pubsub.pb.ControlIHave', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='topicID', full_name='pubsub.pb.ControlIHave.topicID', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='messageIDs', full_name='pubsub.pb.ControlIHave.messageIDs', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=490, + serialized_end=541, +) + + +_CONTROLIWANT = _descriptor.Descriptor( + name='ControlIWant', + full_name='pubsub.pb.ControlIWant', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='messageIDs', full_name='pubsub.pb.ControlIWant.messageIDs', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=543, + serialized_end=577, +) + + +_CONTROLGRAFT = _descriptor.Descriptor( + name='ControlGraft', + full_name='pubsub.pb.ControlGraft', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='topicID', full_name='pubsub.pb.ControlGraft.topicID', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=579, + serialized_end=610, +) + + +_CONTROLPRUNE = _descriptor.Descriptor( + name='ControlPrune', + full_name='pubsub.pb.ControlPrune', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='topicID', full_name='pubsub.pb.ControlPrune.topicID', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=612, + serialized_end=643, +) + + +_TOPICDESCRIPTOR_AUTHOPTS = _descriptor.Descriptor( + name='AuthOpts', + full_name='pubsub.pb.TopicDescriptor.AuthOpts', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='mode', full_name='pubsub.pb.TopicDescriptor.AuthOpts.mode', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='keys', full_name='pubsub.pb.TopicDescriptor.AuthOpts.keys', index=1, + number=2, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE, + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=779, + serialized_end=903, +) + +_TOPICDESCRIPTOR_ENCOPTS = _descriptor.Descriptor( + name='EncOpts', + full_name='pubsub.pb.TopicDescriptor.EncOpts', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='mode', full_name='pubsub.pb.TopicDescriptor.EncOpts.mode', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='keyHashes', full_name='pubsub.pb.TopicDescriptor.EncOpts.keyHashes', index=1, + number=2, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _TOPICDESCRIPTOR_ENCOPTS_ENCMODE, + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=906, + serialized_end=1037, +) + +_TOPICDESCRIPTOR = _descriptor.Descriptor( + name='TopicDescriptor', + full_name='pubsub.pb.TopicDescriptor', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='pubsub.pb.TopicDescriptor.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='auth', full_name='pubsub.pb.TopicDescriptor.auth', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='enc', full_name='pubsub.pb.TopicDescriptor.enc', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_TOPICDESCRIPTOR_AUTHOPTS, _TOPICDESCRIPTOR_ENCOPTS, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=646, + serialized_end=1037, +) + +_RPC_SUBOPTS.containing_type = _RPC +_RPC.fields_by_name['subscriptions'].message_type = _RPC_SUBOPTS +_RPC.fields_by_name['publish'].message_type = _MESSAGE +_RPC.fields_by_name['control'].message_type = _CONTROLMESSAGE +_CONTROLMESSAGE.fields_by_name['ihave'].message_type = _CONTROLIHAVE +_CONTROLMESSAGE.fields_by_name['iwant'].message_type = _CONTROLIWANT +_CONTROLMESSAGE.fields_by_name['graft'].message_type = _CONTROLGRAFT +_CONTROLMESSAGE.fields_by_name['prune'].message_type = _CONTROLPRUNE +_TOPICDESCRIPTOR_AUTHOPTS.fields_by_name['mode'].enum_type = _TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE +_TOPICDESCRIPTOR_AUTHOPTS.containing_type = _TOPICDESCRIPTOR +_TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE.containing_type = _TOPICDESCRIPTOR_AUTHOPTS +_TOPICDESCRIPTOR_ENCOPTS.fields_by_name['mode'].enum_type = _TOPICDESCRIPTOR_ENCOPTS_ENCMODE +_TOPICDESCRIPTOR_ENCOPTS.containing_type = _TOPICDESCRIPTOR +_TOPICDESCRIPTOR_ENCOPTS_ENCMODE.containing_type = _TOPICDESCRIPTOR_ENCOPTS +_TOPICDESCRIPTOR.fields_by_name['auth'].message_type = _TOPICDESCRIPTOR_AUTHOPTS +_TOPICDESCRIPTOR.fields_by_name['enc'].message_type = _TOPICDESCRIPTOR_ENCOPTS +DESCRIPTOR.message_types_by_name['RPC'] = _RPC +DESCRIPTOR.message_types_by_name['Message'] = _MESSAGE +DESCRIPTOR.message_types_by_name['ControlMessage'] = _CONTROLMESSAGE +DESCRIPTOR.message_types_by_name['ControlIHave'] = _CONTROLIHAVE +DESCRIPTOR.message_types_by_name['ControlIWant'] = _CONTROLIWANT +DESCRIPTOR.message_types_by_name['ControlGraft'] = _CONTROLGRAFT +DESCRIPTOR.message_types_by_name['ControlPrune'] = _CONTROLPRUNE +DESCRIPTOR.message_types_by_name['TopicDescriptor'] = _TOPICDESCRIPTOR +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +RPC = _reflection.GeneratedProtocolMessageType('RPC', (_message.Message,), dict( + + SubOpts = _reflection.GeneratedProtocolMessageType('SubOpts', (_message.Message,), dict( + DESCRIPTOR = _RPC_SUBOPTS, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.RPC.SubOpts) + )) + , + DESCRIPTOR = _RPC, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.RPC) + )) +_sym_db.RegisterMessage(RPC) +_sym_db.RegisterMessage(RPC.SubOpts) + +Message = _reflection.GeneratedProtocolMessageType('Message', (_message.Message,), dict( + DESCRIPTOR = _MESSAGE, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.Message) + )) +_sym_db.RegisterMessage(Message) + +ControlMessage = _reflection.GeneratedProtocolMessageType('ControlMessage', (_message.Message,), dict( + DESCRIPTOR = _CONTROLMESSAGE, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlMessage) + )) +_sym_db.RegisterMessage(ControlMessage) + +ControlIHave = _reflection.GeneratedProtocolMessageType('ControlIHave', (_message.Message,), dict( + DESCRIPTOR = _CONTROLIHAVE, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlIHave) + )) +_sym_db.RegisterMessage(ControlIHave) + +ControlIWant = _reflection.GeneratedProtocolMessageType('ControlIWant', (_message.Message,), dict( + DESCRIPTOR = _CONTROLIWANT, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlIWant) + )) +_sym_db.RegisterMessage(ControlIWant) + +ControlGraft = _reflection.GeneratedProtocolMessageType('ControlGraft', (_message.Message,), dict( + DESCRIPTOR = _CONTROLGRAFT, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlGraft) + )) +_sym_db.RegisterMessage(ControlGraft) + +ControlPrune = _reflection.GeneratedProtocolMessageType('ControlPrune', (_message.Message,), dict( + DESCRIPTOR = _CONTROLPRUNE, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.ControlPrune) + )) +_sym_db.RegisterMessage(ControlPrune) + +TopicDescriptor = _reflection.GeneratedProtocolMessageType('TopicDescriptor', (_message.Message,), dict( + + AuthOpts = _reflection.GeneratedProtocolMessageType('AuthOpts', (_message.Message,), dict( + DESCRIPTOR = _TOPICDESCRIPTOR_AUTHOPTS, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.TopicDescriptor.AuthOpts) + )) + , + + EncOpts = _reflection.GeneratedProtocolMessageType('EncOpts', (_message.Message,), dict( + DESCRIPTOR = _TOPICDESCRIPTOR_ENCOPTS, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.TopicDescriptor.EncOpts) + )) + , + DESCRIPTOR = _TOPICDESCRIPTOR, + __module__ = 'rpc_pb2' + # @@protoc_insertion_point(class_scope:pubsub.pb.TopicDescriptor) + )) +_sym_db.RegisterMessage(TopicDescriptor) +_sym_db.RegisterMessage(TopicDescriptor.AuthOpts) +_sym_db.RegisterMessage(TopicDescriptor.EncOpts) + + +# @@protoc_insertion_point(module_scope) diff --git a/libp2p/pubsub/pb/rpc_pb2_grpc.py b/libp2p/pubsub/pb/rpc_pb2_grpc.py new file mode 100644 index 0000000..a894352 --- /dev/null +++ b/libp2p/pubsub/pb/rpc_pb2_grpc.py @@ -0,0 +1,3 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + From f1a20f12dcd13b1832b2fd866b234a90fe1b4203 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 15:23:56 -0400 Subject: [PATCH 097/131] update from to from_id in proto --- libp2p/pubsub/pb/rpc.proto | 2 +- libp2p/pubsub/pb/rpc_pb2.py | 46 ++++++++++++++++++------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/libp2p/pubsub/pb/rpc.proto b/libp2p/pubsub/pb/rpc.proto index 764b033..f07da20 100644 --- a/libp2p/pubsub/pb/rpc.proto +++ b/libp2p/pubsub/pb/rpc.proto @@ -15,7 +15,7 @@ message RPC { } message Message { - optional bytes from = 1; + optional bytes from_id = 1; optional bytes data = 2; optional bytes seqno = 3; repeated string topicIDs = 4; diff --git a/libp2p/pubsub/pb/rpc_pb2.py b/libp2p/pubsub/pb/rpc_pb2.py index 57035ed..d315782 100644 --- a/libp2p/pubsub/pb/rpc_pb2.py +++ b/libp2p/pubsub/pb/rpc_pb2.py @@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( package='pubsub.pb', syntax='proto2', serialized_options=None, - serialized_pb=_b('\n\trpc.proto\x12\tpubsub.pb\"\xb4\x01\n\x03RPC\x12-\n\rsubscriptions\x18\x01 \x03(\x0b\x32\x16.pubsub.pb.RPC.SubOpts\x12#\n\x07publish\x18\x02 \x03(\x0b\x32\x12.pubsub.pb.Message\x12*\n\x07\x63ontrol\x18\x03 \x01(\x0b\x32\x19.pubsub.pb.ControlMessage\x1a-\n\x07SubOpts\x12\x11\n\tsubscribe\x18\x01 \x01(\x08\x12\x0f\n\x07topicid\x18\x02 \x01(\t\"f\n\x07Message\x12\x0c\n\x04\x66rom\x18\x01 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\r\n\x05seqno\x18\x03 \x01(\x0c\x12\x10\n\x08topicIDs\x18\x04 \x03(\t\x12\x11\n\tsignature\x18\x05 \x01(\x0c\x12\x0b\n\x03key\x18\x06 \x01(\x0c\"\xb0\x01\n\x0e\x43ontrolMessage\x12&\n\x05ihave\x18\x01 \x03(\x0b\x32\x17.pubsub.pb.ControlIHave\x12&\n\x05iwant\x18\x02 \x03(\x0b\x32\x17.pubsub.pb.ControlIWant\x12&\n\x05graft\x18\x03 \x03(\x0b\x32\x17.pubsub.pb.ControlGraft\x12&\n\x05prune\x18\x04 \x03(\x0b\x32\x17.pubsub.pb.ControlPrune\"3\n\x0c\x43ontrolIHave\x12\x0f\n\x07topicID\x18\x01 \x01(\t\x12\x12\n\nmessageIDs\x18\x02 \x03(\t\"\"\n\x0c\x43ontrolIWant\x12\x12\n\nmessageIDs\x18\x01 \x03(\t\"\x1f\n\x0c\x43ontrolGraft\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x1f\n\x0c\x43ontrolPrune\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x87\x03\n\x0fTopicDescriptor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x31\n\x04\x61uth\x18\x02 \x01(\x0b\x32#.pubsub.pb.TopicDescriptor.AuthOpts\x12/\n\x03\x65nc\x18\x03 \x01(\x0b\x32\".pubsub.pb.TopicDescriptor.EncOpts\x1a|\n\x08\x41uthOpts\x12:\n\x04mode\x18\x01 \x01(\x0e\x32,.pubsub.pb.TopicDescriptor.AuthOpts.AuthMode\x12\x0c\n\x04keys\x18\x02 \x03(\x0c\"&\n\x08\x41uthMode\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03KEY\x10\x01\x12\x07\n\x03WOT\x10\x02\x1a\x83\x01\n\x07\x45ncOpts\x12\x38\n\x04mode\x18\x01 \x01(\x0e\x32*.pubsub.pb.TopicDescriptor.EncOpts.EncMode\x12\x11\n\tkeyHashes\x18\x02 \x03(\x0c\"+\n\x07\x45ncMode\x12\x08\n\x04NONE\x10\x00\x12\r\n\tSHAREDKEY\x10\x01\x12\x07\n\x03WOT\x10\x02') + serialized_pb=_b('\n\trpc.proto\x12\tpubsub.pb\"\xb4\x01\n\x03RPC\x12-\n\rsubscriptions\x18\x01 \x03(\x0b\x32\x16.pubsub.pb.RPC.SubOpts\x12#\n\x07publish\x18\x02 \x03(\x0b\x32\x12.pubsub.pb.Message\x12*\n\x07\x63ontrol\x18\x03 \x01(\x0b\x32\x19.pubsub.pb.ControlMessage\x1a-\n\x07SubOpts\x12\x11\n\tsubscribe\x18\x01 \x01(\x08\x12\x0f\n\x07topicid\x18\x02 \x01(\t\"i\n\x07Message\x12\x0f\n\x07\x66rom_id\x18\x01 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\r\n\x05seqno\x18\x03 \x01(\x0c\x12\x10\n\x08topicIDs\x18\x04 \x03(\t\x12\x11\n\tsignature\x18\x05 \x01(\x0c\x12\x0b\n\x03key\x18\x06 \x01(\x0c\"\xb0\x01\n\x0e\x43ontrolMessage\x12&\n\x05ihave\x18\x01 \x03(\x0b\x32\x17.pubsub.pb.ControlIHave\x12&\n\x05iwant\x18\x02 \x03(\x0b\x32\x17.pubsub.pb.ControlIWant\x12&\n\x05graft\x18\x03 \x03(\x0b\x32\x17.pubsub.pb.ControlGraft\x12&\n\x05prune\x18\x04 \x03(\x0b\x32\x17.pubsub.pb.ControlPrune\"3\n\x0c\x43ontrolIHave\x12\x0f\n\x07topicID\x18\x01 \x01(\t\x12\x12\n\nmessageIDs\x18\x02 \x03(\t\"\"\n\x0c\x43ontrolIWant\x12\x12\n\nmessageIDs\x18\x01 \x03(\t\"\x1f\n\x0c\x43ontrolGraft\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x1f\n\x0c\x43ontrolPrune\x12\x0f\n\x07topicID\x18\x01 \x01(\t\"\x87\x03\n\x0fTopicDescriptor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x31\n\x04\x61uth\x18\x02 \x01(\x0b\x32#.pubsub.pb.TopicDescriptor.AuthOpts\x12/\n\x03\x65nc\x18\x03 \x01(\x0b\x32\".pubsub.pb.TopicDescriptor.EncOpts\x1a|\n\x08\x41uthOpts\x12:\n\x04mode\x18\x01 \x01(\x0e\x32,.pubsub.pb.TopicDescriptor.AuthOpts.AuthMode\x12\x0c\n\x04keys\x18\x02 \x03(\x0c\"&\n\x08\x41uthMode\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03KEY\x10\x01\x12\x07\n\x03WOT\x10\x02\x1a\x83\x01\n\x07\x45ncOpts\x12\x38\n\x04mode\x18\x01 \x01(\x0e\x32*.pubsub.pb.TopicDescriptor.EncOpts.EncMode\x12\x11\n\tkeyHashes\x18\x02 \x03(\x0c\"+\n\x07\x45ncMode\x12\x08\n\x04NONE\x10\x00\x12\r\n\tSHAREDKEY\x10\x01\x12\x07\n\x03WOT\x10\x02') ) @@ -45,8 +45,8 @@ _TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE = _descriptor.EnumDescriptor( ], containing_type=None, serialized_options=None, - serialized_start=865, - serialized_end=903, + serialized_start=868, + serialized_end=906, ) _sym_db.RegisterEnumDescriptor(_TOPICDESCRIPTOR_AUTHOPTS_AUTHMODE) @@ -71,8 +71,8 @@ _TOPICDESCRIPTOR_ENCOPTS_ENCMODE = _descriptor.EnumDescriptor( ], containing_type=None, serialized_options=None, - serialized_start=994, - serialized_end=1037, + serialized_start=997, + serialized_end=1040, ) _sym_db.RegisterEnumDescriptor(_TOPICDESCRIPTOR_ENCOPTS_ENCMODE) @@ -167,7 +167,7 @@ _MESSAGE = _descriptor.Descriptor( containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='from', full_name='pubsub.pb.Message.from', index=0, + name='from_id', full_name='pubsub.pb.Message.from_id', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, @@ -221,7 +221,7 @@ _MESSAGE = _descriptor.Descriptor( oneofs=[ ], serialized_start=207, - serialized_end=309, + serialized_end=312, ) @@ -272,8 +272,8 @@ _CONTROLMESSAGE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=312, - serialized_end=488, + serialized_start=315, + serialized_end=491, ) @@ -310,8 +310,8 @@ _CONTROLIHAVE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=490, - serialized_end=541, + serialized_start=493, + serialized_end=544, ) @@ -341,8 +341,8 @@ _CONTROLIWANT = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=543, - serialized_end=577, + serialized_start=546, + serialized_end=580, ) @@ -372,8 +372,8 @@ _CONTROLGRAFT = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=579, - serialized_end=610, + serialized_start=582, + serialized_end=613, ) @@ -403,8 +403,8 @@ _CONTROLPRUNE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=612, - serialized_end=643, + serialized_start=615, + serialized_end=646, ) @@ -442,8 +442,8 @@ _TOPICDESCRIPTOR_AUTHOPTS = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=779, - serialized_end=903, + serialized_start=782, + serialized_end=906, ) _TOPICDESCRIPTOR_ENCOPTS = _descriptor.Descriptor( @@ -480,8 +480,8 @@ _TOPICDESCRIPTOR_ENCOPTS = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=906, - serialized_end=1037, + serialized_start=909, + serialized_end=1040, ) _TOPICDESCRIPTOR = _descriptor.Descriptor( @@ -524,8 +524,8 @@ _TOPICDESCRIPTOR = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=646, - serialized_end=1037, + serialized_start=649, + serialized_end=1040, ) _RPC_SUBOPTS.containing_type = _RPC From a25aa0b034de77ecc8c3436a123dd5c6a7a49095 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 15:25:33 -0400 Subject: [PATCH 098/131] rewrote get_hello_packet --- libp2p/pubsub/pubsub.py | 74 +++++++++++++---------------------------- 1 file changed, 24 insertions(+), 50 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 9bd072f..17ecdd5 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -1,5 +1,7 @@ import asyncio +from .pb import rpc_pb2_grpc +from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee from .message import MessageSub from .message import create_message_talk, create_message_sub @@ -7,37 +9,6 @@ from. message import generate_message_id class Pubsub(): - """ - For now, because I'm on a plane and don't have access to the go repo/protobuf stuff, - this is going to be the message format for the two types: subscription and talk - subscription indicates subscribing or unsubscribing from a topic - talk is sending a message on topic(s) - subscription format: - subscription - 'from' - :'topicid' - :'topicid' - ... - Ex. - subscription - msg_sender_peer_id - origin_peer_id - sub:topic1 - sub:topic2 - unsub:fav_topic - talk format: - talk - 'from' - 'origin' - [topic_ids comma-delimited] - 'data' - Ex. - talk - msg_sender_peer_id - origin_peer_id - topic1,topics_are_cool,foo - I like tacos - """ # pylint: disable=too-many-instance-attributes def __init__(self, host, router, my_id): @@ -90,34 +61,37 @@ class Pubsub(): """ Generate subscription message with all topics we are subscribed to """ - subs_map = {} - for topic in self.my_topics: - subs_map[topic] = True - sub_msg = MessageSub( - str(self.host.get_id()),\ - str(self.host.get_id()), subs_map, generate_message_id()\ - ) - return sub_msg.to_str() + packet = rpc_pb2.RPC() + message = rpc_pb2.Message( + from_id=str(self.host.get_id()).encode('utf-8'), + seqno=str(generate_message_id()).encode('utf-8') + ) + packet.publish.extend([message]) + for topic_id in self.my_topics: + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe=True, topicid=topic_id)]) + + return packet.SerializeToString() async def continously_read_stream(self, stream): """ Read from input stream in an infinite loop. Process messages from other nodes, which for now are considered MessageTalk and MessageSub messages. - TODO: Handle RPC messages instead of my Aspyn's own custom message format :param stream: stream to continously read from """ while True: - incoming = (await stream.read()).decode() - msg_comps = incoming.split('\n') - msg_type = msg_comps[0] + incoming = (await stream.read()) + rpc_incoming = rpc_pb2.RPC() + rpc_incoming.ParseFromString(incoming) + print (rpc_incoming.publish) - msg_sender = msg_comps[1] + # msg_type = msg_comps[0] + + msg_sender = rpc_incoming.publish.from_id # msg_origin = msg_comps[2] - msg_id = msg_comps[3] - print("HIT ME1") + msg_id = rpc_incoming.publish.seqno if msg_id not in self.seen_messages: - print("HIT ME") # Do stuff with incoming unseen message should_publish = True if msg_type == "subscription": @@ -161,7 +135,7 @@ class Pubsub(): # Send hello packet hello = self.get_hello_packet() - await stream.write(hello.encode()) + await stream.write(hello) # Pass stream off to stream reader asyncio.ensure_future(self.continously_read_stream(stream)) @@ -187,7 +161,7 @@ class Pubsub(): # Send hello packet hello = self.get_hello_packet() - await stream.write(hello.encode()) + await stream.write(hello) # Pass stream off to stream reader asyncio.ensure_future(self.continously_read_stream(stream)) @@ -291,4 +265,4 @@ class Pubsub(): stream = self.peers[peer] # Write message to stream - await stream.write(encoded_msg) + await stream.write(encoded_msg) \ No newline at end of file From dea31531da905f2bb8acf3454f24bf19ff50f9d3 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Thu, 28 Mar 2019 15:42:55 -0400 Subject: [PATCH 099/131] update dependencies --- requirements_dev.txt | 2 ++ setup.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/requirements_dev.txt b/requirements_dev.txt index 4bf18f6..e3d8c0e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -3,3 +3,5 @@ codecov pytest-cov pytest-asyncio pylint +grpcio +grpcio-tools diff --git a/setup.py b/setup.py index abaa1c7..c44a276 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,8 @@ setuptools.setup( "base58", "pymultihash", "multiaddr", + "grpcio", + "grpcio-tools" ], packages=["libp2p"], zip_safe=False, From 83de4f2972f792f4b965c96e850f25ba404b7cfa Mon Sep 17 00:00:00 2001 From: Alex Haynes Date: Fri, 29 Mar 2019 16:23:30 -0400 Subject: [PATCH 100/131] RPC conversion progress --- libp2p/pubsub/floodsub.py | 41 ++++++++++++++----- libp2p/pubsub/pubsub.py | 85 +++++++++++++++++++++++++++------------ 2 files changed, 91 insertions(+), 35 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 68e542c..1cea17e 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -1,4 +1,6 @@ from .pubsub_router_interface import IPubsubRouter +from .pb import rpc_pb2 +from .message import MessageSub, MessageTalk from .message import create_message_talk class FloodSub(IPubsubRouter): @@ -56,28 +58,49 @@ class FloodSub(IPubsubRouter): """ # Encode message - encoded_msg = message.encode() + # encoded_msg = message.encode() + + if isinstance(message, str): + msg_talk = create_message_talk(message) + message = rpc_pb2.Message( + from_id=str(msg_talk.origin_id).encode('utf-8'), + seqno=str(msg_talk.message_id).encode('utf-8'), + topicIDs=msg_talk.topics, + data=msg_talk.data.encode() + + ) + packet = rpc_pb2.RPC() + print("YEET") + print(type(message)) + packet.publish.extend([message]) + + # Get message sender, origin, and topics - msg_talk = create_message_talk(message) + # msg_talk = create_message_talk(message) msg_sender = str(sender_peer_id) - msg_origin = msg_talk.origin_id - topics = msg_talk.topics + # msg_origin = msg_talk.origin_id + # topics = msg_talk.topics # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message - if msg_sender == msg_origin and msg_sender == str(self.pubsub.host.get_id()): - await self.pubsub.handle_talk(message) + if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): + old_format = MessageTalk(sender_peer_id, + message.from_id, + message.topicIDs, + message.data, + message.seqno) + await self.pubsub.handle_talk(old_format) # Deliver to self and peers - for topic in topics: + for topic in message.topicIDs: if topic in self.pubsub.peer_topics: for peer_id_in_topic in self.pubsub.peer_topics[topic]: # Forward to all known peers in the topic that are not the # message sender and are not the message origin - if peer_id_in_topic not in (msg_sender, msg_origin): + if peer_id_in_topic not in (msg_sender, message.from_id): stream = self.pubsub.peers[peer_id_in_topic] - await stream.write(encoded_msg) + await stream.write(packet.SerializeToString()) else: # Implies publish did not write print("publish did not write") diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 17ecdd5..b1ee363 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -3,7 +3,7 @@ import asyncio from .pb import rpc_pb2_grpc from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee -from .message import MessageSub +from .message import MessageSub, MessageTalk from .message import create_message_talk, create_message_sub from. message import generate_message_id @@ -61,6 +61,7 @@ class Pubsub(): """ Generate subscription message with all topics we are subscribed to """ + print("MAKING HELLO PACKET") packet = rpc_pb2.RPC() message = rpc_pb2.Message( from_id=str(self.host.get_id()).encode('utf-8'), @@ -80,43 +81,74 @@ class Pubsub(): and MessageSub messages. :param stream: stream to continously read from """ + + # TODO check on types here + peer_id = stream.mplex_conn.peer_id + while True: incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() + print("JUST GOT STRING") + print(incoming) rpc_incoming.ParseFromString(incoming) - print (rpc_incoming.publish) + print("JUST GOT") + print (rpc_incoming) + + if hasattr(rpc_incoming, 'publish'): + # deal with "talk messages" + for msg in rpc_incoming.publish: + # TODO check what from_id is in go? is it origin or peer + old_format = MessageTalk(peer_id, + msg.from_id, + msg.topicIDs, + msg.data, + msg.seqno) + self.seen_messages.append(msg.seqno) + await self.handle_talk(old_format) + await self.router.publish(peer_id, msg) + + if hasattr(rpc_incoming, 'subscriptions'): + # deal with "subscription messages" + subs_map = {} + for msg in rpc_incoming.subscriptions: + if msg.subscribe: + subs_map[msg.topic_id] = "sub" + else: + subs_map[msg.topic_id] = "unsub" + old_format = MessageSub(peer_id, peer_id, subs_map, generate_message_id()) + self.handle_subscription(old_format) # msg_type = msg_comps[0] - msg_sender = rpc_incoming.publish.from_id - # msg_origin = msg_comps[2] - msg_id = rpc_incoming.publish.seqno - if msg_id not in self.seen_messages: - # Do stuff with incoming unseen message - should_publish = True - if msg_type == "subscription": - self.handle_subscription(incoming) + # msg_sender = rpc_incoming.publish.from_id + # # msg_origin = msg_comps[2] + # msg_id = rpc_incoming.publish.seqno + # if msg_id not in self.seen_messages: + # # Do stuff with incoming unseen message + # should_publish = True + # if msg_type == "subscription": + # self.handle_subscription(incoming) - # We don't need to relay the subscription to our - # peers because a given node only needs its peers - # to know that it is subscribed to the topic (doesn't - # need everyone to know) - should_publish = False - elif msg_type == "talk": - await self.handle_talk(incoming) + # # We don't need to relay the subscription to our + # # peers because a given node only needs its peers + # # to know that it is subscribed to the topic (doesn't + # # need everyone to know) + # should_publish = False + # elif msg_type == "talk": + # await self.handle_talk(incoming) # Add message id to seen - self.seen_messages.append(msg_id) + # self.seen_messages.append(msg_id) # Publish message using router's publish - if should_publish: - msg = create_message_talk(incoming) + # if should_publish: + # msg = create_message_talk(incoming) - # Adjust raw_msg to that the message sender - # is now our peer_id - msg.from_id = str(self.host.get_id()) + # # Adjust raw_msg to that the message sender + # # is now our peer_id + # msg.from_id = str(self.host.get_id()) - await self.router.publish(msg_sender, msg.to_str()) + # await self.router.publish(msg_sender, msg.to_str()) # Force context switch await asyncio.sleep(0) @@ -135,6 +167,7 @@ class Pubsub(): # Send hello packet hello = self.get_hello_packet() + print(hello) await stream.write(hello) # Pass stream off to stream reader asyncio.ensure_future(self.continously_read_stream(stream)) @@ -176,7 +209,7 @@ class Pubsub(): defined in the subscription message :param subscription: raw data constituting a subscription message """ - sub_msg = create_message_sub(subscription) + sub_msg = subscription if sub_msg.subs_map: print("handle_subscription my_id: " + self.my_id + ", subber: " + sub_msg.origin_id) for topic_id in sub_msg.subs_map: @@ -198,7 +231,7 @@ class Pubsub(): custom message that is published on a given topic(s) :param talk: raw data constituting a talk message """ - msg = create_message_talk(talk) + msg = talk # Check if this message has any topics that we are subscribed to for topic in msg.topics: From c03ba881f294ea9e86fac8012fc96ca6496ba146 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sat, 30 Mar 2019 17:59:08 -0400 Subject: [PATCH 101/131] reworked subscribe unsubsrcibe --- libp2p/pubsub/pubsub.py | 113 +++++++++++++++------------------------- 1 file changed, 43 insertions(+), 70 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index b1ee363..85abacb 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -61,7 +61,7 @@ class Pubsub(): """ Generate subscription message with all topics we are subscribed to """ - print("MAKING HELLO PACKET") + packet = rpc_pb2.RPC() message = rpc_pb2.Message( from_id=str(self.host.get_id()).encode('utf-8'), @@ -74,7 +74,7 @@ class Pubsub(): return packet.SerializeToString() - async def continously_read_stream(self, stream): + async def continuously_read_stream(self, stream): """ Read from input stream in an infinite loop. Process messages from other nodes, which for now are considered MessageTalk @@ -88,16 +88,12 @@ class Pubsub(): while True: incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() - print("JUST GOT STRING") - print(incoming) rpc_incoming.ParseFromString(incoming) - print("JUST GOT") + print ("CONTINUOUSLY") print (rpc_incoming) - - if hasattr(rpc_incoming, 'publish'): + if rpc_incoming.publish: # deal with "talk messages" for msg in rpc_incoming.publish: - # TODO check what from_id is in go? is it origin or peer old_format = MessageTalk(peer_id, msg.from_id, msg.topicIDs, @@ -107,48 +103,16 @@ class Pubsub(): await self.handle_talk(old_format) await self.router.publish(peer_id, msg) - if hasattr(rpc_incoming, 'subscriptions'): + if rpc_incoming.subscriptions: # deal with "subscription messages" subs_map = {} for msg in rpc_incoming.subscriptions: if msg.subscribe: - subs_map[msg.topic_id] = "sub" + subs_map[msg.topicid] = "sub" else: - subs_map[msg.topic_id] = "unsub" - old_format = MessageSub(peer_id, peer_id, subs_map, generate_message_id()) - self.handle_subscription(old_format) + subs_map[msg.topicid] = "unsub" - # msg_type = msg_comps[0] - - # msg_sender = rpc_incoming.publish.from_id - # # msg_origin = msg_comps[2] - # msg_id = rpc_incoming.publish.seqno - # if msg_id not in self.seen_messages: - # # Do stuff with incoming unseen message - # should_publish = True - # if msg_type == "subscription": - # self.handle_subscription(incoming) - - # # We don't need to relay the subscription to our - # # peers because a given node only needs its peers - # # to know that it is subscribed to the topic (doesn't - # # need everyone to know) - # should_publish = False - # elif msg_type == "talk": - # await self.handle_talk(incoming) - - # Add message id to seen - # self.seen_messages.append(msg_id) - - # Publish message using router's publish - # if should_publish: - # msg = create_message_talk(incoming) - - # # Adjust raw_msg to that the message sender - # # is now our peer_id - # msg.from_id = str(self.host.get_id()) - - # await self.router.publish(msg_sender, msg.to_str()) + self.handle_subscription(rpc_incoming) # Force context switch await asyncio.sleep(0) @@ -167,10 +131,10 @@ class Pubsub(): # Send hello packet hello = self.get_hello_packet() - print(hello) + await stream.write(hello) # Pass stream off to stream reader - asyncio.ensure_future(self.continously_read_stream(stream)) + asyncio.ensure_future(self.continuously_read_stream(stream)) async def handle_peer_queue(self): """ @@ -197,30 +161,30 @@ class Pubsub(): await stream.write(hello) # Pass stream off to stream reader - asyncio.ensure_future(self.continously_read_stream(stream)) + asyncio.ensure_future(self.continuously_read_stream(stream)) # Force context switch await asyncio.sleep(0) - def handle_subscription(self, subscription): + def handle_subscription(self, rpc_message): """ Handle an incoming subscription message from a peer. Update internal mapping to mark the peer as subscribed or unsubscribed to topics as defined in the subscription message :param subscription: raw data constituting a subscription message """ - sub_msg = subscription - if sub_msg.subs_map: - print("handle_subscription my_id: " + self.my_id + ", subber: " + sub_msg.origin_id) - for topic_id in sub_msg.subs_map: + for sub_msg in rpc_message.subscriptions: + # Look at each subscription in the msg individually - if sub_msg.subs_map[topic_id]: - if topic_id not in self.peer_topics: + if sub_msg.subscribe: + origin_id = rpc_message.publish[0].from_id + + if sub_msg.topicid not in self.peer_topics: # Create topic list if it did not yet exist - self.peer_topics[topic_id] = [sub_msg.origin_id] - elif sub_msg.origin_id not in self.peer_topics[topic_id]: + self.peer_topics[sub_msg.topicid] = origin_id + elif orgin_id not in self.peer_topics[sub_msg.topicid]: # Add peer to topic - self.peer_topics[topic_id].append(sub_msg.origin_id) + self.peer_topics[sub_msg.topicid].append(origin_id) else: # TODO: Remove peer from topic pass @@ -250,13 +214,18 @@ class Pubsub(): self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message - sub_msg = MessageSub( - str(self.host.get_id()),\ - str(self.host.get_id()), {topic_id: True}, generate_message_id()\ - ) + packet = rpc_pb2.RPC() + packet.publish.extend([rpc_pb2.Message( + from_id=str(self.host.get_id()).encode('utf-8'), + seqno=str(generate_message_id()).encode('utf-8') + )]) + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe = True, + topicid = topic_id.encode('utf-8') + )]) # Send out subscribe message to all peers - await self.message_all_peers(sub_msg.to_str()) + await self.message_all_peers(packet.SerializeToString()) # Tell router we are joining this topic self.router.join(topic_id) @@ -275,27 +244,31 @@ class Pubsub(): del self.my_topics[topic_id] # Create unsubscribe message - unsub_msg = MessageSub(str(self.host.get_id()), str(self.host.get_id()),\ - {topic_id: False}, generate_message_id()) + packet = rpc_pb2.RPC() + packet.publish.extend([rpc_pb2.Message( + from_id=str(self.host.get_id()).encode('utf-8'), + seqno=str(generate_message_id()).encode('utf-8') + )]) + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe = False, + topicid = topic_id.encode('utf-8') + )]) # Send out unsubscribe message to all peers - await self.message_all_peers(unsub_msg.to_str()) + await self.message_all_peers(packet.SerializeToString()) # Tell router we are leaving this topic self.router.leave(topic_id) - async def message_all_peers(self, raw_msg): + async def message_all_peers(self, rpc_msg): """ Broadcast a message to peers :param raw_msg: raw contents of the message to broadcast """ - # Encode message for sending - encoded_msg = raw_msg.encode() - # Broadcast message for peer in self.peers: stream = self.peers[peer] # Write message to stream - await stream.write(encoded_msg) \ No newline at end of file + await stream.write(rpc_msg) From 35a587f72a97224b74f9e2a7332edbcdf853c555 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sat, 30 Mar 2019 18:49:50 -0400 Subject: [PATCH 102/131] remove message.py --- libp2p/pubsub/floodsub.py | 31 +--------- libp2p/pubsub/message.py | 118 -------------------------------------- libp2p/pubsub/pubsub.py | 28 +++++---- 3 files changed, 15 insertions(+), 162 deletions(-) delete mode 100644 libp2p/pubsub/message.py diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 1cea17e..dd4dfc1 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -1,7 +1,6 @@ from .pubsub_router_interface import IPubsubRouter from .pb import rpc_pb2 -from .message import MessageSub, MessageTalk -from .message import create_message_talk + class FloodSub(IPubsubRouter): @@ -57,40 +56,14 @@ class FloodSub(IPubsubRouter): :param message: message to forward """ - # Encode message - # encoded_msg = message.encode() - - if isinstance(message, str): - msg_talk = create_message_talk(message) - message = rpc_pb2.Message( - from_id=str(msg_talk.origin_id).encode('utf-8'), - seqno=str(msg_talk.message_id).encode('utf-8'), - topicIDs=msg_talk.topics, - data=msg_talk.data.encode() - - ) packet = rpc_pb2.RPC() - print("YEET") - print(type(message)) packet.publish.extend([message]) - - - - # Get message sender, origin, and topics - # msg_talk = create_message_talk(message) msg_sender = str(sender_peer_id) - # msg_origin = msg_talk.origin_id - # topics = msg_talk.topics # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): - old_format = MessageTalk(sender_peer_id, - message.from_id, - message.topicIDs, - message.data, - message.seqno) - await self.pubsub.handle_talk(old_format) + await self.pubsub.handle_talk(sender_peer_id, message) # Deliver to self and peers for topic in message.topicIDs: diff --git a/libp2p/pubsub/message.py b/libp2p/pubsub/message.py deleted file mode 100644 index 2f839dc..0000000 --- a/libp2p/pubsub/message.py +++ /dev/null @@ -1,118 +0,0 @@ -import uuid - - -class MessageTalk(): - - """ - Object to make parsing talk messages easier, where talk messages are - defined as custom messages published to a set of topics - """ - # pylint: disable=too-few-public-methods - def __init__(self, from_id, origin_id, topics, data, message_id): - # pylint: disable=too-many-arguments - self.msg_type = "talk" - self.from_id = from_id - self.origin_id = origin_id - self.topics = topics - self.data = data - self.message_id = message_id - - def to_str(self): - """ - Convert to string - :return: MessageTalk object in string representation - """ - out = self.msg_type + '\n' - out += self.from_id + '\n' - out += self.origin_id + '\n' - out += self.message_id + '\n' - for i in range(len(self.topics)): - out += self.topics[i] - if i < len(self.topics) - 1: - out += ',' - out += '\n' + self.data - return out - - -class MessageSub(): - """ - Object to make parsing subscription messages easier, where subscription - messages are defined as indicating the topics a node wishes to subscribe to - or unsubscribe from - """ - # pylint: disable=too-few-public-methods - def __init__(self, from_id, origin_id, subs_map, message_id): - self.msg_type = "subscription" - self.from_id = from_id - self.origin_id = origin_id - self.subs_map = subs_map - self.message_id = message_id - - def to_str(self): - """ - Convert to string - :return: MessageSub object in string representation - """ - out = self.msg_type + '\n' - out += self.from_id + '\n' - out += self.origin_id + '\n' - out += self.message_id - - if self.subs_map: - out += '\n' - - keys = list(self.subs_map) - - for i, topic in enumerate(keys): - sub = self.subs_map[topic] - if sub: - out += "sub:" - else: - out += "unsub:" - out += topic - if i < len(keys) - 1: - out += '\n' - - return out - -def create_message_talk(msg_talk_as_str): - """ - Create a MessageTalk object from a MessageTalk string representation - :param msg_talk_as_str: a MessageTalk object in its string representation - :return: MessageTalk object - """ - msg_comps = msg_talk_as_str.split('\n') - from_id = msg_comps[1] - origin_id = msg_comps[2] - message_id = msg_comps[3] - topics = msg_comps[4].split(',') - data = msg_comps[5] - return MessageTalk(from_id, origin_id, topics, data, message_id) - -def create_message_sub(msg_sub_as_str): - """ - Create a MessageSub object from a MessageSub string representation - :param msg_talk_as_str: a MessageSub object in its string representation - :return: MessageSub object - """ - msg_comps = msg_sub_as_str.split('\n') - from_id = msg_comps[1] - origin_id = msg_comps[2] - message_id = msg_comps[3] - - subs_map = {} - for i in range(4, len(msg_comps)): - sub_comps = msg_comps[i].split(":") - topic = sub_comps[1] - if sub_comps[0] == "sub": - subs_map[topic] = True - else: - subs_map[topic] = False - return MessageSub(from_id, origin_id, subs_map, message_id) - -def generate_message_id(): - """ - Generate a unique message id - :return: messgae id - """ - return str(uuid.uuid1()) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 85abacb..976e6b4 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -1,11 +1,9 @@ import asyncio +import uuid from .pb import rpc_pb2_grpc from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee -from .message import MessageSub, MessageTalk -from .message import create_message_talk, create_message_sub -from. message import generate_message_id class Pubsub(): @@ -77,8 +75,7 @@ class Pubsub(): async def continuously_read_stream(self, stream): """ Read from input stream in an infinite loop. Process - messages from other nodes, which for now are considered MessageTalk - and MessageSub messages. + messages from other nodes :param stream: stream to continously read from """ @@ -94,13 +91,8 @@ class Pubsub(): if rpc_incoming.publish: # deal with "talk messages" for msg in rpc_incoming.publish: - old_format = MessageTalk(peer_id, - msg.from_id, - msg.topicIDs, - msg.data, - msg.seqno) self.seen_messages.append(msg.seqno) - await self.handle_talk(old_format) + await self.handle_talk(peer_id, msg) await self.router.publish(peer_id, msg) if rpc_incoming.subscriptions: @@ -189,21 +181,20 @@ class Pubsub(): # TODO: Remove peer from topic pass - async def handle_talk(self, talk): + async def handle_talk(self, peer_id, publish_message): """ Handle incoming Talk message from a peer. A Talk message contains some custom message that is published on a given topic(s) :param talk: raw data constituting a talk message """ - msg = talk # Check if this message has any topics that we are subscribed to - for topic in msg.topics: + for topic in publish_message.topicIDs: if topic in self.my_topics: # we are subscribed to a topic this message was sent for, # so add message to the subscription output queue # for each topic - await self.my_topics[topic].put(talk) + await self.my_topics[topic].put(publish_message) async def subscribe(self, topic_id): """ @@ -272,3 +263,10 @@ class Pubsub(): # Write message to stream await stream.write(rpc_msg) + +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) \ No newline at end of file From 65526a3319eb18d44edd0e70c7e34efc28ba6463 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sat, 30 Mar 2019 19:12:31 -0400 Subject: [PATCH 103/131] remove Message from dummy account --- tests/pubsub/dummy_account_node.py | 57 +++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index f328a6e..de78211 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -1,12 +1,12 @@ import asyncio +import uuid import multiaddr from libp2p import new_node -from libp2p.pubsub.message import create_message_talk +from libp2p.pubsub.pb import rpc_pb2_grpc +from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub -from libp2p.pubsub.message import MessageTalk -from libp2p.pubsub.message import generate_message_id SUPPORTED_PUBSUB_PROTOCOLS = ["/floodsub/1.0.0"] CRYPTO_TOPIC = "ethereum" @@ -53,16 +53,15 @@ class DummyAccountNode(): Handle all incoming messages on the CRYPTO_TOPIC from peers """ while True: - message_raw = await self.q.get() - message = create_message_talk(message_raw) - contents = message.data - - msg_comps = contents.split(",") - - if msg_comps[0] == "send": - self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) - elif msg_comps[0] == "set": - self.handle_set_crypto_for_user(msg_comps[1], int(msg_comps[2])) + incoming = await self.q.get() + rpc_incoming = rpc_pb2.RPC() + rpc_incoming.ParseFromString(incoming) + for message in rpc_incoming.publish: + msg_comps = message.split(",") + if msg_comps[0] == "send": + self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) + elif msg_comps[0] == "set": + self.handle_set_crypto_for_user(msg_comps[1], int(msg_comps[2])) async def setup_crypto_networking(self): """ @@ -82,8 +81,8 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "send," + source_user + "," + dest_user + "," + str(amount) - msg = MessageTalk(my_id, my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) - await self.floodsub.publish(my_id, msg.to_str()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + await self.floodsub.publish(my_id, msg.SerializeToString()) async def publish_set_crypto(self, user, amount): """ @@ -93,8 +92,9 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "set," + user + "," + str(amount) - msg = MessageTalk(my_id, my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) - await self.floodsub.publish(my_id, msg.to_str()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + + await self.floodsub.publish(my_id, packet.SerializeToString()) def handle_send_crypto(self, source_user, dest_user, amount): """ @@ -132,3 +132,26 @@ class DummyAccountNode(): else: return -1 +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) + +def generate_RPC_packet(origin_id, topics, msg_content, msg_id): + packet = rpc_pb2.RPC() + packet.publish.extend([rpc_pb2.Message( + from_id=origin_id.encode('utf-8'), + seqno=msg_id.encode('utf-8'), + data=msg_content.encode('utf-8'), + topicIDs=topics.encode('utf-8')) + ]) + + for topic in topics: + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe=True, + topicid = topic.encode('utf-8') + )]) + + return packet From 3836aa65f143f029f509f5870c0d5141855e947c Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sat, 30 Mar 2019 19:30:58 -0400 Subject: [PATCH 104/131] fix encoding issue --- libp2p/pubsub/floodsub.py | 3 +++ tests/pubsub/dummy_account_node.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index dd4dfc1..8b9ba1c 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -57,6 +57,9 @@ class FloodSub(IPubsubRouter): """ packet = rpc_pb2.RPC() + print ("IN FLOOODSUB PUBLISH") + print (message) + print ("++++++++++++++++") packet.publish.extend([message]) msg_sender = str(sender_peer_id) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index de78211..930dc57 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -141,17 +141,21 @@ def generate_message_id(): def generate_RPC_packet(origin_id, topics, msg_content, msg_id): packet = rpc_pb2.RPC() - packet.publish.extend([rpc_pb2.Message( + message = rpc_pb2.Message( from_id=origin_id.encode('utf-8'), seqno=msg_id.encode('utf-8'), - data=msg_content.encode('utf-8'), - topicIDs=topics.encode('utf-8')) - ]) + data=msg_content.encode('utf-8') + ) for topic in topics: + message.topicIDs.extend([topic.encode('utf-8')]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe=True, topicid = topic.encode('utf-8') )]) + packet.publish.extend([message]) + print ("HEYHO") + print (packet) + return packet From fb5b3e4a247bf650f32af6d2f4f496af010a001f Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 31 Mar 2019 22:16:28 -0400 Subject: [PATCH 105/131] reworked floodsub logic --- libp2p/pubsub/floodsub.py | 46 +++++++++++++--------- libp2p/pubsub/pubsub.py | 83 ++++++++++++++++++++++----------------- 2 files changed, 74 insertions(+), 55 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 8b9ba1c..d4736d0 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -41,7 +41,7 @@ class FloodSub(IPubsubRouter): :param rpc: rpc message """ - async def publish(self, sender_peer_id, message): + async def publish(self, sender_peer_id, rpc_message): """ Invoked to forward a new message that has been validated. This is where the "flooding" part of floodsub happens @@ -53,33 +53,41 @@ class FloodSub(IPubsubRouter): It also never forwards a message back to the source or the peer that forwarded the message. :param sender_peer_id: peer_id of message sender - :param message: message to forward + :param rpc_message: pubsub message in RPC string format """ packet = rpc_pb2.RPC() + packet.ParseFromString(rpc_message) print ("IN FLOOODSUB PUBLISH") - print (message) + print (packet) print ("++++++++++++++++") - packet.publish.extend([message]) msg_sender = str(sender_peer_id) - # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message - if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): - await self.pubsub.handle_talk(sender_peer_id, message) + for message in packet.publish: + if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): + await self.pubsub.handle_talk(sender_peer_id, message) - # Deliver to self and peers - for topic in message.topicIDs: - if topic in self.pubsub.peer_topics: - for peer_id_in_topic in self.pubsub.peer_topics[topic]: - # Forward to all known peers in the topic that are not the - # message sender and are not the message origin - if peer_id_in_topic not in (msg_sender, message.from_id): - stream = self.pubsub.peers[peer_id_in_topic] - await stream.write(packet.SerializeToString()) - else: - # Implies publish did not write - print("publish did not write") + + print ("OHOHOHOH") + print (self.pubsub.peer_topics) + print ("UUUJUJUJ") + print (self.pubsub.peers) + print ("********") + # Deliver to self and peers + for topic in message.topicIDs: + if topic in self.pubsub.peer_topics: + for peer_id_in_topic in self.pubsub.peer_topics[topic]: + # Forward to all known peers in the topic that are not the + # message sender and are not the message origin + print ("PEERID") + print (peer_id_in_topic) + if peer_id_in_topic not in (msg_sender, message.from_id): + stream = self.pubsub.peers[peer_id_in_topic] + await stream.write(packet.SerializeToString()) + else: + # Implies publish did not write + print("publish did not write") def join(self, topic): """ diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 976e6b4..7705dfb 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -80,31 +80,43 @@ class Pubsub(): """ # TODO check on types here - peer_id = stream.mplex_conn.peer_id + peer_id = str(stream.mplex_conn.peer_id) while True: incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() rpc_incoming.ParseFromString(incoming) - print ("CONTINUOUSLY") + + print ("IN PUBSUB CONTINUOUSLY READ") print (rpc_incoming) + print ("###########################") + + should_publish = True + if rpc_incoming.publish: - # deal with "talk messages" - for msg in rpc_incoming.publish: - self.seen_messages.append(msg.seqno) - await self.handle_talk(peer_id, msg) - await self.router.publish(peer_id, msg) + # deal with RPC.publish + for message in rpc_incoming.publish: + self.seen_messages.append(message.seqno) + await self.handle_talk(peer_id, message) + if rpc_incoming.subscriptions: - # deal with "subscription messages" - subs_map = {} - for msg in rpc_incoming.subscriptions: - if msg.subscribe: - subs_map[msg.topicid] = "sub" - else: - subs_map[msg.topicid] = "unsub" + # deal with RPC.subscriptions + # We don't need to relay the subscription to our + # peers because a given node only needs its peers + # to know that it is subscribed to the topic (doesn't + # need everyone to know) + should_publish = False - self.handle_subscription(rpc_incoming) + # TODO check that peer_id is the same as origin_id + from_id = str(rpc_incoming.publish[0].from_id.decode('utf-8')) + for message in rpc_incoming.subscriptions: + if message.subscribe: + self.handle_subscription(from_id, message) + + if should_publish: + # relay message to peers with router + await self.router.publish(peer_id, incoming) # Force context switch await asyncio.sleep(0) @@ -136,7 +148,10 @@ class Pubsub(): pubsub protocols we support """ while True: + print ("PUBSUB HANDLE PEER QUEUE") peer_id = await self.peer_queue.get() + print (peer_id) + print ("++++++++++++++++++++++++") # Open a stream to peer on existing connection # (we know connection exists since that's the only way @@ -158,34 +173,30 @@ class Pubsub(): # Force context switch await asyncio.sleep(0) - def handle_subscription(self, rpc_message): + def handle_subscription(self, peer_id, sub_message): """ Handle an incoming subscription message from a peer. Update internal mapping to mark the peer as subscribed or unsubscribed to topics as defined in the subscription message - :param subscription: raw data constituting a subscription message + :param origin_id: id of the peer who subscribe to the message + :param sub_message: RPC.SubOpts """ - for sub_msg in rpc_message.subscriptions: - - # Look at each subscription in the msg individually - if sub_msg.subscribe: - origin_id = rpc_message.publish[0].from_id - - if sub_msg.topicid not in self.peer_topics: - # Create topic list if it did not yet exist - self.peer_topics[sub_msg.topicid] = origin_id - elif orgin_id not in self.peer_topics[sub_msg.topicid]: - # Add peer to topic - self.peer_topics[sub_msg.topicid].append(origin_id) - else: - # TODO: Remove peer from topic - pass + # TODO verify logic here + if sub_message.subscribe: + if sub_message.topicid not in self.peer_topics: + self.peer_topics[sub_message.topicid] = [peer_id] + elif peer_id not in self.peer_topics[sub_message.topicid]: + # Add peer to topic + self.peer_topics[sub_message.topicid].append(peer_id) + else: + # TODO: Remove peer from topic + pass async def handle_talk(self, peer_id, publish_message): """ - Handle incoming Talk message from a peer. A Talk message contains some - custom message that is published on a given topic(s) - :param talk: raw data constituting a talk message + Put incoming message from a peer onto my blocking queue + :param peer_id: peer id whom forwarded this message + :param talk: RPC.Message format """ # Check if this message has any topics that we are subscribed to @@ -269,4 +280,4 @@ def generate_message_id(): Generate a unique message id :return: messgae id """ - return str(uuid.uuid1()) \ No newline at end of file + return str(uuid.uuid1()) From 73c1d87db1f8e1be001ab93786a8c0dbeebccdbf Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 31 Mar 2019 22:16:49 -0400 Subject: [PATCH 106/131] update dummy account node --- tests/pubsub/dummy_account_node.py | 32 +++++++++++++++++++----------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 930dc57..0773533 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -37,6 +37,7 @@ class DummyAccountNode(): We use create as this serves as a factory function and allows us to use async await, unlike the init function """ + print ("**DUMMY** CREATE ACCOUNT NODE") self = DummyAccountNode() libp2p_node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -54,14 +55,17 @@ class DummyAccountNode(): """ while True: incoming = await self.q.get() - rpc_incoming = rpc_pb2.RPC() - rpc_incoming.ParseFromString(incoming) - for message in rpc_incoming.publish: - msg_comps = message.split(",") - if msg_comps[0] == "send": - self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) - elif msg_comps[0] == "set": - self.handle_set_crypto_for_user(msg_comps[1], int(msg_comps[2])) + print ("**DUMMY** HANDLE INCOMING") + print (incoming) + print ("========================") + + msg_comps = incoming.data.decode('utf-8').split(",") + print (msg_comps) + print ("--------") + if msg_comps[0] == "send": + self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) + elif msg_comps[0] == "set": + self.handle_set_crypto(msg_comps[1], int(msg_comps[2])) async def setup_crypto_networking(self): """ @@ -103,6 +107,7 @@ class DummyAccountNode(): :param dest_user: user to send crypto to :param amount: amount of crypto to send """ + print ("**DUMMY** IN HANDLE SEND CRYPTO") if source_user in self.balances: self.balances[source_user] -= amount else: @@ -113,12 +118,15 @@ class DummyAccountNode(): else: self.balances[dest_user] = amount - def handle_set_crypto_for_user(self, dest_user, amount): + def handle_set_crypto(self, dest_user, amount): """ Handle incoming set_crypto message :param dest_user: user to set crypto for :param amount: amount of crypto """ + print ("**DUMMY** IN HANDLE SET CRYPTO") + print (type (dest_user)) + print (amount) self.balances[dest_user] = amount def get_balance(self, user): @@ -127,6 +135,9 @@ class DummyAccountNode(): :param user: user to get balance for :return: balance of user """ + print ("GET BALACNCE") + print (user) + print (self.balances) if user in self.balances: return self.balances[user] else: @@ -155,7 +166,4 @@ def generate_RPC_packet(origin_id, topics, msg_content, msg_id): )]) packet.publish.extend([message]) - print ("HEYHO") - print (packet) - return packet From 372ed8336c65636ea9a5e7caa574dae7bd0de4e7 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Mon, 1 Apr 2019 15:04:20 -0400 Subject: [PATCH 107/131] remove message from test dummy --- tests/pubsub/dummy_account_node.py | 2 +- tests/pubsub/test_dummyaccount_demo.py | 148 ++++++++++++------------- 2 files changed, 74 insertions(+), 76 deletions(-) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 0773533..eaca614 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -125,7 +125,7 @@ class DummyAccountNode(): :param amount: amount of crypto """ print ("**DUMMY** IN HANDLE SET CRYPTO") - print (type (dest_user)) + print (dest_user) print (amount) self.balances[dest_user] = amount diff --git a/tests/pubsub/test_dummyaccount_demo.py b/tests/pubsub/test_dummyaccount_demo.py index a071fa6..9b3a149 100644 --- a/tests/pubsub/test_dummyaccount_demo.py +++ b/tests/pubsub/test_dummyaccount_demo.py @@ -8,8 +8,6 @@ from libp2p import new_node from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub -from libp2p.pubsub.message import MessageTalk -from libp2p.pubsub.message import create_message_talk from dummy_account_node import DummyAccountNode # pylint: disable=too-many-locals @@ -66,7 +64,7 @@ async def perform_test(num_nodes, adjacency_map, action_func, assertion_func): await action_func(dummy_nodes) # Allow time for action function to be performed (i.e. messages to propogate) - await asyncio.sleep(0.25) + await asyncio.sleep(1) # Perform assertion function for dummy_node in dummy_nodes: @@ -88,102 +86,102 @@ async def test_simple_two_nodes(): await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_simple_three_nodes_line_topography(): - num_nodes = 3 - adj_map = {0: [1], 1: [2]} +# @pytest.mark.asyncio +# async def test_simple_three_nodes_line_topography(): +# num_nodes = 3 +# adj_map = {0: [1], 1: [2]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 10) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 10) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 10 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 10 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_simple_three_nodes_triangle_topography(): - num_nodes = 3 - adj_map = {0: [1, 2], 1: [2]} +# @pytest.mark.asyncio +# async def test_simple_three_nodes_triangle_topography(): +# num_nodes = 3 +# adj_map = {0: [1, 2], 1: [2]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 20) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 20) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 20 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 20 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_simple_seven_nodes_tree_topography(): - num_nodes = 7 - adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +# @pytest.mark.asyncio +# async def test_simple_seven_nodes_tree_topography(): +# num_nodes = 7 +# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 20) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 20) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 20 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 20 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_set_then_send_from_root_seven_nodes_tree_topography(): - num_nodes = 7 - adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +# @pytest.mark.asyncio +# async def test_set_then_send_from_root_seven_nodes_tree_topography(): +# num_nodes = 7 +# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 20) - await asyncio.sleep(0.25) - await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 20) +# await asyncio.sleep(0.25) +# await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 15 - assert dummy_node.get_balance("alex") == 5 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 15 +# assert dummy_node.get_balance("alex") == 5 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): - num_nodes = 7 - adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +# @pytest.mark.asyncio +# async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): +# num_nodes = 7 +# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} - async def action_func(dummy_nodes): - await dummy_nodes[6].publish_set_crypto("aspyn", 20) - await asyncio.sleep(0.25) - await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) +# async def action_func(dummy_nodes): +# await dummy_nodes[6].publish_set_crypto("aspyn", 20) +# await asyncio.sleep(0.25) +# await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 15 - assert dummy_node.get_balance("alex") == 5 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 15 +# assert dummy_node.get_balance("alex") == 5 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_simple_five_nodes_ring_topography(): - num_nodes = 5 - adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} +# @pytest.mark.asyncio +# async def test_simple_five_nodes_ring_topography(): +# num_nodes = 5 +# adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("aspyn", 20) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("aspyn", 20) - def assertion_func(dummy_node): - assert dummy_node.get_balance("aspyn") == 20 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("aspyn") == 20 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) -@pytest.mark.asyncio -async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): - num_nodes = 5 - adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} +# @pytest.mark.asyncio +# async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): +# num_nodes = 5 +# adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} - async def action_func(dummy_nodes): - await dummy_nodes[0].publish_set_crypto("alex", 20) - await asyncio.sleep(0.25) - await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) +# async def action_func(dummy_nodes): +# await dummy_nodes[0].publish_set_crypto("alex", 20) +# await asyncio.sleep(0.25) +# await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) - def assertion_func(dummy_node): - assert dummy_node.get_balance("alex") == 8 - assert dummy_node.get_balance("rob") == 12 +# def assertion_func(dummy_node): +# assert dummy_node.get_balance("alex") == 8 +# assert dummy_node.get_balance("rob") == 12 - await perform_test(num_nodes, adj_map, action_func, assertion_func) +# await perform_test(num_nodes, adj_map, action_func, assertion_func) From bc6a27a762893cbba50492ced3c38b28a6d33014 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Mon, 1 Apr 2019 16:23:20 -0400 Subject: [PATCH 108/131] fix all tests --- libp2p/pubsub/floodsub.py | 18 ++- libp2p/pubsub/pubsub.py | 62 ++++++----- tests/pubsub/dummy_account_node.py | 4 +- tests/pubsub/test_dummyaccount_demo.py | 144 ++++++++++++------------ tests/pubsub/test_floodsub.py | 147 ++++++++++++++++--------- 5 files changed, 221 insertions(+), 154 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index d4736d0..305607e 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -65,7 +65,16 @@ class FloodSub(IPubsubRouter): # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message for message in packet.publish: - if msg_sender == message.from_id and msg_sender == str(self.pubsub.host.get_id()): + decoded_from_id = message.from_id.decode('utf-8') + + print ("MESSAGE SENDER") + print (msg_sender) + print ("FROM ID") + print (message.from_id) + print (str(self.pubsub.host.get_id())) + + + if msg_sender == decoded_from_id and msg_sender == str(self.pubsub.host.get_id()): await self.pubsub.handle_talk(sender_peer_id, message) @@ -82,9 +91,12 @@ class FloodSub(IPubsubRouter): # message sender and are not the message origin print ("PEERID") print (peer_id_in_topic) - if peer_id_in_topic not in (msg_sender, message.from_id): + if peer_id_in_topic not in (msg_sender, decoded_from_id): stream = self.pubsub.peers[peer_id_in_topic] - await stream.write(packet.SerializeToString()) + # create new packet with just publish message + new_packet = rpc_pb2.RPC() + new_packet.publish.extend([message]) + await stream.write(new_packet.SerializeToString()) else: # Implies publish did not write print("publish did not write") diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 7705dfb..c7bfe06 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -58,17 +58,19 @@ class Pubsub(): def get_hello_packet(self): """ Generate subscription message with all topics we are subscribed to + only send hello packet if we have subscribed topics """ - packet = rpc_pb2.RPC() - message = rpc_pb2.Message( - from_id=str(self.host.get_id()).encode('utf-8'), - seqno=str(generate_message_id()).encode('utf-8') - ) - packet.publish.extend([message]) - for topic_id in self.my_topics: - packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - subscribe=True, topicid=topic_id)]) + if self.my_topics: + for topic_id in self.my_topics: + packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + subscribe=True, topicid=topic_id)]) + + # message = rpc_pb2.Message( + # from_id=str(self.host.get_id()).encode('utf-8'), + # seqno=str(generate_message_id()).encode('utf-8') + # ) + # packet.publish.extend([message]) return packet.SerializeToString() @@ -80,9 +82,11 @@ class Pubsub(): """ # TODO check on types here + print ("++++++++++ASPYN+++++++++++++++++") peer_id = str(stream.mplex_conn.peer_id) while True: + print ("HIT ME") incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() rpc_incoming.ParseFromString(incoming) @@ -91,13 +95,15 @@ class Pubsub(): print (rpc_incoming) print ("###########################") - should_publish = True + should_publish = False if rpc_incoming.publish: # deal with RPC.publish for message in rpc_incoming.publish: - self.seen_messages.append(message.seqno) - await self.handle_talk(peer_id, message) + if message.seqno not in self.seen_messages: + should_publish = True + self.seen_messages.append(message.seqno) + await self.handle_talk(peer_id, message) if rpc_incoming.subscriptions: @@ -106,13 +112,9 @@ class Pubsub(): # peers because a given node only needs its peers # to know that it is subscribed to the topic (doesn't # need everyone to know) - should_publish = False - - # TODO check that peer_id is the same as origin_id - from_id = str(rpc_incoming.publish[0].from_id.decode('utf-8')) for message in rpc_incoming.subscriptions: if message.subscribe: - self.handle_subscription(from_id, message) + self.handle_subscription(peer_id, message) if should_publish: # relay message to peers with router @@ -182,6 +184,8 @@ class Pubsub(): :param sub_message: RPC.SubOpts """ # TODO verify logic here + + if sub_message.subscribe: if sub_message.topicid not in self.peer_topics: self.peer_topics[sub_message.topicid] = [peer_id] @@ -213,19 +217,23 @@ class Pubsub(): :param topic_id: topic_id to subscribe to """ # Map topic_id to blocking queue + + print ("**PUBSUB** in SUBSCRIBE") self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message packet = rpc_pb2.RPC() - packet.publish.extend([rpc_pb2.Message( - from_id=str(self.host.get_id()).encode('utf-8'), - seqno=str(generate_message_id()).encode('utf-8') - )]) + # packet.publish.extend([rpc_pb2.Message( + # from_id=str(self.host.get_id()).encode('utf-8'), + # seqno=str(generate_message_id()).encode('utf-8') + # )]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe = True, topicid = topic_id.encode('utf-8') )]) - + print (packet) + print ("**PUBSUB** PEEERS") + print (self.peers) # Send out subscribe message to all peers await self.message_all_peers(packet.SerializeToString()) @@ -247,10 +255,10 @@ class Pubsub(): # Create unsubscribe message packet = rpc_pb2.RPC() - packet.publish.extend([rpc_pb2.Message( - from_id=str(self.host.get_id()).encode('utf-8'), - seqno=str(generate_message_id()).encode('utf-8') - )]) + # packet.publish.extend([rpc_pb2.Message( + # from_id=str(self.host.get_id()).encode('utf-8'), + # seqno=str(generate_message_id()).encode('utf-8') + # )]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe = False, topicid = topic_id.encode('utf-8') @@ -267,6 +275,8 @@ class Pubsub(): Broadcast a message to peers :param raw_msg: raw contents of the message to broadcast """ + print ("**PUBSU** IN MESSAGE ALL PEERS") + print (rpc_msg) # Broadcast message for peer in self.peers: diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index eaca614..1951fc6 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -86,7 +86,7 @@ class DummyAccountNode(): my_id = str(self.libp2p_node.get_id()) msg_contents = "send," + source_user + "," + dest_user + "," + str(amount) packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) - await self.floodsub.publish(my_id, msg.SerializeToString()) + await self.floodsub.publish(my_id, packet.SerializeToString()) async def publish_set_crypto(self, user, amount): """ @@ -128,6 +128,8 @@ class DummyAccountNode(): print (dest_user) print (amount) self.balances[dest_user] = amount + print (self.balances) + print ("^^ balance") def get_balance(self, user): """ diff --git a/tests/pubsub/test_dummyaccount_demo.py b/tests/pubsub/test_dummyaccount_demo.py index 9b3a149..1f08c8d 100644 --- a/tests/pubsub/test_dummyaccount_demo.py +++ b/tests/pubsub/test_dummyaccount_demo.py @@ -86,102 +86,102 @@ async def test_simple_two_nodes(): await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_simple_three_nodes_line_topography(): -# num_nodes = 3 -# adj_map = {0: [1], 1: [2]} +@pytest.mark.asyncio +async def test_simple_three_nodes_line_topography(): + num_nodes = 3 + adj_map = {0: [1], 1: [2]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 10) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 10) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 10 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 10 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_simple_three_nodes_triangle_topography(): -# num_nodes = 3 -# adj_map = {0: [1, 2], 1: [2]} +@pytest.mark.asyncio +async def test_simple_three_nodes_triangle_topography(): + num_nodes = 3 + adj_map = {0: [1, 2], 1: [2]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 20) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 20 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_simple_seven_nodes_tree_topography(): -# num_nodes = 7 -# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +@pytest.mark.asyncio +async def test_simple_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 20) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 20 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_set_then_send_from_root_seven_nodes_tree_topography(): -# num_nodes = 7 -# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +@pytest.mark.asyncio +async def test_set_then_send_from_root_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 20) -# await asyncio.sleep(0.25) -# await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) + await asyncio.sleep(0.25) + await dummy_nodes[0].publish_send_crypto("aspyn", "alex", 5) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 15 -# assert dummy_node.get_balance("alex") == 5 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 15 + assert dummy_node.get_balance("alex") == 5 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): -# num_nodes = 7 -# adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} +@pytest.mark.asyncio +async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography(): + num_nodes = 7 + adj_map = {0: [1, 2], 1: [3, 4], 2: [5, 6]} -# async def action_func(dummy_nodes): -# await dummy_nodes[6].publish_set_crypto("aspyn", 20) -# await asyncio.sleep(0.25) -# await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) + async def action_func(dummy_nodes): + await dummy_nodes[6].publish_set_crypto("aspyn", 20) + await asyncio.sleep(0.25) + await dummy_nodes[4].publish_send_crypto("aspyn", "alex", 5) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 15 -# assert dummy_node.get_balance("alex") == 5 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 15 + assert dummy_node.get_balance("alex") == 5 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_simple_five_nodes_ring_topography(): -# num_nodes = 5 -# adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} +@pytest.mark.asyncio +async def test_simple_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("aspyn", 20) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("aspyn", 20) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("aspyn") == 20 + def assertion_func(dummy_node): + assert dummy_node.get_balance("aspyn") == 20 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) -# @pytest.mark.asyncio -# async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): -# num_nodes = 5 -# adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} +@pytest.mark.asyncio +async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} -# async def action_func(dummy_nodes): -# await dummy_nodes[0].publish_set_crypto("alex", 20) -# await asyncio.sleep(0.25) -# await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("alex", 20) + await asyncio.sleep(0.25) + await dummy_nodes[3].publish_send_crypto("alex", "rob", 12) -# def assertion_func(dummy_node): -# assert dummy_node.get_balance("alex") == 8 -# assert dummy_node.get_balance("rob") == 12 + def assertion_func(dummy_node): + assert dummy_node.get_balance("alex") == 8 + assert dummy_node.get_balance("rob") == 12 -# await perform_test(num_nodes, adj_map, action_func, assertion_func) + await perform_test(num_nodes, adj_map, action_func, assertion_func) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index f4a5826..c4c2681 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -1,15 +1,14 @@ import asyncio +import uuid import multiaddr import pytest from tests.utils import cleanup from libp2p import new_node from libp2p.peer.peerinfo import info_from_p2p_addr +from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub -from libp2p.pubsub.message import MessageTalk -from libp2p.pubsub.message import create_message_talk -from libp2p.pubsub.message import generate_message_id # pylint: disable=too-many-locals @@ -22,7 +21,7 @@ async def connect(node1, node2): await node1.connect(info) @pytest.mark.asyncio -async def test_simple_two_nodes(): +async def test_simple_two_nodes_RPC(): node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -45,71 +44,77 @@ async def test_simple_two_nodes(): node_a_id = str(node_a.get_id()) - msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) - - await floodsub_a.publish(node_a.get_id(), msg.to_str()) - + msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", generate_message_id()) + await floodsub_a.publish(node_a_id, msg.SerializeToString()) + print ("MESSAGE B") + print (msg.SerializeToString()) + print ("=========") await asyncio.sleep(0.25) res_b = await qb.get() + print ("RES B") + print (res_b) + print ("-----") # Check that the msg received by node_b is the same # as the message sent by node_a - assert res_b == msg.to_str() + assert res_b.SerializeToString() == msg.publish[0].SerializeToString() + + # Success, terminate pending tasks. await cleanup() -@pytest.mark.asyncio -async def test_simple_three_nodes(): - # Want to pass message from A -> B -> C - node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - node_c = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) +# @pytest.mark.asyncio +# async def test_simple_three_nodes(): +# # Want to pass message from A -> B -> C +# node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) +# node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) +# node_c = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) - await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) - await node_c.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) +# await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) +# await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) +# await node_c.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) - supported_protocols = ["/floodsub/1.0.0"] +# supported_protocols = ["/floodsub/1.0.0"] - floodsub_a = FloodSub(supported_protocols) - pubsub_a = Pubsub(node_a, floodsub_a, "a") - floodsub_b = FloodSub(supported_protocols) - pubsub_b = Pubsub(node_b, floodsub_b, "b") - floodsub_c = FloodSub(supported_protocols) - pubsub_c = Pubsub(node_c, floodsub_c, "c") +# floodsub_a = FloodSub(supported_protocols) +# pubsub_a = Pubsub(node_a, floodsub_a, "a") +# floodsub_b = FloodSub(supported_protocols) +# pubsub_b = Pubsub(node_b, floodsub_b, "b") +# floodsub_c = FloodSub(supported_protocols) +# pubsub_c = Pubsub(node_c, floodsub_c, "c") - await connect(node_a, node_b) - await connect(node_b, node_c) +# await connect(node_a, node_b) +# await connect(node_b, node_c) - await asyncio.sleep(0.25) - qb = await pubsub_b.subscribe("my_topic") - qc = await pubsub_c.subscribe("my_topic") - await asyncio.sleep(0.25) +# await asyncio.sleep(0.25) +# qb = await pubsub_b.subscribe("my_topic") +# qc = await pubsub_c.subscribe("my_topic") +# await asyncio.sleep(0.25) - node_a_id = str(node_a.get_id()) +# node_a_id = str(node_a.get_id()) - msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) +# msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) - await floodsub_a.publish(node_a.get_id(), msg.to_str()) +# await floodsub_a.publish(node_a.get_id(), msg.to_str()) - await asyncio.sleep(0.25) - res_b = await qb.get() - res_c = await qc.get() +# await asyncio.sleep(0.25) +# res_b = await qb.get() +# res_c = await qc.get() - # Check that the msg received by node_b is the same - # as the message sent by node_a - assert res_b == msg.to_str() +# # Check that the msg received by node_b is the same +# # as the message sent by node_a +# assert res_b == msg.to_str() - # res_c should match original msg but with b as sender - node_b_id = str(node_b.get_id()) - msg.from_id = node_b_id +# # res_c should match original msg but with b as sender +# node_b_id = str(node_b.get_id()) +# msg.from_id = node_b_id - assert res_c == msg.to_str() +# assert res_c == msg.to_str() - # Success, terminate pending tasks. - await cleanup() +# # Success, terminate pending tasks. +# await cleanup() async def perform_test_from_obj(obj): """ @@ -237,11 +242,14 @@ async def perform_test_from_obj(obj): actual_node_id = str(node_map[node_id].get_id()) # Create correctly formatted message - msg_talk = MessageTalk(actual_node_id, actual_node_id, topics, data, generate_message_id()) + msg_talk = generate_RPC_packet(actual_node_id, topics, data, generate_message_id()) + print ("**TEST FLOODSUB** MESSAGE TALK") + print (msg_talk) # Publish message # await floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()) - tasks_publish.append(asyncio.ensure_future(floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()))) + tasks_publish.append(asyncio.ensure_future(floodsub_map[node_id].publish(\ + actual_node_id, msg_talk.SerializeToString()))) # For each topic in topics, add topic, msg_talk tuple to ordered test list # TODO: Update message sender to be correct message sender before @@ -261,12 +269,20 @@ async def perform_test_from_obj(obj): for node_id in topic_map[topic]: # Get message from subscription queue msg_on_node_str = await queues_map[node_id][topic].get() - msg_on_node = create_message_talk(msg_on_node_str) + + print ("MESSAGE ON NODE STR") + print (msg_on_node_str) + + print ("ACTUAL MESSSSAGE") + print (actual_msg) + + assert actual_msg.publish[0].SerializeToString() == msg_on_node_str.SerializeToString() + # msg_on_node = create_message_talk(msg_on_node_str) # Perform checks - assert actual_msg.origin_id == msg_on_node.origin_id - assert actual_msg.topics == msg_on_node.topics - assert actual_msg.data == msg_on_node.data + # assert actual_msg.origin_id == msg_on_node.origin_id + # assert actual_msg.topics == msg_on_node.topics + # assert actual_msg.data == msg_on_node.data # Success, terminate pending tasks. await cleanup() @@ -484,3 +500,30 @@ async def test_three_nodes_clique_two_topic_diff_origin_test_obj(): ] } await perform_test_from_obj(test_obj) + +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) + +def generate_RPC_packet(origin_id, topics, msg_content, msg_id): + packet = rpc_pb2.RPC() + message = rpc_pb2.Message( + from_id=origin_id.encode('utf-8'), + seqno=msg_id.encode('utf-8'), + data=msg_content.encode('utf-8'), + ) + for topic in topics: + message.topicIDs.extend([topic.encode('utf-8')]) + + # for topic in topics: + # message.topicIDs.extend([topic.encode('utf-8')]) + # packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( + # subscribe=True, + # topicid = topic.encode('utf-8') + # )]) + + packet.publish.extend([message]) + return packet From 7d3a8b5e77110b57dde5897189ce52dd2cfc86c0 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Mon, 1 Apr 2019 16:55:44 -0400 Subject: [PATCH 109/131] clean up --- libp2p/pubsub/floodsub.py | 25 +----- libp2p/pubsub/pubsub.py | 57 +++---------- libp2p/pubsub/pubsub_router_interface.py | 4 +- tests/pubsub/dummy_account_node.py | 47 +---------- tests/pubsub/test_floodsub.py | 103 +---------------------- tests/pubsub/utils.py | 30 +++++++ 6 files changed, 54 insertions(+), 212 deletions(-) create mode 100644 tests/pubsub/utils.py diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 305607e..a4f2e86 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -1,8 +1,9 @@ -from .pubsub_router_interface import IPubsubRouter from .pb import rpc_pb2 +from .pubsub_router_interface import IPubsubRouter class FloodSub(IPubsubRouter): + # pylint: disable=no-member def __init__(self, protocols): self.protocols = protocols @@ -55,42 +56,22 @@ class FloodSub(IPubsubRouter): :param sender_peer_id: peer_id of message sender :param rpc_message: pubsub message in RPC string format """ - packet = rpc_pb2.RPC() packet.ParseFromString(rpc_message) - print ("IN FLOOODSUB PUBLISH") - print (packet) - print ("++++++++++++++++") msg_sender = str(sender_peer_id) # Deliver to self if self was origin # Note: handle_talk checks if self is subscribed to topics in message for message in packet.publish: decoded_from_id = message.from_id.decode('utf-8') - - print ("MESSAGE SENDER") - print (msg_sender) - print ("FROM ID") - print (message.from_id) - print (str(self.pubsub.host.get_id())) - - if msg_sender == decoded_from_id and msg_sender == str(self.pubsub.host.get_id()): - await self.pubsub.handle_talk(sender_peer_id, message) + await self.pubsub.handle_talk(message) - - print ("OHOHOHOH") - print (self.pubsub.peer_topics) - print ("UUUJUJUJ") - print (self.pubsub.peers) - print ("********") # Deliver to self and peers for topic in message.topicIDs: if topic in self.pubsub.peer_topics: for peer_id_in_topic in self.pubsub.peer_topics[topic]: # Forward to all known peers in the topic that are not the # message sender and are not the message origin - print ("PEERID") - print (peer_id_in_topic) if peer_id_in_topic not in (msg_sender, decoded_from_id): stream = self.pubsub.peers[peer_id_in_topic] # create new packet with just publish message diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index c7bfe06..bfad873 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -1,13 +1,12 @@ import asyncio import uuid -from .pb import rpc_pb2_grpc from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee class Pubsub(): - # pylint: disable=too-many-instance-attributes + # pylint: disable=too-many-instance-attributes, no-member def __init__(self, host, router, my_id): """ @@ -65,12 +64,6 @@ class Pubsub(): for topic_id in self.my_topics: packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( subscribe=True, topicid=topic_id)]) - - # message = rpc_pb2.Message( - # from_id=str(self.host.get_id()).encode('utf-8'), - # seqno=str(generate_message_id()).encode('utf-8') - # ) - # packet.publish.extend([message]) return packet.SerializeToString() @@ -82,19 +75,13 @@ class Pubsub(): """ # TODO check on types here - print ("++++++++++ASPYN+++++++++++++++++") peer_id = str(stream.mplex_conn.peer_id) while True: - print ("HIT ME") incoming = (await stream.read()) rpc_incoming = rpc_pb2.RPC() rpc_incoming.ParseFromString(incoming) - print ("IN PUBSUB CONTINUOUSLY READ") - print (rpc_incoming) - print ("###########################") - should_publish = False if rpc_incoming.publish: @@ -103,8 +90,7 @@ class Pubsub(): if message.seqno not in self.seen_messages: should_publish = True self.seen_messages.append(message.seqno) - await self.handle_talk(peer_id, message) - + await self.handle_talk(message) if rpc_incoming.subscriptions: # deal with RPC.subscriptions @@ -150,10 +136,8 @@ class Pubsub(): pubsub protocols we support """ while True: - print ("PUBSUB HANDLE PEER QUEUE") + peer_id = await self.peer_queue.get() - print (peer_id) - print ("++++++++++++++++++++++++") # Open a stream to peer on existing connection # (we know connection exists since that's the only way @@ -175,7 +159,7 @@ class Pubsub(): # Force context switch await asyncio.sleep(0) - def handle_subscription(self, peer_id, sub_message): + def handle_subscription(self, origin_id, sub_message): """ Handle an incoming subscription message from a peer. Update internal mapping to mark the peer as subscribed or unsubscribed to topics as @@ -183,23 +167,19 @@ class Pubsub(): :param origin_id: id of the peer who subscribe to the message :param sub_message: RPC.SubOpts """ - # TODO verify logic here - - if sub_message.subscribe: if sub_message.topicid not in self.peer_topics: - self.peer_topics[sub_message.topicid] = [peer_id] - elif peer_id not in self.peer_topics[sub_message.topicid]: + self.peer_topics[sub_message.topicid] = [origin_id] + elif origin_id not in self.peer_topics[sub_message.topicid]: # Add peer to topic - self.peer_topics[sub_message.topicid].append(peer_id) + self.peer_topics[sub_message.topicid].append(origin_id) else: # TODO: Remove peer from topic pass - async def handle_talk(self, peer_id, publish_message): + async def handle_talk(self, publish_message): """ Put incoming message from a peer onto my blocking queue - :param peer_id: peer id whom forwarded this message :param talk: RPC.Message format """ @@ -216,9 +196,8 @@ class Pubsub(): Subscribe ourself to a topic :param topic_id: topic_id to subscribe to """ - # Map topic_id to blocking queue - print ("**PUBSUB** in SUBSCRIBE") + # Map topic_id to blocking queue self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message @@ -228,12 +207,10 @@ class Pubsub(): # seqno=str(generate_message_id()).encode('utf-8') # )]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - subscribe = True, - topicid = topic_id.encode('utf-8') + subscribe=True, + topicid=topic_id.encode('utf-8') )]) - print (packet) - print ("**PUBSUB** PEEERS") - print (self.peers) + # Send out subscribe message to all peers await self.message_all_peers(packet.SerializeToString()) @@ -255,13 +232,9 @@ class Pubsub(): # Create unsubscribe message packet = rpc_pb2.RPC() - # packet.publish.extend([rpc_pb2.Message( - # from_id=str(self.host.get_id()).encode('utf-8'), - # seqno=str(generate_message_id()).encode('utf-8') - # )]) packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - subscribe = False, - topicid = topic_id.encode('utf-8') + subscribe=False, + topicid=topic_id.encode('utf-8') )]) # Send out unsubscribe message to all peers @@ -275,8 +248,6 @@ class Pubsub(): Broadcast a message to peers :param raw_msg: raw contents of the message to broadcast """ - print ("**PUBSU** IN MESSAGE ALL PEERS") - print (rpc_msg) # Broadcast message for peer in self.peers: diff --git a/libp2p/pubsub/pubsub_router_interface.py b/libp2p/pubsub/pubsub_router_interface.py index 727b39e..ec5132e 100644 --- a/libp2p/pubsub/pubsub_router_interface.py +++ b/libp2p/pubsub/pubsub_router_interface.py @@ -39,11 +39,11 @@ class IPubsubRouter(ABC): """ @abstractmethod - def publish(self, sender_peer_id, message): + def publish(self, sender_peer_id, rpc_message): """ Invoked to forward a new message that has been validated :param sender_peer_id: peer_id of message sender - :param message: message to forward + :param rpc_message: message to forward """ @abstractmethod diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 1951fc6..05a02bf 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -1,10 +1,8 @@ import asyncio -import uuid import multiaddr +from utils import generate_message_id, generate_RPC_packet from libp2p import new_node -from libp2p.pubsub.pb import rpc_pb2_grpc -from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub @@ -37,7 +35,6 @@ class DummyAccountNode(): We use create as this serves as a factory function and allows us to use async await, unlike the init function """ - print ("**DUMMY** CREATE ACCOUNT NODE") self = DummyAccountNode() libp2p_node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -54,14 +51,9 @@ class DummyAccountNode(): Handle all incoming messages on the CRYPTO_TOPIC from peers """ while True: - incoming = await self.q.get() - print ("**DUMMY** HANDLE INCOMING") - print (incoming) - print ("========================") - + incoming = await self.q.get() msg_comps = incoming.data.decode('utf-8').split(",") - print (msg_comps) - print ("--------") + if msg_comps[0] == "send": self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3])) elif msg_comps[0] == "set": @@ -107,7 +99,6 @@ class DummyAccountNode(): :param dest_user: user to send crypto to :param amount: amount of crypto to send """ - print ("**DUMMY** IN HANDLE SEND CRYPTO") if source_user in self.balances: self.balances[source_user] -= amount else: @@ -124,12 +115,7 @@ class DummyAccountNode(): :param dest_user: user to set crypto for :param amount: amount of crypto """ - print ("**DUMMY** IN HANDLE SET CRYPTO") - print (dest_user) - print (amount) self.balances[dest_user] = amount - print (self.balances) - print ("^^ balance") def get_balance(self, user): """ @@ -137,35 +123,8 @@ class DummyAccountNode(): :param user: user to get balance for :return: balance of user """ - print ("GET BALACNCE") - print (user) - print (self.balances) if user in self.balances: return self.balances[user] else: return -1 -def generate_message_id(): - """ - Generate a unique message id - :return: messgae id - """ - return str(uuid.uuid1()) - -def generate_RPC_packet(origin_id, topics, msg_content, msg_id): - packet = rpc_pb2.RPC() - message = rpc_pb2.Message( - from_id=origin_id.encode('utf-8'), - seqno=msg_id.encode('utf-8'), - data=msg_content.encode('utf-8') - ) - - for topic in topics: - message.topicIDs.extend([topic.encode('utf-8')]) - packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - subscribe=True, - topicid = topic.encode('utf-8') - )]) - - packet.publish.extend([message]) - return packet diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index c4c2681..4fc059b 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -1,5 +1,4 @@ import asyncio -import uuid import multiaddr import pytest @@ -9,6 +8,7 @@ from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub +from utils import generate_message_id, generate_RPC_packet # pylint: disable=too-many-locals @@ -46,16 +46,10 @@ async def test_simple_two_nodes_RPC(): msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", generate_message_id()) await floodsub_a.publish(node_a_id, msg.SerializeToString()) - print ("MESSAGE B") - print (msg.SerializeToString()) - print ("=========") await asyncio.sleep(0.25) res_b = await qb.get() - print ("RES B") - print (res_b) - print ("-----") # Check that the msg received by node_b is the same # as the message sent by node_a assert res_b.SerializeToString() == msg.publish[0].SerializeToString() @@ -65,57 +59,6 @@ async def test_simple_two_nodes_RPC(): # Success, terminate pending tasks. await cleanup() -# @pytest.mark.asyncio -# async def test_simple_three_nodes(): -# # Want to pass message from A -> B -> C -# node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) -# node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) -# node_c = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) - -# await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) -# await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) -# await node_c.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) - -# supported_protocols = ["/floodsub/1.0.0"] - -# floodsub_a = FloodSub(supported_protocols) -# pubsub_a = Pubsub(node_a, floodsub_a, "a") -# floodsub_b = FloodSub(supported_protocols) -# pubsub_b = Pubsub(node_b, floodsub_b, "b") -# floodsub_c = FloodSub(supported_protocols) -# pubsub_c = Pubsub(node_c, floodsub_c, "c") - -# await connect(node_a, node_b) -# await connect(node_b, node_c) - -# await asyncio.sleep(0.25) -# qb = await pubsub_b.subscribe("my_topic") -# qc = await pubsub_c.subscribe("my_topic") -# await asyncio.sleep(0.25) - -# node_a_id = str(node_a.get_id()) - -# msg = MessageTalk(node_a_id, node_a_id, ["my_topic"], "some data", generate_message_id()) - -# await floodsub_a.publish(node_a.get_id(), msg.to_str()) - -# await asyncio.sleep(0.25) -# res_b = await qb.get() -# res_c = await qc.get() - -# # Check that the msg received by node_b is the same -# # as the message sent by node_a -# assert res_b == msg.to_str() - -# # res_c should match original msg but with b as sender -# node_b_id = str(node_b.get_id()) -# msg.from_id = node_b_id - -# assert res_c == msg.to_str() - -# # Success, terminate pending tasks. -# await cleanup() - async def perform_test_from_obj(obj): """ Perform a floodsub test from a test obj. @@ -243,9 +186,7 @@ async def perform_test_from_obj(obj): # Create correctly formatted message msg_talk = generate_RPC_packet(actual_node_id, topics, data, generate_message_id()) - - print ("**TEST FLOODSUB** MESSAGE TALK") - print (msg_talk) + # Publish message # await floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()) tasks_publish.append(asyncio.ensure_future(floodsub_map[node_id].publish(\ @@ -269,20 +210,7 @@ async def perform_test_from_obj(obj): for node_id in topic_map[topic]: # Get message from subscription queue msg_on_node_str = await queues_map[node_id][topic].get() - - print ("MESSAGE ON NODE STR") - print (msg_on_node_str) - - print ("ACTUAL MESSSSAGE") - print (actual_msg) - assert actual_msg.publish[0].SerializeToString() == msg_on_node_str.SerializeToString() - # msg_on_node = create_message_talk(msg_on_node_str) - - # Perform checks - # assert actual_msg.origin_id == msg_on_node.origin_id - # assert actual_msg.topics == msg_on_node.topics - # assert actual_msg.data == msg_on_node.data # Success, terminate pending tasks. await cleanup() @@ -500,30 +428,3 @@ async def test_three_nodes_clique_two_topic_diff_origin_test_obj(): ] } await perform_test_from_obj(test_obj) - -def generate_message_id(): - """ - Generate a unique message id - :return: messgae id - """ - return str(uuid.uuid1()) - -def generate_RPC_packet(origin_id, topics, msg_content, msg_id): - packet = rpc_pb2.RPC() - message = rpc_pb2.Message( - from_id=origin_id.encode('utf-8'), - seqno=msg_id.encode('utf-8'), - data=msg_content.encode('utf-8'), - ) - for topic in topics: - message.topicIDs.extend([topic.encode('utf-8')]) - - # for topic in topics: - # message.topicIDs.extend([topic.encode('utf-8')]) - # packet.subscriptions.extend([rpc_pb2.RPC.SubOpts( - # subscribe=True, - # topicid = topic.encode('utf-8') - # )]) - - packet.publish.extend([message]) - return packet diff --git a/tests/pubsub/utils.py b/tests/pubsub/utils.py new file mode 100644 index 0000000..d4695d7 --- /dev/null +++ b/tests/pubsub/utils.py @@ -0,0 +1,30 @@ +import uuid +from libp2p.pubsub.pb import rpc_pb2 + +def generate_message_id(): + """ + Generate a unique message id + :return: messgae id + """ + return str(uuid.uuid1()) + +def generate_RPC_packet(origin_id, topics, msg_content, msg_id): + """ + Generate RPC packet to send over wire + :param origin_id: peer id of the message origin + :param topics: list of topics + :param msg_content: string of content in data + :param msg_id: seqno for the message + """ + packet = rpc_pb2.RPC() + message = rpc_pb2.Message( + from_id=origin_id.encode('utf-8'), + seqno=msg_id.encode('utf-8'), + data=msg_content.encode('utf-8'), + ) + + for topic in topics: + message.topicIDs.extend([topic.encode('utf-8')]) + + packet.publish.extend([message]) + return packet From 502bb01242eaf69131855bcc7b8d7ac9b673679f Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Tue, 2 Apr 2019 22:05:14 -0400 Subject: [PATCH 110/131] Add priority queues to handle seqno --- libp2p/pubsub/pubsub.py | 14 ++++++++------ libp2p/pubsub/read_only_queue.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 libp2p/pubsub/read_only_queue.py diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index bfad873..37b8714 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -3,6 +3,7 @@ import uuid from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee +from .read_only_queue import ReadOnlyQueue class Pubsub(): @@ -188,8 +189,9 @@ class Pubsub(): if topic in self.my_topics: # we are subscribed to a topic this message was sent for, # so add message to the subscription output queue - # for each topic - await self.my_topics[topic].put(publish_message) + # for each topic with priority being the message's seqno. + # Note: asyncio.PriorityQueue item format is (priority, data) + await self.my_topics[topic].put((publish_message.seqno, publish_message)) async def subscribe(self, topic_id): """ @@ -197,8 +199,8 @@ class Pubsub(): :param topic_id: topic_id to subscribe to """ - # Map topic_id to blocking queue - self.my_topics[topic_id] = asyncio.Queue() + # Map topic_id to a priority blocking queue + self.my_topics[topic_id] = asyncio.PriorityQueue() # Create subscribe message packet = rpc_pb2.RPC() @@ -217,8 +219,8 @@ class Pubsub(): # Tell router we are joining this topic self.router.join(topic_id) - # Return the asyncio queue for messages on this topic - return self.my_topics[topic_id] + # Return the readonly queue for messages on this topic + return ReadOnlyQueue(self.my_topics[topic_id]) async def unsubscribe(self, topic_id): """ diff --git a/libp2p/pubsub/read_only_queue.py b/libp2p/pubsub/read_only_queue.py new file mode 100644 index 0000000..1656768 --- /dev/null +++ b/libp2p/pubsub/read_only_queue.py @@ -0,0 +1,14 @@ +import asyncio + +class ReadOnlyQueue(): + + def __init__(self, queue): + self.queue = queue + + async def get(self): + """ + Get the next item from queue, which has items in the format (priority, data) + :return: next item from the queue + """ + return (await self.queue.get())[1] + From a47b02dd260077e456649808d946f1c6373dbd03 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Tue, 2 Apr 2019 22:05:32 -0400 Subject: [PATCH 111/131] Adjust floodsub tests for new seqno util --- tests/pubsub/test_floodsub.py | 11 ++++++++--- tests/pubsub/utils.py | 22 ++++++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index 4fc059b..ba9d33c 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -8,7 +8,7 @@ from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub -from utils import generate_message_id, generate_RPC_packet +from utils import message_id_generator, generate_RPC_packet # pylint: disable=too-many-locals @@ -44,7 +44,8 @@ async def test_simple_two_nodes_RPC(): node_a_id = str(node_a.get_id()) - msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", generate_message_id()) + next_msg_id_func = message_id_generator(0) + msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", next_msg_id_func()) await floodsub_a.publish(node_a_id, msg.SerializeToString()) await asyncio.sleep(0.25) @@ -175,6 +176,8 @@ async def perform_test_from_obj(obj): topics_in_msgs_ordered = [] messages = obj["messages"] tasks_publish = [] + next_msg_id_func = message_id_generator(0) + for msg in messages: topics = msg["topics"] @@ -185,7 +188,7 @@ async def perform_test_from_obj(obj): actual_node_id = str(node_map[node_id].get_id()) # Create correctly formatted message - msg_talk = generate_RPC_packet(actual_node_id, topics, data, generate_message_id()) + msg_talk = generate_RPC_packet(actual_node_id, topics, data, next_msg_id_func()) # Publish message # await floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()) @@ -207,6 +210,8 @@ async def perform_test_from_obj(obj): # TODO: Check message sender too for i in range(len(topics_in_msgs_ordered)): topic, actual_msg = topics_in_msgs_ordered[i] + + # Look at each node in each topic for node_id in topic_map[topic]: # Get message from subscription queue msg_on_node_str = await queues_map[node_id][topic].get() diff --git a/tests/pubsub/utils.py b/tests/pubsub/utils.py index d4695d7..8aa3f88 100644 --- a/tests/pubsub/utils.py +++ b/tests/pubsub/utils.py @@ -1,12 +1,26 @@ import uuid +import struct from libp2p.pubsub.pb import rpc_pb2 -def generate_message_id(): + +def message_id_generator(start_val): """ Generate a unique message id - :return: messgae id + :param start_val: value to start generating messages at + :return: message id """ - return str(uuid.uuid1()) + val = start_val + def generator(): + # Allow manipulation of val within closure + nonlocal val + + # Increment id + val += 1 + + # Convert val to big endian + return struct.pack('>I', val) + + return generator def generate_RPC_packet(origin_id, topics, msg_content, msg_id): """ @@ -19,7 +33,7 @@ def generate_RPC_packet(origin_id, topics, msg_content, msg_id): packet = rpc_pb2.RPC() message = rpc_pb2.Message( from_id=origin_id.encode('utf-8'), - seqno=msg_id.encode('utf-8'), + seqno=msg_id, data=msg_content.encode('utf-8'), ) From 91dd3c0230636610c3bde51c76596bd4723cd67f Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Tue, 2 Apr 2019 22:34:01 -0400 Subject: [PATCH 112/131] Modify pubsub to have seen message check incorporate seqno and node id --- libp2p/pubsub/pubsub.py | 16 +++++++--------- libp2p/pubsub/read_only_queue.py | 14 -------------- tests/pubsub/dummy_account_node.py | 13 +++++++++---- 3 files changed, 16 insertions(+), 27 deletions(-) delete mode 100644 libp2p/pubsub/read_only_queue.py diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 37b8714..d5d4b52 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -3,7 +3,6 @@ import uuid from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee -from .read_only_queue import ReadOnlyQueue class Pubsub(): @@ -90,7 +89,7 @@ class Pubsub(): for message in rpc_incoming.publish: if message.seqno not in self.seen_messages: should_publish = True - self.seen_messages.append(message.seqno) + self.seen_messages.append((message.seqno, message.from_id)) await self.handle_talk(message) if rpc_incoming.subscriptions: @@ -189,9 +188,8 @@ class Pubsub(): if topic in self.my_topics: # we are subscribed to a topic this message was sent for, # so add message to the subscription output queue - # for each topic with priority being the message's seqno. - # Note: asyncio.PriorityQueue item format is (priority, data) - await self.my_topics[topic].put((publish_message.seqno, publish_message)) + # for each topic + await self.my_topics[topic].put(publish_message) async def subscribe(self, topic_id): """ @@ -199,8 +197,8 @@ class Pubsub(): :param topic_id: topic_id to subscribe to """ - # Map topic_id to a priority blocking queue - self.my_topics[topic_id] = asyncio.PriorityQueue() + # Map topic_id to blocking queue + self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message packet = rpc_pb2.RPC() @@ -219,8 +217,8 @@ class Pubsub(): # Tell router we are joining this topic self.router.join(topic_id) - # Return the readonly queue for messages on this topic - return ReadOnlyQueue(self.my_topics[topic_id]) + # Return the asyncio queue for messages on this topic + return self.my_topics[topic_id] async def unsubscribe(self, topic_id): """ diff --git a/libp2p/pubsub/read_only_queue.py b/libp2p/pubsub/read_only_queue.py deleted file mode 100644 index 1656768..0000000 --- a/libp2p/pubsub/read_only_queue.py +++ /dev/null @@ -1,14 +0,0 @@ -import asyncio - -class ReadOnlyQueue(): - - def __init__(self, queue): - self.queue = queue - - async def get(self): - """ - Get the next item from queue, which has items in the format (priority, data) - :return: next item from the queue - """ - return (await self.queue.get())[1] - diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 05a02bf..8fb7310 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -1,7 +1,8 @@ import asyncio import multiaddr +import uuid -from utils import generate_message_id, generate_RPC_packet +from utils import message_id_generator, generate_RPC_packet from libp2p import new_node from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub @@ -25,6 +26,8 @@ class DummyAccountNode(): def __init__(self): self.balances = {} + self.next_msg_id_func = message_id_generator(0) + self.node_id = str(uuid.uuid1()) @classmethod async def create(cls): @@ -51,7 +54,7 @@ class DummyAccountNode(): Handle all incoming messages on the CRYPTO_TOPIC from peers """ while True: - incoming = await self.q.get() + incoming = await self.q.get() msg_comps = incoming.data.decode('utf-8').split(",") if msg_comps[0] == "send": @@ -77,7 +80,7 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "send," + source_user + "," + dest_user + "," + str(amount) - packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, self.next_msg_id_func()) await self.floodsub.publish(my_id, packet.SerializeToString()) async def publish_set_crypto(self, user, amount): @@ -88,7 +91,7 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "set," + user + "," + str(amount) - packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, self.next_msg_id_func()) await self.floodsub.publish(my_id, packet.SerializeToString()) @@ -99,6 +102,7 @@ class DummyAccountNode(): :param dest_user: user to send crypto to :param amount: amount of crypto to send """ + print("handle send " + self.node_id) if source_user in self.balances: self.balances[source_user] -= amount else: @@ -115,6 +119,7 @@ class DummyAccountNode(): :param dest_user: user to set crypto for :param amount: amount of crypto """ + print("handle set " + self.node_id) self.balances[dest_user] = amount def get_balance(self, user): From b73b2f0841209a88840c111be72b7b52925f049d Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 00:34:39 -0400 Subject: [PATCH 113/131] Fix seen messages bug --- libp2p/pubsub/pubsub.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index d5d4b52..d8fb3c0 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -87,9 +87,10 @@ class Pubsub(): if rpc_incoming.publish: # deal with RPC.publish for message in rpc_incoming.publish: - if message.seqno not in self.seen_messages: + id_in_seen_msgs = (message.seqno, message.from_id) + if id_in_seen_msgs not in self.seen_messages: should_publish = True - self.seen_messages.append((message.seqno, message.from_id)) + self.seen_messages.append(id_in_seen_msgs) await self.handle_talk(message) if rpc_incoming.subscriptions: From 3f2561e30bcd0c8fd0e51d95bec32d1478779b39 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 00:35:39 -0400 Subject: [PATCH 114/131] Add dummy node test --- tests/pubsub/test_dummyaccount_demo.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/pubsub/test_dummyaccount_demo.py b/tests/pubsub/test_dummyaccount_demo.py index 1f08c8d..9fa2aa7 100644 --- a/tests/pubsub/test_dummyaccount_demo.py +++ b/tests/pubsub/test_dummyaccount_demo.py @@ -185,3 +185,28 @@ async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): assert dummy_node.get_balance("rob") == 12 await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_set_then_send_from_five_diff_nodes_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("alex", 20) + await asyncio.sleep(1) + await dummy_nodes[1].publish_send_crypto("alex", "rob", 3) + await asyncio.sleep(1) + await dummy_nodes[2].publish_send_crypto("rob", "aspyn", 2) + await asyncio.sleep(1) + await dummy_nodes[3].publish_send_crypto("aspyn", "zx", 1) + await asyncio.sleep(1) + await dummy_nodes[4].publish_send_crypto("zx", "raul", 1) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("alex") == 17 + assert dummy_node.get_balance("rob") == 1 + assert dummy_node.get_balance("aspyn") == 1 + assert dummy_node.get_balance("zx") == 0 + assert dummy_node.get_balance("raul") == 1 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) From c270bda45602569e4c74f71ca6ed3245223bf909 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 14:17:33 -0400 Subject: [PATCH 115/131] Add test for multiple messages from two origins --- tests/pubsub/test_floodsub.py | 54 +++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index ba9d33c..00ac956 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -433,3 +433,57 @@ async def test_three_nodes_clique_two_topic_diff_origin_test_obj(): ] } await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_four_nodes_clique_two_topic_diff_origin_many_msgs_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2", "3", "4"], + "2": ["1", "3", "4"], + "3": ["1", "2", "4"], + "4": ["1", "2", "3"] + }, + "topic_map": { + "astrophysics": ["1", "2", "3", "4"], + "school": ["1", "2", "3", "4"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar2", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar3", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic3", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) From 0fe21308ae20c5b64d2b2566538715f6e29ce340 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 14:20:50 -0400 Subject: [PATCH 116/131] Add test for ring topology multiple messages from two origins --- tests/pubsub/test_floodsub.py | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index 00ac956..2d4ed8d 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -487,3 +487,58 @@ async def test_four_nodes_clique_two_topic_diff_origin_many_msgs_test_obj(): ] } await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_five_nodes_ring_two_topic_diff_origin_many_msgs_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2"], + "2": ["3"], + "3": ["4"], + "4": ["5"], + "5": ["1"] + }, + "topic_map": { + "astrophysics": ["1", "2", "3", "4", "5"], + "school": ["1", "2", "3", "4", "5"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar2", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar3", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic3", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) \ No newline at end of file From f6299c7dee33cfcb72c42e2c786f0fd6bd54ba9a Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Tue, 2 Apr 2019 22:05:14 -0400 Subject: [PATCH 117/131] Add priority queues to handle seqno --- libp2p/pubsub/pubsub.py | 14 ++++++++------ libp2p/pubsub/read_only_queue.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 libp2p/pubsub/read_only_queue.py diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index bee5fba..5f018d7 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -2,6 +2,7 @@ import asyncio from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee +from .read_only_queue import ReadOnlyQueue class Pubsub(): @@ -187,8 +188,9 @@ class Pubsub(): if topic in self.my_topics: # we are subscribed to a topic this message was sent for, # so add message to the subscription output queue - # for each topic - await self.my_topics[topic].put(publish_message) + # for each topic with priority being the message's seqno. + # Note: asyncio.PriorityQueue item format is (priority, data) + await self.my_topics[topic].put((publish_message.seqno, publish_message)) async def subscribe(self, topic_id): """ @@ -196,8 +198,8 @@ class Pubsub(): :param topic_id: topic_id to subscribe to """ - # Map topic_id to blocking queue - self.my_topics[topic_id] = asyncio.Queue() + # Map topic_id to a priority blocking queue + self.my_topics[topic_id] = asyncio.PriorityQueue() # Create subscribe message packet = rpc_pb2.RPC() @@ -212,8 +214,8 @@ class Pubsub(): # Tell router we are joining this topic self.router.join(topic_id) - # Return the asyncio queue for messages on this topic - return self.my_topics[topic_id] + # Return the readonly queue for messages on this topic + return ReadOnlyQueue(self.my_topics[topic_id]) async def unsubscribe(self, topic_id): """ diff --git a/libp2p/pubsub/read_only_queue.py b/libp2p/pubsub/read_only_queue.py new file mode 100644 index 0000000..1656768 --- /dev/null +++ b/libp2p/pubsub/read_only_queue.py @@ -0,0 +1,14 @@ +import asyncio + +class ReadOnlyQueue(): + + def __init__(self, queue): + self.queue = queue + + async def get(self): + """ + Get the next item from queue, which has items in the format (priority, data) + :return: next item from the queue + """ + return (await self.queue.get())[1] + From c2b538936224b18169a036ca5ee46b64d0a027ea Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Tue, 2 Apr 2019 22:05:32 -0400 Subject: [PATCH 118/131] Adjust floodsub tests for new seqno util --- tests/pubsub/test_floodsub.py | 11 ++++++++--- tests/pubsub/utils.py | 22 ++++++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index 272af79..4e01237 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -8,7 +8,7 @@ from libp2p.peer.peerinfo import info_from_p2p_addr from libp2p.pubsub.pb import rpc_pb2 from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub -from utils import generate_message_id, generate_RPC_packet +from utils import message_id_generator, generate_RPC_packet # pylint: disable=too-many-locals @@ -44,7 +44,8 @@ async def test_simple_two_nodes_RPC(): node_a_id = str(node_a.get_id()) - msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", generate_message_id()) + next_msg_id_func = message_id_generator(0) + msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data", next_msg_id_func()) await floodsub_a.publish(node_a_id, msg.SerializeToString()) await asyncio.sleep(0.25) @@ -173,6 +174,8 @@ async def perform_test_from_obj(obj): topics_in_msgs_ordered = [] messages = obj["messages"] tasks_publish = [] + next_msg_id_func = message_id_generator(0) + for msg in messages: topics = msg["topics"] @@ -183,7 +186,7 @@ async def perform_test_from_obj(obj): actual_node_id = str(node_map[node_id].get_id()) # Create correctly formatted message - msg_talk = generate_RPC_packet(actual_node_id, topics, data, generate_message_id()) + msg_talk = generate_RPC_packet(actual_node_id, topics, data, next_msg_id_func()) # Publish message # await floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str()) @@ -205,6 +208,8 @@ async def perform_test_from_obj(obj): # TODO: Check message sender too for i in range(len(topics_in_msgs_ordered)): topic, actual_msg = topics_in_msgs_ordered[i] + + # Look at each node in each topic for node_id in topic_map[topic]: # Get message from subscription queue msg_on_node_str = await queues_map[node_id][topic].get() diff --git a/tests/pubsub/utils.py b/tests/pubsub/utils.py index d4695d7..8aa3f88 100644 --- a/tests/pubsub/utils.py +++ b/tests/pubsub/utils.py @@ -1,12 +1,26 @@ import uuid +import struct from libp2p.pubsub.pb import rpc_pb2 -def generate_message_id(): + +def message_id_generator(start_val): """ Generate a unique message id - :return: messgae id + :param start_val: value to start generating messages at + :return: message id """ - return str(uuid.uuid1()) + val = start_val + def generator(): + # Allow manipulation of val within closure + nonlocal val + + # Increment id + val += 1 + + # Convert val to big endian + return struct.pack('>I', val) + + return generator def generate_RPC_packet(origin_id, topics, msg_content, msg_id): """ @@ -19,7 +33,7 @@ def generate_RPC_packet(origin_id, topics, msg_content, msg_id): packet = rpc_pb2.RPC() message = rpc_pb2.Message( from_id=origin_id.encode('utf-8'), - seqno=msg_id.encode('utf-8'), + seqno=msg_id, data=msg_content.encode('utf-8'), ) From 9d16aa834d3220d85b6d087d85e2da4e555536a9 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Tue, 2 Apr 2019 22:34:01 -0400 Subject: [PATCH 119/131] Modify pubsub to have seen message check incorporate seqno and node id --- libp2p/pubsub/pubsub.py | 16 +++++++--------- libp2p/pubsub/read_only_queue.py | 14 -------------- tests/pubsub/dummy_account_node.py | 13 +++++++++---- 3 files changed, 16 insertions(+), 27 deletions(-) delete mode 100644 libp2p/pubsub/read_only_queue.py diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 5f018d7..aec04de 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -2,7 +2,6 @@ import asyncio from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee -from .read_only_queue import ReadOnlyQueue class Pubsub(): @@ -89,7 +88,7 @@ class Pubsub(): for message in rpc_incoming.publish: if message.seqno not in self.seen_messages: should_publish = True - self.seen_messages.append(message.seqno) + self.seen_messages.append((message.seqno, message.from_id)) await self.handle_talk(message) if rpc_incoming.subscriptions: @@ -188,9 +187,8 @@ class Pubsub(): if topic in self.my_topics: # we are subscribed to a topic this message was sent for, # so add message to the subscription output queue - # for each topic with priority being the message's seqno. - # Note: asyncio.PriorityQueue item format is (priority, data) - await self.my_topics[topic].put((publish_message.seqno, publish_message)) + # for each topic + await self.my_topics[topic].put(publish_message) async def subscribe(self, topic_id): """ @@ -198,8 +196,8 @@ class Pubsub(): :param topic_id: topic_id to subscribe to """ - # Map topic_id to a priority blocking queue - self.my_topics[topic_id] = asyncio.PriorityQueue() + # Map topic_id to blocking queue + self.my_topics[topic_id] = asyncio.Queue() # Create subscribe message packet = rpc_pb2.RPC() @@ -214,8 +212,8 @@ class Pubsub(): # Tell router we are joining this topic self.router.join(topic_id) - # Return the readonly queue for messages on this topic - return ReadOnlyQueue(self.my_topics[topic_id]) + # Return the asyncio queue for messages on this topic + return self.my_topics[topic_id] async def unsubscribe(self, topic_id): """ diff --git a/libp2p/pubsub/read_only_queue.py b/libp2p/pubsub/read_only_queue.py deleted file mode 100644 index 1656768..0000000 --- a/libp2p/pubsub/read_only_queue.py +++ /dev/null @@ -1,14 +0,0 @@ -import asyncio - -class ReadOnlyQueue(): - - def __init__(self, queue): - self.queue = queue - - async def get(self): - """ - Get the next item from queue, which has items in the format (priority, data) - :return: next item from the queue - """ - return (await self.queue.get())[1] - diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 05a02bf..8fb7310 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -1,7 +1,8 @@ import asyncio import multiaddr +import uuid -from utils import generate_message_id, generate_RPC_packet +from utils import message_id_generator, generate_RPC_packet from libp2p import new_node from libp2p.pubsub.pubsub import Pubsub from libp2p.pubsub.floodsub import FloodSub @@ -25,6 +26,8 @@ class DummyAccountNode(): def __init__(self): self.balances = {} + self.next_msg_id_func = message_id_generator(0) + self.node_id = str(uuid.uuid1()) @classmethod async def create(cls): @@ -51,7 +54,7 @@ class DummyAccountNode(): Handle all incoming messages on the CRYPTO_TOPIC from peers """ while True: - incoming = await self.q.get() + incoming = await self.q.get() msg_comps = incoming.data.decode('utf-8').split(",") if msg_comps[0] == "send": @@ -77,7 +80,7 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "send," + source_user + "," + dest_user + "," + str(amount) - packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, self.next_msg_id_func()) await self.floodsub.publish(my_id, packet.SerializeToString()) async def publish_set_crypto(self, user, amount): @@ -88,7 +91,7 @@ class DummyAccountNode(): """ my_id = str(self.libp2p_node.get_id()) msg_contents = "set," + user + "," + str(amount) - packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, generate_message_id()) + packet = generate_RPC_packet(my_id, [CRYPTO_TOPIC], msg_contents, self.next_msg_id_func()) await self.floodsub.publish(my_id, packet.SerializeToString()) @@ -99,6 +102,7 @@ class DummyAccountNode(): :param dest_user: user to send crypto to :param amount: amount of crypto to send """ + print("handle send " + self.node_id) if source_user in self.balances: self.balances[source_user] -= amount else: @@ -115,6 +119,7 @@ class DummyAccountNode(): :param dest_user: user to set crypto for :param amount: amount of crypto """ + print("handle set " + self.node_id) self.balances[dest_user] = amount def get_balance(self, user): From feaa393c5ff1201a67c2fb54bf04b9c0922e2c98 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 00:34:39 -0400 Subject: [PATCH 120/131] Fix seen messages bug --- libp2p/pubsub/pubsub.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index aec04de..40c4036 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -86,9 +86,10 @@ class Pubsub(): if rpc_incoming.publish: # deal with RPC.publish for message in rpc_incoming.publish: - if message.seqno not in self.seen_messages: + id_in_seen_msgs = (message.seqno, message.from_id) + if id_in_seen_msgs not in self.seen_messages: should_publish = True - self.seen_messages.append((message.seqno, message.from_id)) + self.seen_messages.append(id_in_seen_msgs) await self.handle_talk(message) if rpc_incoming.subscriptions: From 211d6f6860bbf3df2f9e70f59ba04ff07b138947 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 00:35:39 -0400 Subject: [PATCH 121/131] Add dummy node test --- tests/pubsub/test_dummyaccount_demo.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/pubsub/test_dummyaccount_demo.py b/tests/pubsub/test_dummyaccount_demo.py index 1f08c8d..9fa2aa7 100644 --- a/tests/pubsub/test_dummyaccount_demo.py +++ b/tests/pubsub/test_dummyaccount_demo.py @@ -185,3 +185,28 @@ async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography(): assert dummy_node.get_balance("rob") == 12 await perform_test(num_nodes, adj_map, action_func, assertion_func) + +@pytest.mark.asyncio +async def test_set_then_send_from_five_diff_nodes_five_nodes_ring_topography(): + num_nodes = 5 + adj_map = {0: [1], 1: [2], 2: [3], 3: [4], 4: [0]} + + async def action_func(dummy_nodes): + await dummy_nodes[0].publish_set_crypto("alex", 20) + await asyncio.sleep(1) + await dummy_nodes[1].publish_send_crypto("alex", "rob", 3) + await asyncio.sleep(1) + await dummy_nodes[2].publish_send_crypto("rob", "aspyn", 2) + await asyncio.sleep(1) + await dummy_nodes[3].publish_send_crypto("aspyn", "zx", 1) + await asyncio.sleep(1) + await dummy_nodes[4].publish_send_crypto("zx", "raul", 1) + + def assertion_func(dummy_node): + assert dummy_node.get_balance("alex") == 17 + assert dummy_node.get_balance("rob") == 1 + assert dummy_node.get_balance("aspyn") == 1 + assert dummy_node.get_balance("zx") == 0 + assert dummy_node.get_balance("raul") == 1 + + await perform_test(num_nodes, adj_map, action_func, assertion_func) From fc789280370a47206738e8f94811af775f7260da Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 14:17:33 -0400 Subject: [PATCH 122/131] Add test for multiple messages from two origins --- tests/pubsub/test_floodsub.py | 54 +++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index 4e01237..00c68b5 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -431,3 +431,57 @@ async def test_three_nodes_clique_two_topic_diff_origin_test_obj(): ] } await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_four_nodes_clique_two_topic_diff_origin_many_msgs_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2", "3", "4"], + "2": ["1", "3", "4"], + "3": ["1", "2", "4"], + "4": ["1", "2", "3"] + }, + "topic_map": { + "astrophysics": ["1", "2", "3", "4"], + "school": ["1", "2", "3", "4"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar2", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar3", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic3", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) From 22e503a2606ba4f3cf7212908301fcd74970be34 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Wed, 3 Apr 2019 14:20:50 -0400 Subject: [PATCH 123/131] Add test for ring topology multiple messages from two origins --- tests/pubsub/test_floodsub.py | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index 00c68b5..69b2f8c 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -485,3 +485,58 @@ async def test_four_nodes_clique_two_topic_diff_origin_many_msgs_test_obj(): ] } await perform_test_from_obj(test_obj) + +@pytest.mark.asyncio +async def test_five_nodes_ring_two_topic_diff_origin_many_msgs_test_obj(): + test_obj = { + "supported_protocols": ["/floodsub/1.0.0"], + "adj_list": { + "1": ["2"], + "2": ["3"], + "3": ["4"], + "4": ["5"], + "5": ["1"] + }, + "topic_map": { + "astrophysics": ["1", "2", "3", "4", "5"], + "school": ["1", "2", "3", "4", "5"] + }, + "messages": [ + { + "topics": ["astrophysics"], + "data": "e=mc^2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar2", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic2", + "node_id": "1" + }, + { + "topics": ["school"], + "data": "foobar3", + "node_id": "2" + }, + { + "topics": ["astrophysics"], + "data": "I am allergic3", + "node_id": "1" + } + ] + } + await perform_test_from_obj(test_obj) \ No newline at end of file From 3c0fca8979c647aaacc1287bf8133961456b2c0f Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Thu, 4 Apr 2019 17:10:41 -0400 Subject: [PATCH 124/131] Remove prints --- tests/pubsub/dummy_account_node.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/pubsub/dummy_account_node.py b/tests/pubsub/dummy_account_node.py index 8fb7310..a640997 100644 --- a/tests/pubsub/dummy_account_node.py +++ b/tests/pubsub/dummy_account_node.py @@ -102,7 +102,6 @@ class DummyAccountNode(): :param dest_user: user to send crypto to :param amount: amount of crypto to send """ - print("handle send " + self.node_id) if source_user in self.balances: self.balances[source_user] -= amount else: @@ -119,7 +118,6 @@ class DummyAccountNode(): :param dest_user: user to set crypto for :param amount: amount of crypto """ - print("handle set " + self.node_id) self.balances[dest_user] = amount def get_balance(self, user): From b4aa64bc3956b92b5457689b592c9e3f27c13409 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Fri, 5 Apr 2019 17:30:35 -0400 Subject: [PATCH 125/131] Change >I to >Q for 64-bit big endian int --- tests/pubsub/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pubsub/utils.py b/tests/pubsub/utils.py index 8aa3f88..056baac 100644 --- a/tests/pubsub/utils.py +++ b/tests/pubsub/utils.py @@ -18,7 +18,7 @@ def message_id_generator(start_val): val += 1 # Convert val to big endian - return struct.pack('>I', val) + return struct.pack('>Q', val) return generator From 686c55b09c56c4403112cdf8b4185e937dc6d198 Mon Sep 17 00:00:00 2001 From: Stuckinaboot Date: Fri, 5 Apr 2019 17:38:29 -0400 Subject: [PATCH 126/131] Remove unnecessary print #146 --- libp2p/pubsub/floodsub.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index a4f2e86..14394c7 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -74,13 +74,12 @@ class FloodSub(IPubsubRouter): # message sender and are not the message origin if peer_id_in_topic not in (msg_sender, decoded_from_id): stream = self.pubsub.peers[peer_id_in_topic] - # create new packet with just publish message + # Create new packet with just publish message new_packet = rpc_pb2.RPC() new_packet.publish.extend([message]) + + # Publish the packet await stream.write(new_packet.SerializeToString()) - else: - # Implies publish did not write - print("publish did not write") def join(self, topic): """ From 1f3aaded1d89301dac03179aae86913a1586d2e4 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Fri, 5 Apr 2019 21:45:31 -0400 Subject: [PATCH 127/131] add lru-dict as requirement --- requirements_dev.txt | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index e3d8c0e..3d5ca3c 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -5,3 +5,4 @@ pytest-asyncio pylint grpcio grpcio-tools +lru-dict>=1.1.6 \ No newline at end of file diff --git a/setup.py b/setup.py index c44a276..0c62145 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,8 @@ setuptools.setup( "pymultihash", "multiaddr", "grpcio", - "grpcio-tools" + "grpcio-tools", + "lru-dict>=1.1.6" ], packages=["libp2p"], zip_safe=False, From d04798ce7c95c22e9fb448e1fe359426a43a0bdb Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Fri, 5 Apr 2019 21:46:03 -0400 Subject: [PATCH 128/131] lru cache seen_messages --- libp2p/pubsub/pubsub.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 40c4036..2109a9c 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -1,4 +1,6 @@ +# pylint: disable=no-name-in-module import asyncio +from lru import LRU from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee @@ -7,7 +9,7 @@ from .pubsub_notifee import PubsubNotifee class Pubsub(): # pylint: disable=too-many-instance-attributes, no-member - def __init__(self, host, router, my_id): + def __init__(self, host, router, my_id, cache_size=None): """ Construct a new Pubsub object, which is responsible for handling all Pubsub-related messages and relaying messages as appropriate to the @@ -37,8 +39,12 @@ class Pubsub(): self.incoming_msgs_from_peers = asyncio.Queue() self.outgoing_messages = asyncio.Queue() - # TODO: Make seen_messages a cache (LRU cache?) - self.seen_messages = [] + # keeps track of seen messages as LRU cache + if cache_size is None: + self.cache_size = 128 + else: + self.cache_size = cache_size + self.seen_messages = LRU(self.cache_size) # Map of topics we are subscribed to to handler functions # for when the given topic receives a message @@ -89,7 +95,7 @@ class Pubsub(): id_in_seen_msgs = (message.seqno, message.from_id) if id_in_seen_msgs not in self.seen_messages: should_publish = True - self.seen_messages.append(id_in_seen_msgs) + self.seen_messages[id_in_seen_msgs] = 1 await self.handle_talk(message) if rpc_incoming.subscriptions: From 7a298adc3309181f9639ead20242d1d4f1f8d266 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Fri, 5 Apr 2019 21:46:18 -0400 Subject: [PATCH 129/131] add simple lru test --- tests/pubsub/test_floodsub.py | 76 ++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index 69b2f8c..06c1cbd 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -21,7 +21,7 @@ async def connect(node1, node2): await node1.connect(info) @pytest.mark.asyncio -async def test_simple_two_nodes_RPC(): +async def test_simple_two_nodes(): node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) @@ -58,6 +58,80 @@ async def test_simple_two_nodes_RPC(): # Success, terminate pending tasks. await cleanup() +@pytest.mark.asyncio +async def test_lru_cache_two_nodes(): + # two nodes with cache_size of 4 + # node_a send the following messages to node_b + # [1, 1, 2, 1, 3, 1, 4, 1, 5, 1] + # node_b should only receive the following + # [1, 2, 3, 4, 5, 1] + node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"]) + + await node_a.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + await node_b.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")) + + supported_protocols = ["/floodsub/1.0.0"] + + # initialize PubSub with a cache_size of 4 + floodsub_a = FloodSub(supported_protocols) + pubsub_a = Pubsub(node_a, floodsub_a, "a", 4) + floodsub_b = FloodSub(supported_protocols) + pubsub_b = Pubsub(node_b, floodsub_b, "b", 4) + + await connect(node_a, node_b) + + await asyncio.sleep(0.25) + qb = await pubsub_b.subscribe("my_topic") + + await asyncio.sleep(0.25) + + node_a_id = str(node_a.get_id()) + + # initialize message_id_generator + # store first message + next_msg_id_func = message_id_generator(0) + first_message = generate_RPC_packet(node_a_id, ["my_topic"], "some data 1", next_msg_id_func()) + + await floodsub_a.publish(node_a_id, first_message.SerializeToString()) + await asyncio.sleep(0.25) + print (first_message) + + messages = [first_message] + # for the next 5 messages + for i in range(2, 6): + # write first message + await floodsub_a.publish(node_a_id, first_message.SerializeToString()) + await asyncio.sleep(0.25) + + # generate and write next message + msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data " + str(i), next_msg_id_func()) + messages.append(msg) + + await floodsub_a.publish(node_a_id, msg.SerializeToString()) + await asyncio.sleep(0.25) + + # write first message again + await floodsub_a.publish(node_a_id, first_message.SerializeToString()) + await asyncio.sleep(0.25) + + # check the first five messages in queue + # should only see 1 first_message + for i in range(5): + # Check that the msg received by node_b is the same + # as the message sent by node_a + res_b = await qb.get() + assert res_b.SerializeToString() == messages[i].publish[0].SerializeToString() + + # the 6th message should be first_message + res_b = await qb.get() + assert res_b.SerializeToString() == first_message.publish[0].SerializeToString() + assert qb.empty() + + # Success, terminate pending tasks. + await cleanup() + + async def perform_test_from_obj(obj): """ Perform a floodsub test from a test obj. From c8908f7e673e9dfa0349990c5ff404ef45c3aabe Mon Sep 17 00:00:00 2001 From: Robert Zajac Date: Tue, 9 Apr 2019 20:50:21 -0400 Subject: [PATCH 130/131] keep lru_dict in setup.py only --- requirements_dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 3d5ca3c..e3d8c0e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -5,4 +5,3 @@ pytest-asyncio pylint grpcio grpcio-tools -lru-dict>=1.1.6 \ No newline at end of file From 3ee75f0ecd5b7f1089c8bf7924240585d98a9305 Mon Sep 17 00:00:00 2001 From: Alex Haynes Date: Wed, 17 Apr 2019 21:03:35 -0400 Subject: [PATCH 131/131] updated travis config --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a2df56d..a5dfefb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ install: script: - pytest --cov=./libp2p tests/ - - pylint --rcfile=.pylintrc libp2p/!(kademlia) tests + - pylint --rcfile=.pylintrc libp2p tests after_success: - codecov