Merge branch 'magniff-feature/issue-104'

This commit is contained in:
zixuanzh 2019-01-10 20:46:44 +08:00
commit 7c2b2800a8
71 changed files with 154 additions and 107 deletions

View File

@ -5,16 +5,17 @@ python:
install:
- pip install --upgrade pip
- pip install -r requirements.txt
- pip install "pytest>=3.6"
- pip install codecov pytest-cov pytest-asyncio pylint
- python setup.py develop
script:
- pytest --cov=./ -v
- pylint --rcfile=.pylintrc encryption host libp2p network peer protocol_muxer stream_muxer transport tests
- pytest --cov=./libp2p tests/
- pylint --rcfile=.pylintrc libp2p/!(kademlia) tests
after_success:
- codecov
notifications:
slack: py-libp2p:RK0WVoQZhQXLgIKfHNPL1TR2

View File

View File

@ -1,15 +1,10 @@
#!/usr/bin/env python3
import asyncio
import sys
from os.path import abspath, dirname
import click
from libp2p.libp2p import *
from network.multiaddr import MultiAddr
from peer.peerinfo import info_from_p2p_addr
sys.path.append(dirname(dirname(dirname(abspath(__file__)))))
from libp2p import new_node
from libp2p.peer.peerinfo import info_from_p2p_addr
PROTOCOL_ID = '/chat/1.0.0'

View File

@ -0,0 +1,38 @@
from Crypto.PublicKey import RSA
import multiaddr
from .peer.peerstore import PeerStore
from .peer.id import id_from_public_key
from .network.swarm import Swarm
from .host.basic_host import BasicHost
from .transport.upgrader import TransportUpgrader
from .transport.tcp.tcp import TCP
async def new_node(
id_opt=None, transport_opt=None,
muxer_opt=None, sec_opt=None, peerstore=None):
if id_opt is None:
new_key = RSA.generate(2048, e=65537)
id_opt = id_from_public_key(new_key.publickey())
# private_key = new_key.exportKey("PEM")
transport_opt = transport_opt or ["/ip4/127.0.0.1/tcp/8001"]
transport_opt = [multiaddr.Multiaddr(t) for t in transport_opt]
muxer_opt = muxer_opt or ["mplex/6.7.0"]
sec_opt = sec_opt or ["secio"]
peerstore = peerstore or PeerStore()
upgrader = TransportUpgrader(sec_opt, transport_opt)
swarm = Swarm(id_opt, peerstore, upgrader)
tcp = TCP()
swarm.add_transport(tcp)
await swarm.listen(transport_opt[0])
# TODO enable support for other host type
# TODO routing unimplemented
host = BasicHost(swarm)
return host

View File

@ -1,13 +1,13 @@
from collections import Counter
import logging
from kademlia.node import Node, NodeHeap
from kademlia.utils import gather_dict
from .kademlia.node import Node, NodeHeap
from .kademlia.utils import gather_dict
log = logging.getLogger(__name__)
class SpiderCrawl(object):
class SpiderCrawl:
"""
Crawl the network and look for given 160-bit keys.
"""
@ -148,7 +148,7 @@ class NodeSpiderCrawl(SpiderCrawl):
return await self.find()
class RPCFindResponse(object):
class RPCFindResponse:
def __init__(self, response):
"""
A wrapper for the result of a RPC find.

View File

@ -6,17 +6,17 @@ import pickle
import asyncio
import logging
from kademlia.protocol import KademliaProtocol
from kademlia.utils import digest
from kademlia.storage import ForgetfulStorage
from kademlia.node import Node
from kademlia.crawling import ValueSpiderCrawl
from kademlia.crawling import NodeSpiderCrawl
from .kademlia.protocol import KademliaProtocol
from .kademlia.utils import digest
from .kademlia.storage import ForgetfulStorage
from .kademlia.node import Node
from .kademlia.crawling import ValueSpiderCrawl
from .kademlia.crawling import NodeSpiderCrawl
log = logging.getLogger(__name__)
class Server(object):
class Server:
"""
High level view of a node instance. This is the object that should be
created to start listening as an active node on the network.

View File

@ -31,7 +31,7 @@ class Node:
return "%s:%s" % (self.ip, str(self.port))
class NodeHeap(object):
class NodeHeap:
"""
A heap of nodes ordered by distance to a given node.
"""

View File

@ -4,9 +4,9 @@ import logging
from rpcudp.protocol import RPCProtocol
from kademlia.node import Node
from kademlia.routing import RoutingTable
from kademlia.utils import digest
from .kademlia.node import Node
from .kademlia.routing import RoutingTable
from .kademlia.utils import digest
log = logging.getLogger(__name__)

View File

@ -4,10 +4,11 @@ import operator
import asyncio
from collections import OrderedDict
from kademlia.utils import OrderedSet, sharedPrefix, bytesToBitString
from .kademlia.utils import OrderedSet, sharedPrefix, bytesToBitString
class KBucket(object):
class KBucket:
def __init__(self, rangeLower, rangeUpper, ksize):
self.range = (rangeLower, rangeUpper)
self.nodes = OrderedDict()
@ -79,7 +80,7 @@ class KBucket(object):
return len(self.nodes)
class TableTraverser(object):
class TableTraverser:
def __init__(self, table, startNode):
index = table.getBucketFor(startNode)
table.buckets[index].touchLastUpdated()
@ -111,7 +112,7 @@ class TableTraverser(object):
raise StopIteration
class RoutingTable(object):
class RoutingTable:
def __init__(self, protocol, ksize, node):
"""
@param node: The node that represents this server. It won't

View File

@ -1,35 +0,0 @@
from Crypto.PublicKey import RSA
import multiaddr
from peer.peerstore import PeerStore
from peer.id import id_from_public_key
from network.swarm import Swarm
from host.basic_host import BasicHost
from transport.upgrader import TransportUpgrader
from transport.tcp.tcp import TCP
async def new_node(id_opt=None, transport_opt=None, \
muxer_opt=None, sec_opt=None, peerstore=None):
if id_opt is None:
new_key = RSA.generate(2048, e=65537)
id_opt = id_from_public_key(new_key.publickey())
# private_key = new_key.exportKey("PEM")
transport_opt = transport_opt or ["/ip4/127.0.0.1/tcp/8001"]
transport_opt = [multiaddr.Multiaddr(t) for t in transport_opt]
muxer_opt = muxer_opt or ["mplex/6.7.0"]
sec_opt = sec_opt or ["secio"]
peerstore = peerstore or PeerStore()
upgrader = TransportUpgrader(sec_opt, transport_opt)
swarm = Swarm(id_opt, peerstore, upgrader)
tcp = TCP()
swarm.add_transport(tcp)
await swarm.listen(transport_opt[0])
# TODO enable support for other host type
# TODO routing unimplemented
host = BasicHost(swarm)
return host

View File

@ -2,6 +2,7 @@ from abc import ABC
# pylint: disable=too-few-public-methods
class IRawConnection(ABC):
"""
A Raw Connection provides a Reader and a Writer

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod
class INetStream(ABC):
@abstractmethod

View File

@ -1,5 +1,7 @@
from protocol_muxer.multiselect_client import MultiselectClient
from protocol_muxer.multiselect import Multiselect
from libp2p.protocol_muxer.multiselect_client import MultiselectClient
from libp2p.protocol_muxer.multiselect import Multiselect
from .network_interface import INetwork
from .stream.net_stream import NetStream
from .connection.raw_connection import RawConnection

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod
class IAddrBook(ABC):
def __init__(self):
@ -44,4 +45,3 @@ class IAddrBook(ABC):
"""
:return: all of the peer IDs stored with addresses
"""

View File

@ -1,5 +1,6 @@
from .peerdata_interface import IPeerData
class PeerData(IPeerData):
def __init__(self):
@ -33,5 +34,6 @@ class PeerData(IPeerData):
return self.metadata[key]
raise PeerDataError("key not found")
class PeerDataError(KeyError):
"""Raised when a key is not found in peer metadata"""

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod
class IPeerData(ABC):
@abstractmethod

View File

@ -1,7 +1,8 @@
import multiaddr
import multiaddr.util
from peer.id import id_b58_decode
from peer.peerdata import PeerData
from .id import id_b58_decode
from .peerdata import PeerData
class PeerInfo:

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod
class IPeerMetadata(ABC):
def __init__(self):
@ -22,4 +23,3 @@ class IPeerMetadata(ABC):
:param val: value to associated with key
:raise Exception: unsuccessful put
"""

View File

@ -2,6 +2,7 @@ from .peerstore_interface import IPeerStore
from .peerdata import PeerData
from .peerinfo import PeerInfo
class PeerStore(IPeerStore):
def __init__(self):
@ -83,5 +84,6 @@ class PeerStore(IPeerStore):
output.append(key)
return output
class PeerStoreError(KeyError):
"""Raised when peer ID is not found in peer store"""

View File

@ -1,7 +1,9 @@
from abc import abstractmethod
from .addrbook_interface import IAddrBook
from .peermetadata_interface import IPeerMetadata
class IPeerStore(IAddrBook, IPeerMetadata):
def __init__(self):

View File

@ -1,9 +1,11 @@
from .multiselect_muxer_interface import IMultiselectMuxer
from .multiselect_communicator import MultiselectCommunicator
MULTISELECT_PROTOCOL_ID = "/multistream/1.0.0"
PROTOCOL_NOT_FOUND_MSG = "na"
class Multiselect(IMultiselectMuxer):
"""
Multiselect module that is responsible for responding to
@ -89,5 +91,6 @@ def validate_handshake(handshake_contents):
# is added
return handshake_contents == MULTISELECT_PROTOCOL_ID
class MultiselectError(ValueError):
"""Raised when an error occurs in multiselect process"""

View File

@ -1,9 +1,11 @@
from .multiselect_client_interface import IMultiselectClient
from .multiselect_communicator import MultiselectCommunicator
MULTISELECT_PROTOCOL_ID = "/multistream/1.0.0"
PROTOCOL_NOT_FOUND_MSG = "na"
class MultiselectClient(IMultiselectClient):
"""
Client for communicating with receiver's multiselect

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod
class IMultiselectClient(ABC):
"""
Client for communicating with receiver's multiselect

View File

@ -1,5 +1,6 @@
from .multiselect_communicator_interface import IMultiselectCommunicator
class MultiselectCommunicator(IMultiselectCommunicator):
"""
Communicator helper class that ensures both the client

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod
class IMultiselectCommunicator(ABC):
"""
Communicator helper class that ensures both the client

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod
class IMultiselectMuxer(ABC):
"""
Multiselect module that is responsible for responding to

View File

@ -1,4 +1,5 @@
import asyncio
from .utils import encode_uvarint, decode_uvarint_from_stream
from .mplex_stream import MplexStream
from ..muxed_connection_interface import IMuxedConn

View File

@ -1,4 +1,5 @@
import asyncio
from .constants import HEADER_TAGS
from ..muxed_stream_interface import IMuxedStream

View File

@ -1,6 +1,7 @@
import asyncio
import struct
def encode_uvarint(number):
"""Pack `number` into varint bytes"""
buf = b''
@ -14,6 +15,7 @@ def encode_uvarint(number):
break
return buf
def decode_uvarint(buff, index):
shift = 0
result = 0
@ -27,6 +29,7 @@ def decode_uvarint(buff, index):
return result, index + 1
async def decode_uvarint_from_stream(reader):
shift = 0
result = 0

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod
class IMuxedStream(ABC):
@abstractmethod

View File

@ -2,9 +2,10 @@ import asyncio
import multiaddr
from network.connection.raw_connection import RawConnection
from transport.listener_interface import IListener
from transport.transport_interface import ITransport
from libp2p.network.connection.raw_connection import RawConnection
from ..listener_interface import IListener
from ..transport_interface import ITransport
class TCP(ITransport):

View File

@ -1,7 +1,7 @@
from stream_muxer.mplex.mplex import Mplex
from libp2p.stream_muxer.mplex.mplex import Mplex
class TransportUpgrader():
class TransportUpgrader:
# pylint: disable=no-self-use
def __init__(self, secOpt, muxerOpt):

View File

@ -1,9 +0,0 @@
asyncio
pylint
pytest
pycryptodome
pytest-asyncio
click
base58
pymultihash
multiaddr

28
setup.py Normal file
View File

@ -0,0 +1,28 @@
import setuptools
classifiers = [
(
"Programming Language :: Python :: %s" % version
)
for version in "3.5 3.6".split()
]
setuptools.setup(
name="libp2p",
description="libp2p implementation written in python",
version="0.0.1",
license="MIT/APACHE2.0",
platforms=["unix", "linux", "osx"],
classifiers=classifiers,
install_requires=[
"pycryptodome",
"click",
"base58",
"pymultihash",
"multiaddr",
],
packages=["libp2p"],
zip_safe=False,
)

View File

@ -1,14 +1,9 @@
#!/bin/env python3
import asyncio
import sys
import click
import multiaddr
import pytest
from libp2p.libp2p import *
from peer.peerinfo import info_from_p2p_addr
from protocol_muxer.multiselect_client import MultiselectClientError
from libp2p import new_node
from libp2p.peer.peerinfo import info_from_p2p_addr
from libp2p.protocol_muxer.multiselect_client import MultiselectClientError
PROTOCOL_ID = '/chat/1.0.0'

View File

@ -1,8 +1,8 @@
import multiaddr
import pytest
from libp2p.libp2p import new_node
from peer.peerinfo import info_from_p2p_addr
from libp2p import new_node
from libp2p.peer.peerinfo import info_from_p2p_addr
@pytest.mark.asyncio

View File

@ -1,13 +1,14 @@
import asyncio
import pytest
async def handle_echo(reader, writer):
data = await reader.read(100)
writer.write(data)
await writer.drain()
writer.close()
@pytest.mark.asyncio
# TODO: this test should develop out into a fuller test between MPlex
# modules communicating with each other.

View File

@ -1,7 +1,7 @@
import pytest
from peer.peerstore import PeerStoreError
from peer.peerstore import PeerStore
from libp2p.peer.peerstore import PeerStoreError
from libp2p.peer.peerstore import PeerStore
# Testing methods from IAddrBook base class.

View File

@ -1,5 +1,6 @@
from Crypto.PublicKey import RSA
from peer.id import id_from_private_key, id_from_public_key
from libp2p.peer.id import id_from_private_key, id_from_public_key
def test_id_from_private_key():

View File

@ -1,5 +1,6 @@
import multiaddr
from peer.peerinfo import info_from_p2p_addr
from libp2p.peer.peerinfo import info_from_p2p_addr
def test_info_from_p2p_addr():

View File

@ -1,10 +1,12 @@
import pytest
from peer.peerstore import PeerStoreError
from peer.peerstore import PeerStore
from libp2p.peer.peerstore import PeerStoreError
from libp2p.peer.peerstore import PeerStore
# Testing methods from IPeerMetadata base class.
def test_get_empty():
with pytest.raises(PeerStoreError):
store = PeerStore()

View File

@ -1,4 +1,4 @@
from peer.peerstore import PeerStore
from libp2p.peer.peerstore import PeerStore
# Testing methods from IPeerStore base class.

View File

@ -1,7 +1,7 @@
import pytest
from libp2p.libp2p import new_node
from protocol_muxer.multiselect_client import MultiselectClientError
from libp2p import new_node
from libp2p.protocol_muxer.multiselect_client import MultiselectClientError
# TODO: Add tests for multiple streams being opened on different
# protocols through the same connection

View File

@ -2,7 +2,7 @@ import asyncio
import pytest
from transport.tcp.tcp import _multiaddr_from_socket
from libp2p.transport.tcp.tcp import _multiaddr_from_socket
@pytest.mark.asyncio