Migrate to new project structure.

This commit is contained in:
Alexander Koshkin 2019-01-09 21:38:56 +03:00
parent 880ae748d1
commit ce6ddb27a9
69 changed files with 132 additions and 100 deletions

View File

@ -4,15 +4,16 @@ python:
- "3.6" - "3.6"
install: install:
- pip install -r requirements.txt
- pip install codecov pytest pytest-cov - pip install codecov pytest pytest-cov
- python setup.py develop
script: script:
- pytest --cov=./ -v - py.test --cov=./libp2p tests/
- pylint --rcfile=.pylintrc encryption host libp2p network peer stream_muxer transport tests - pylint --rcfile=.pylintrc libp2p tests
after_success: after_success:
- codecov - codecov
notifications: notifications:
slack: py-libp2p:RK0WVoQZhQXLgIKfHNPL1TR2 slack: py-libp2p:RK0WVoQZhQXLgIKfHNPL1TR2

View File

View File

@ -0,0 +1,38 @@
from Crypto.PublicKey import RSA
import multiaddr
from .peer.peerstore import PeerStore
from .peer.id import id_from_public_key
from .network.swarm import Swarm
from .host.basic_host import BasicHost
from .transport.upgrader import TransportUpgrader
from .transport.tcp.tcp import TCP
async def new_node(
id_opt=None, transport_opt=None,
muxer_opt=None, sec_opt=None, peerstore=None):
if id_opt is None:
new_key = RSA.generate(2048, e=65537)
id_opt = id_from_public_key(new_key.publickey())
# private_key = new_key.exportKey("PEM")
transport_opt = transport_opt or ["/ip4/127.0.0.1/tcp/8001"]
transport_opt = [multiaddr.Multiaddr(t) for t in transport_opt]
muxer_opt = muxer_opt or ["mplex/6.7.0"]
sec_opt = sec_opt or ["secio"]
peerstore = peerstore or PeerStore()
upgrader = TransportUpgrader(sec_opt, transport_opt)
swarm = Swarm(id_opt, peerstore, upgrader)
tcp = TCP()
swarm.add_transport(tcp)
await swarm.listen(transport_opt[0])
# TODO enable support for other host type
# TODO routing unimplemented
host = BasicHost(swarm)
return host

View File

@ -1,13 +1,13 @@
from collections import Counter from collections import Counter
import logging import logging
from kademlia.node import Node, NodeHeap from .kademlia.node import Node, NodeHeap
from kademlia.utils import gather_dict from .kademlia.utils import gather_dict
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class SpiderCrawl(object): class SpiderCrawl:
""" """
Crawl the network and look for given 160-bit keys. Crawl the network and look for given 160-bit keys.
""" """
@ -148,7 +148,7 @@ class NodeSpiderCrawl(SpiderCrawl):
return await self.find() return await self.find()
class RPCFindResponse(object): class RPCFindResponse:
def __init__(self, response): def __init__(self, response):
""" """
A wrapper for the result of a RPC find. A wrapper for the result of a RPC find.

View File

@ -6,17 +6,17 @@ import pickle
import asyncio import asyncio
import logging import logging
from kademlia.protocol import KademliaProtocol from .kademlia.protocol import KademliaProtocol
from kademlia.utils import digest from .kademlia.utils import digest
from kademlia.storage import ForgetfulStorage from .kademlia.storage import ForgetfulStorage
from kademlia.node import Node from .kademlia.node import Node
from kademlia.crawling import ValueSpiderCrawl from .kademlia.crawling import ValueSpiderCrawl
from kademlia.crawling import NodeSpiderCrawl from .kademlia.crawling import NodeSpiderCrawl
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class Server(object): class Server:
""" """
High level view of a node instance. This is the object that should be High level view of a node instance. This is the object that should be
created to start listening as an active node on the network. created to start listening as an active node on the network.

View File

@ -31,7 +31,7 @@ class Node:
return "%s:%s" % (self.ip, str(self.port)) return "%s:%s" % (self.ip, str(self.port))
class NodeHeap(object): class NodeHeap:
""" """
A heap of nodes ordered by distance to a given node. A heap of nodes ordered by distance to a given node.
""" """

View File

@ -4,9 +4,9 @@ import logging
from rpcudp.protocol import RPCProtocol from rpcudp.protocol import RPCProtocol
from kademlia.node import Node from .kademlia.node import Node
from kademlia.routing import RoutingTable from .kademlia.routing import RoutingTable
from kademlia.utils import digest from .kademlia.utils import digest
log = logging.getLogger(__name__) log = logging.getLogger(__name__)

View File

@ -4,10 +4,11 @@ import operator
import asyncio import asyncio
from collections import OrderedDict from collections import OrderedDict
from kademlia.utils import OrderedSet, sharedPrefix, bytesToBitString
from .kademlia.utils import OrderedSet, sharedPrefix, bytesToBitString
class KBucket(object): class KBucket:
def __init__(self, rangeLower, rangeUpper, ksize): def __init__(self, rangeLower, rangeUpper, ksize):
self.range = (rangeLower, rangeUpper) self.range = (rangeLower, rangeUpper)
self.nodes = OrderedDict() self.nodes = OrderedDict()
@ -79,7 +80,7 @@ class KBucket(object):
return len(self.nodes) return len(self.nodes)
class TableTraverser(object): class TableTraverser:
def __init__(self, table, startNode): def __init__(self, table, startNode):
index = table.getBucketFor(startNode) index = table.getBucketFor(startNode)
table.buckets[index].touchLastUpdated() table.buckets[index].touchLastUpdated()
@ -111,7 +112,7 @@ class TableTraverser(object):
raise StopIteration raise StopIteration
class RoutingTable(object): class RoutingTable:
def __init__(self, protocol, ksize, node): def __init__(self, protocol, ksize, node):
""" """
@param node: The node that represents this server. It won't @param node: The node that represents this server. It won't

View File

@ -1,35 +0,0 @@
from Crypto.PublicKey import RSA
import multiaddr
from peer.peerstore import PeerStore
from peer.id import id_from_public_key
from network.swarm import Swarm
from host.basic_host import BasicHost
from transport.upgrader import TransportUpgrader
from transport.tcp.tcp import TCP
async def new_node(id_opt=None, transport_opt=None, \
muxer_opt=None, sec_opt=None, peerstore=None):
if id_opt is None:
new_key = RSA.generate(2048, e=65537)
id_opt = id_from_public_key(new_key.publickey())
# private_key = new_key.exportKey("PEM")
transport_opt = transport_opt or ["/ip4/127.0.0.1/tcp/8001"]
transport_opt = [multiaddr.Multiaddr(t) for t in transport_opt]
muxer_opt = muxer_opt or ["mplex/6.7.0"]
sec_opt = sec_opt or ["secio"]
peerstore = peerstore or PeerStore()
upgrader = TransportUpgrader(sec_opt, transport_opt)
swarm = Swarm(id_opt, peerstore, upgrader)
tcp = TCP()
swarm.add_transport(tcp)
await swarm.listen(transport_opt[0])
# TODO enable support for other host type
# TODO routing unimplemented
host = BasicHost(swarm)
return host

View File

@ -2,6 +2,7 @@ from abc import ABC
# pylint: disable=too-few-public-methods # pylint: disable=too-few-public-methods
class IRawConnection(ABC): class IRawConnection(ABC):
""" """
A Raw Connection provides a Reader and a Writer A Raw Connection provides a Reader and a Writer

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
class INetStream(ABC): class INetStream(ABC):
@abstractmethod @abstractmethod

View File

@ -1,5 +1,7 @@
from protocol_muxer.multiselect_client import MultiselectClient from libp2p.protocol_muxer.multiselect_client import MultiselectClient
from protocol_muxer.multiselect import Multiselect from libp2p.protocol_muxer.multiselect import Multiselect
from .network_interface import INetwork from .network_interface import INetwork
from .stream.net_stream import NetStream from .stream.net_stream import NetStream
from .connection.raw_connection import RawConnection from .connection.raw_connection import RawConnection

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
class IAddrBook(ABC): class IAddrBook(ABC):
def __init__(self): def __init__(self):
@ -44,4 +45,4 @@ class IAddrBook(ABC):
""" """
:return: all of the peer IDs stored with addresses :return: all of the peer IDs stored with addresses
""" """

View File

@ -1,5 +1,6 @@
from .peerdata_interface import IPeerData from .peerdata_interface import IPeerData
class PeerData(IPeerData): class PeerData(IPeerData):
def __init__(self): def __init__(self):
@ -33,5 +34,6 @@ class PeerData(IPeerData):
return self.metadata[key] return self.metadata[key]
raise PeerDataError("key not found") raise PeerDataError("key not found")
class PeerDataError(KeyError): class PeerDataError(KeyError):
"""Raised when a key is not found in peer metadata""" """Raised when a key is not found in peer metadata"""

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
class IPeerData(ABC): class IPeerData(ABC):
@abstractmethod @abstractmethod

View File

@ -1,11 +1,12 @@
import multiaddr import multiaddr
import multiaddr.util import multiaddr.util
from peer.id import id_b58_decode
from peer.peerdata import PeerData from .id import id_b58_decode
from .peerdata import PeerData
class PeerInfo: class PeerInfo:
# pylint: disable=too-few-public-methods # pylint: disable=too-few-public-methods
def __init__(self, peer_id, peer_data): def __init__(self, peer_id, peer_data):
self.peer_id = peer_id self.peer_id = peer_id
self.addrs = peer_data.get_addrs() self.addrs = peer_data.get_addrs()

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
class IPeerMetadata(ABC): class IPeerMetadata(ABC):
def __init__(self): def __init__(self):
@ -22,4 +23,3 @@ class IPeerMetadata(ABC):
:param val: value to associated with key :param val: value to associated with key
:raise Exception: unsuccessful put :raise Exception: unsuccessful put
""" """

View File

@ -2,6 +2,7 @@ from .peerstore_interface import IPeerStore
from .peerdata import PeerData from .peerdata import PeerData
from .peerinfo import PeerInfo from .peerinfo import PeerInfo
class PeerStore(IPeerStore): class PeerStore(IPeerStore):
def __init__(self): def __init__(self):
@ -83,5 +84,6 @@ class PeerStore(IPeerStore):
output.append(key) output.append(key)
return output return output
class PeerStoreError(KeyError): class PeerStoreError(KeyError):
"""Raised when peer ID is not found in peer store""" """Raised when peer ID is not found in peer store"""

View File

@ -1,7 +1,9 @@
from abc import abstractmethod from abc import abstractmethod
from .addrbook_interface import IAddrBook from .addrbook_interface import IAddrBook
from .peermetadata_interface import IPeerMetadata from .peermetadata_interface import IPeerMetadata
class IPeerStore(IAddrBook, IPeerMetadata): class IPeerStore(IAddrBook, IPeerMetadata):
def __init__(self): def __init__(self):

View File

@ -1,9 +1,11 @@
from .multiselect_muxer_interface import IMultiselectMuxer from .multiselect_muxer_interface import IMultiselectMuxer
from .multiselect_communicator import MultiselectCommunicator from .multiselect_communicator import MultiselectCommunicator
MULTISELECT_PROTOCOL_ID = "/multistream/1.0.0" MULTISELECT_PROTOCOL_ID = "/multistream/1.0.0"
PROTOCOL_NOT_FOUND_MSG = "na" PROTOCOL_NOT_FOUND_MSG = "na"
class Multiselect(IMultiselectMuxer): class Multiselect(IMultiselectMuxer):
""" """
Multiselect module that is responsible for responding to Multiselect module that is responsible for responding to
@ -88,5 +90,6 @@ class Multiselect(IMultiselectMuxer):
# is added # is added
return handshake_contents == MULTISELECT_PROTOCOL_ID return handshake_contents == MULTISELECT_PROTOCOL_ID
class MultiselectError(ValueError): class MultiselectError(ValueError):
"""Raised when an error occurs in multiselect process""" """Raised when an error occurs in multiselect process"""

View File

@ -1,17 +1,16 @@
from .multiselect_client_interface import IMultiselectClient from .multiselect_client_interface import IMultiselectClient
from .multiselect_communicator import MultiselectCommunicator from .multiselect_communicator import MultiselectCommunicator
MULTISELECT_PROTOCOL_ID = "/multistream/1.0.0" MULTISELECT_PROTOCOL_ID = "/multistream/1.0.0"
PROTOCOL_NOT_FOUND_MSG = "na" PROTOCOL_NOT_FOUND_MSG = "na"
class MultiselectClient(IMultiselectClient): class MultiselectClient(IMultiselectClient):
""" """
Client for communicating with receiver's multiselect Client for communicating with receiver's multiselect
module in order to select a protocol id to communicate over module in order to select a protocol id to communicate over
""" """
def __init__(self):
pass
async def handshake(self, communicator): async def handshake(self, communicator):
""" """
@ -117,5 +116,6 @@ class MultiselectClient(IMultiselectClient):
else: else:
raise MultiselectClientError("unrecognized response: " + response) raise MultiselectClientError("unrecognized response: " + response)
class MultiselectClientError(ValueError): class MultiselectClientError(ValueError):
"""Raised when an error occurs in protocol selection process""" """Raised when an error occurs in protocol selection process"""

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
class IMultiselectClient(ABC): class IMultiselectClient(ABC):
""" """
Client for communicating with receiver's multiselect Client for communicating with receiver's multiselect

View File

@ -1,12 +1,13 @@
from .multiselect_communicator_interface import IMultiselectCommunicator from .multiselect_communicator_interface import IMultiselectCommunicator
class MultiselectCommunicator(IMultiselectCommunicator): class MultiselectCommunicator(IMultiselectCommunicator):
""" """
Communicator helper class that ensures both the client Communicator helper class that ensures both the client
and multistream module will follow the same multistream protocol, and multistream module will follow the same multistream protocol,
which is necessary for them to work which is necessary for them to work
""" """
def __init__(self, stream): def __init__(self, stream):
self.stream = stream self.stream = stream

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
class IMultiselectCommunicator(ABC): class IMultiselectCommunicator(ABC):
""" """
Communicator helper class that ensures both the client Communicator helper class that ensures both the client

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
class IMultiselectMuxer(ABC): class IMultiselectMuxer(ABC):
""" """
Multiselect module that is responsible for responding to Multiselect module that is responsible for responding to

View File

@ -1,4 +1,5 @@
import asyncio import asyncio
from .utils import encode_uvarint, decode_uvarint_from_stream from .utils import encode_uvarint, decode_uvarint_from_stream
from .mplex_stream import MplexStream from .mplex_stream import MplexStream
from ..muxed_connection_interface import IMuxedConn from ..muxed_connection_interface import IMuxedConn

View File

@ -1,4 +1,5 @@
import asyncio import asyncio
from .constants import HEADER_TAGS from .constants import HEADER_TAGS
from ..muxed_stream_interface import IMuxedStream from ..muxed_stream_interface import IMuxedStream

View File

@ -1,6 +1,7 @@
import asyncio import asyncio
import struct import struct
def encode_uvarint(number): def encode_uvarint(number):
"""Pack `number` into varint bytes""" """Pack `number` into varint bytes"""
buf = b'' buf = b''
@ -14,6 +15,7 @@ def encode_uvarint(number):
break break
return buf return buf
def decode_uvarint(buff, index): def decode_uvarint(buff, index):
shift = 0 shift = 0
result = 0 result = 0
@ -27,6 +29,7 @@ def decode_uvarint(buff, index):
return result, index + 1 return result, index + 1
async def decode_uvarint_from_stream(reader): async def decode_uvarint_from_stream(reader):
shift = 0 shift = 0
result = 0 result = 0

View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
class IMuxedStream(ABC): class IMuxedStream(ABC):
@abstractmethod @abstractmethod

View File

@ -2,9 +2,10 @@ import asyncio
import multiaddr import multiaddr
from network.connection.raw_connection import RawConnection from libp2p.network.connection.raw_connection import RawConnection
from transport.listener_interface import IListener
from transport.transport_interface import ITransport from ..listener_interface import IListener
from ..transport_interface import ITransport
class TCP(ITransport): class TCP(ITransport):

View File

@ -1,7 +1,7 @@
from stream_muxer.mplex.mplex import Mplex from libp2p.stream_muxer.mplex.mplex import Mplex
class TransportUpgrader(): class TransportUpgrader:
# pylint: disable=no-self-use # pylint: disable=no-self-use
def __init__(self, secOpt, muxerOpt): def __init__(self, secOpt, muxerOpt):

View File

@ -3,9 +3,9 @@ import setuptools
classifiers = [ classifiers = [
( (
"Programming Language :: Python :: %s" % x "Programming Language :: Python :: %s" % version
) )
for x in "3.5 3.6".split() for version in "3.5 3.6".split()
] ]
@ -13,7 +13,7 @@ setuptools.setup(
name="libp2p", name="libp2p",
description="libp2p implementation written in python", description="libp2p implementation written in python",
version="0.0.1", version="0.0.1",
license="MIT", license="MIT/APACHE2.0",
platforms=["unix", "linux", "osx"], platforms=["unix", "linux", "osx"],
classifiers=classifiers, classifiers=classifiers,
install_requires=[ install_requires=[
@ -24,8 +24,8 @@ setuptools.setup(
"click", "click",
"base58", "base58",
"pymultihash", "pymultihash",
"multiaddr",
], ],
packages="libp2p", packages=["libp2p"],
zip_safe=False, zip_safe=False,
) )

View File

@ -1,14 +1,9 @@
#!/bin/env python3
import asyncio
import sys
import click
import multiaddr
import pytest import pytest
from libp2p.libp2p import * from libp2p import new_node
from peer.peerinfo import info_from_p2p_addr from libp2p.peer.peerinfo import info_from_p2p_addr
from protocol_muxer.multiselect_client import MultiselectClientError from libp2p.protocol_muxer.multiselect_client import MultiselectClientError
PROTOCOL_ID = '/chat/1.0.0' PROTOCOL_ID = '/chat/1.0.0'

View File

@ -1,8 +1,8 @@
import multiaddr import multiaddr
import pytest import pytest
from libp2p.libp2p import new_node from libp2p import new_node
from peer.peerinfo import info_from_p2p_addr from libp2p.peer.peerinfo import info_from_p2p_addr
@pytest.mark.asyncio @pytest.mark.asyncio

View File

@ -1,13 +1,14 @@
import asyncio import asyncio
import pytest import pytest
async def handle_echo(reader, writer): async def handle_echo(reader, writer):
data = await reader.read(100) data = await reader.read(100)
writer.write(data) writer.write(data)
await writer.drain() await writer.drain()
writer.close() writer.close()
@pytest.mark.asyncio @pytest.mark.asyncio
# TODO: this test should develop out into a fuller test between MPlex # TODO: this test should develop out into a fuller test between MPlex
# modules communicating with each other. # modules communicating with each other.

View File

@ -1,7 +1,7 @@
import pytest import pytest
from peer.peerstore import PeerStoreError from libp2p.peer.peerstore import PeerStoreError
from peer.peerstore import PeerStore from libp2p.peer.peerstore import PeerStore
# Testing methods from IAddrBook base class. # Testing methods from IAddrBook base class.

View File

@ -1,5 +1,6 @@
from Crypto.PublicKey import RSA from Crypto.PublicKey import RSA
from peer.id import id_from_private_key, id_from_public_key
from libp2p.peer.id import id_from_private_key, id_from_public_key
def test_id_from_private_key(): def test_id_from_private_key():

View File

@ -1,5 +1,6 @@
import multiaddr import multiaddr
from peer.peerinfo import info_from_p2p_addr
from libp2p.peer.peerinfo import info_from_p2p_addr
def test_info_from_p2p_addr(): def test_info_from_p2p_addr():

View File

@ -1,10 +1,12 @@
import pytest import pytest
from peer.peerstore import PeerStoreError
from peer.peerstore import PeerStore from libp2p.peer.peerstore import PeerStoreError
from libp2p.peer.peerstore import PeerStore
# Testing methods from IPeerMetadata base class. # Testing methods from IPeerMetadata base class.
def test_get_empty(): def test_get_empty():
with pytest.raises(PeerStoreError): with pytest.raises(PeerStoreError):
store = PeerStore() store = PeerStore()

View File

@ -1,4 +1,4 @@
from peer.peerstore import PeerStore from libp2p.peer.peerstore import PeerStore
# Testing methods from IPeerStore base class. # Testing methods from IPeerStore base class.

View File

@ -1,7 +1,7 @@
import pytest import pytest
from libp2p.libp2p import new_node from libp2p import new_node
from protocol_muxer.multiselect_client import MultiselectClientError from libp2p.protocol_muxer.multiselect_client import MultiselectClientError
# TODO: Add tests for multiple streams being opened on different # TODO: Add tests for multiple streams being opened on different
# protocols through the same connection # protocols through the same connection

View File

@ -2,7 +2,7 @@ import asyncio
import pytest import pytest
from transport.tcp.tcp import _multiaddr_from_socket from libp2p.transport.tcp.tcp import _multiaddr_from_socket
@pytest.mark.asyncio @pytest.mark.asyncio