From 006002f68706d67263324e8c2c44737d50aaf0e1 Mon Sep 17 00:00:00 2001 From: mhchia Date: Mon, 23 Sep 2019 22:00:40 +0800 Subject: [PATCH 1/5] Move interop tests out of tests It is moved to the top level package `tests_interop`, to avoid circular dependency, with the dependency moved to `tox`. --- .travis.yml | 2 +- Makefile | 2 +- setup.py | 2 -- {tests/interop => tests_interop}/__init__.py | 0 {tests/interop => tests_interop}/conftest.py | 35 ++++++++++++++++++- {tests/interop => tests_interop}/constants.py | 0 {tests/interop => tests_interop}/daemon.py | 0 {tests/interop => tests_interop}/envs.py | 0 .../go_pkgs/examples/README.md | 0 .../go_pkgs/examples/echo/main.go | 0 .../go_pkgs/examples/go.mod | 0 .../go_pkgs/examples/go.sum | 0 .../go_pkgs/examples/utils/host.go | 0 .../go_pkgs/install_interop_go_pkgs.sh | 0 .../test_bindings.py | 0 {tests/interop => tests_interop}/test_echo.py | 0 .../test_net_stream.py | 0 .../interop => tests_interop}/test_pubsub.py | 0 {tests/interop => tests_interop}/utils.py | 0 tox.ini | 20 +++++++---- 20 files changed, 50 insertions(+), 11 deletions(-) rename {tests/interop => tests_interop}/__init__.py (100%) rename {tests/interop => tests_interop}/conftest.py (83%) rename {tests/interop => tests_interop}/constants.py (100%) rename {tests/interop => tests_interop}/daemon.py (100%) rename {tests/interop => tests_interop}/envs.py (100%) rename {tests/interop => tests_interop}/go_pkgs/examples/README.md (100%) rename {tests/interop => tests_interop}/go_pkgs/examples/echo/main.go (100%) rename {tests/interop => tests_interop}/go_pkgs/examples/go.mod (100%) rename {tests/interop => tests_interop}/go_pkgs/examples/go.sum (100%) rename {tests/interop => tests_interop}/go_pkgs/examples/utils/host.go (100%) rename {tests/interop => tests_interop}/go_pkgs/install_interop_go_pkgs.sh (100%) rename {tests/interop => tests_interop}/test_bindings.py (100%) rename {tests/interop => tests_interop}/test_echo.py (100%) rename {tests/interop => tests_interop}/test_net_stream.py (100%) rename {tests/interop => tests_interop}/test_pubsub.py (100%) rename {tests/interop => tests_interop}/utils.py (100%) diff --git a/.travis.yml b/.travis.yml index f5d470d..824fa24 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ matrix: - export GOPATH=$HOME/go - export GOROOT=/usr/local/go - export PATH=$GOROOT/bin:$GOPATH/bin:$PATH - - ./tests/interop/go_pkgs/install_interop_go_pkgs.sh + - ./tests_interop/go_pkgs/install_interop_go_pkgs.sh install: - pip install --upgrade pip diff --git a/Makefile b/Makefile index 4541270..59422bf 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -FILES_TO_LINT = libp2p tests examples setup.py +FILES_TO_LINT = libp2p tests tests_interop examples setup.py PB = libp2p/crypto/pb/crypto.proto libp2p/pubsub/pb/rpc.proto libp2p/security/insecure/pb/plaintext.proto libp2p/security/secio/pb/spipe.proto PY = $(PB:.proto=_pb2.py) PYI = $(PB:.proto=_pb2.pyi) diff --git a/setup.py b/setup.py index e3ef58e..1567c03 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,6 @@ extras_require = { "pytest>=4.6.3,<5.0.0", "pytest-asyncio>=0.10.0,<1.0.0", "pexpect>=4.6,<5", - # FIXME: Master branch. Use PyPI instead after it is released. - "p2pclient @ git+https://git@github.com/mhchia/py-libp2p-daemon-bindings@628266f", ], "lint": [ "mypy>=0.701,<1.0", diff --git a/tests/interop/__init__.py b/tests_interop/__init__.py similarity index 100% rename from tests/interop/__init__.py rename to tests_interop/__init__.py diff --git a/tests/interop/conftest.py b/tests_interop/conftest.py similarity index 83% rename from tests/interop/conftest.py rename to tests_interop/conftest.py index c280a4c..ff4266f 100644 --- a/tests/interop/conftest.py +++ b/tests_interop/conftest.py @@ -7,13 +7,46 @@ import pexpect import pytest from libp2p.io.abc import ReadWriteCloser -from tests.factories import FloodsubFactory, GossipsubFactory, PubsubFactory +from tests.configs import LISTEN_MADDR +from tests.factories import ( + FloodsubFactory, + GossipsubFactory, + HostFactory, + PubsubFactory, +) from tests.pubsub.configs import GOSSIPSUB_PARAMS from .daemon import Daemon, make_p2pd from .utils import connect +@pytest.fixture +def is_host_secure(): + return False + + +@pytest.fixture +def num_hosts(): + return 3 + + +@pytest.fixture +async def hosts(num_hosts, is_host_secure): + _hosts = HostFactory.create_batch(num_hosts, is_secure=is_host_secure) + await asyncio.gather( + *[_host.get_network().listen(LISTEN_MADDR) for _host in _hosts] + ) + try: + yield _hosts + finally: + # TODO: It's possible that `close` raises exceptions currently, + # due to the connection reset things. Though we don't care much about that when + # cleaning up the tasks, it is probably better to handle the exceptions properly. + await asyncio.gather( + *[_host.close() for _host in _hosts], return_exceptions=True + ) + + @pytest.fixture def proc_factory(): procs = [] diff --git a/tests/interop/constants.py b/tests_interop/constants.py similarity index 100% rename from tests/interop/constants.py rename to tests_interop/constants.py diff --git a/tests/interop/daemon.py b/tests_interop/daemon.py similarity index 100% rename from tests/interop/daemon.py rename to tests_interop/daemon.py diff --git a/tests/interop/envs.py b/tests_interop/envs.py similarity index 100% rename from tests/interop/envs.py rename to tests_interop/envs.py diff --git a/tests/interop/go_pkgs/examples/README.md b/tests_interop/go_pkgs/examples/README.md similarity index 100% rename from tests/interop/go_pkgs/examples/README.md rename to tests_interop/go_pkgs/examples/README.md diff --git a/tests/interop/go_pkgs/examples/echo/main.go b/tests_interop/go_pkgs/examples/echo/main.go similarity index 100% rename from tests/interop/go_pkgs/examples/echo/main.go rename to tests_interop/go_pkgs/examples/echo/main.go diff --git a/tests/interop/go_pkgs/examples/go.mod b/tests_interop/go_pkgs/examples/go.mod similarity index 100% rename from tests/interop/go_pkgs/examples/go.mod rename to tests_interop/go_pkgs/examples/go.mod diff --git a/tests/interop/go_pkgs/examples/go.sum b/tests_interop/go_pkgs/examples/go.sum similarity index 100% rename from tests/interop/go_pkgs/examples/go.sum rename to tests_interop/go_pkgs/examples/go.sum diff --git a/tests/interop/go_pkgs/examples/utils/host.go b/tests_interop/go_pkgs/examples/utils/host.go similarity index 100% rename from tests/interop/go_pkgs/examples/utils/host.go rename to tests_interop/go_pkgs/examples/utils/host.go diff --git a/tests/interop/go_pkgs/install_interop_go_pkgs.sh b/tests_interop/go_pkgs/install_interop_go_pkgs.sh similarity index 100% rename from tests/interop/go_pkgs/install_interop_go_pkgs.sh rename to tests_interop/go_pkgs/install_interop_go_pkgs.sh diff --git a/tests/interop/test_bindings.py b/tests_interop/test_bindings.py similarity index 100% rename from tests/interop/test_bindings.py rename to tests_interop/test_bindings.py diff --git a/tests/interop/test_echo.py b/tests_interop/test_echo.py similarity index 100% rename from tests/interop/test_echo.py rename to tests_interop/test_echo.py diff --git a/tests/interop/test_net_stream.py b/tests_interop/test_net_stream.py similarity index 100% rename from tests/interop/test_net_stream.py rename to tests_interop/test_net_stream.py diff --git a/tests/interop/test_pubsub.py b/tests_interop/test_pubsub.py similarity index 100% rename from tests/interop/test_pubsub.py rename to tests_interop/test_pubsub.py diff --git a/tests/interop/utils.py b/tests_interop/utils.py similarity index 100% rename from tests/interop/utils.py rename to tests_interop/utils.py diff --git a/tox.ini b/tox.ini index 2d50b08..4d76675 100644 --- a/tox.ini +++ b/tox.ini @@ -27,11 +27,10 @@ skip_glob= [testenv] deps = -passenv = CI TRAVIS TRAVIS_* GOPATH +passenv = CI TRAVIS TRAVIS_* extras = test commands = - test: py.test --ignore=tests/interop - interop: py.test tests/interop + test: py.test tests/ -v basepython = py37: python3.7 @@ -40,6 +39,15 @@ basepython = python3 extras = dev commands = mypy -p libp2p -p examples --config-file {toxinidir}/mypy.ini - black --check libp2p tests examples setup.py - isort --recursive --check-only libp2p tests examples setup.py - flake8 libp2p tests examples setup.py + black --check libp2p tests tests_interop examples setup.py + isort --recursive --check-only libp2p tests tests_interop examples setup.py + flake8 libp2p tests tests_interop examples setup.py + +[testenv:py37-interop] +deps = p2pclient +passenv = CI TRAVIS TRAVIS_* GOPATH +extras = test +commands = + test: py.test tests_interop/ -v +basepython = + py37: python3.7 From 19c17dd5126c356dde15f3c4806356d50326174f Mon Sep 17 00:00:00 2001 From: mhchia Date: Mon, 23 Sep 2019 22:10:47 +0800 Subject: [PATCH 2/5] Remove the leftover `test` in testenv commands To make `tox -e py37-interop` run. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 4d76675..ef5c618 100644 --- a/tox.ini +++ b/tox.ini @@ -30,7 +30,7 @@ deps = passenv = CI TRAVIS TRAVIS_* extras = test commands = - test: py.test tests/ -v + py.test tests/ basepython = py37: python3.7 @@ -48,6 +48,6 @@ deps = p2pclient passenv = CI TRAVIS TRAVIS_* GOPATH extras = test commands = - test: py.test tests_interop/ -v + py.test tests_interop/ basepython = py37: python3.7 From d1c25b8b1e1f36cf1408c48a6ccd523829209305 Mon Sep 17 00:00:00 2001 From: mhchia Date: Tue, 24 Sep 2019 11:30:52 +0800 Subject: [PATCH 3/5] Fix interop pubsub tests and PR feedback - Use `from_id`, the changed field name in `PSMessage`. - PR feedbacks - Add label `test` in `testenv` in tox.ini, to avoid wrong dispatching an environment's command in the future. - Use `pytest` over `py.test`. --- libp2p/pubsub/pubsub.py | 6 +++++- setup.py | 1 - tests_interop/test_pubsub.py | 2 +- tox.ini | 8 +++++--- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index e413b28..ffc210b 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -195,7 +195,11 @@ class Pubsub: # Ref: https://developers.google.com/protocol-buffers/docs/reference/python-generated#singular-fields-proto2 # noqa: E501 if rpc_incoming.HasField("control"): # Pass rpc to router so router could perform custom logic - logger.debug("received `control` message %s from peer %s", peer_id) + logger.debug( + "received `control` message %s from peer %s", + rpc_incoming.control, + peer_id, + ) await self.router.handle_rpc(rpc_incoming, peer_id) # Force context switch diff --git a/setup.py b/setup.py index 1567c03..6e6ad67 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,6 @@ extras_require = { "factory-boy>=2.12.0,<3.0.0", "pytest>=4.6.3,<5.0.0", "pytest-asyncio>=0.10.0,<1.0.0", - "pexpect>=4.6,<5", ], "lint": [ "mypy>=0.701,<1.0", diff --git a/tests_interop/test_pubsub.py b/tests_interop/test_pubsub.py index bb37e35..87fbfa7 100644 --- a/tests_interop/test_pubsub.py +++ b/tests_interop/test_pubsub.py @@ -38,7 +38,7 @@ async def p2pd_subscribe(p2pd, topic) -> "asyncio.Queue[rpc_pb2.Message]": ps_msg.ParseFromString(msg_bytes) # Fill in the message used in py-libp2p msg = rpc_pb2.Message( - from_id=ps_msg.from_field, + from_id=ps_msg.from_id, data=ps_msg.data, seqno=ps_msg.seqno, topicIDs=ps_msg.topicIDs, diff --git a/tox.ini b/tox.ini index ef5c618..cbef62c 100644 --- a/tox.ini +++ b/tox.ini @@ -30,7 +30,7 @@ deps = passenv = CI TRAVIS TRAVIS_* extras = test commands = - py.test tests/ + test: pytest tests/ basepython = py37: python3.7 @@ -44,10 +44,12 @@ commands = flake8 libp2p tests tests_interop examples setup.py [testenv:py37-interop] -deps = p2pclient +deps = + p2pclient + pexpect passenv = CI TRAVIS TRAVIS_* GOPATH extras = test commands = - py.test tests_interop/ + pytest tests_interop/ basepython = py37: python3.7 From 1bfc6b41e42f2d5937a6362252a2095472b9a2da Mon Sep 17 00:00:00 2001 From: mhchia Date: Tue, 24 Sep 2019 12:18:31 +0800 Subject: [PATCH 4/5] Add pexpect in isort third party --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index cbef62c..7a6b652 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ select = B,C,E,F,W,T4,B9 [isort] force_sort_within_sections=True -known_third_party=pytest,p2pclient +known_third_party=pytest,p2pclient,pexpect multi_line_output=3 include_trailing_comma=True force_grid_wrap=0 From e217acf4acd4c5854ce699782977d98a3b5220ad Mon Sep 17 00:00:00 2001 From: mhchia Date: Tue, 24 Sep 2019 13:28:14 +0800 Subject: [PATCH 5/5] Add additional sleep To be more confident that the stream handler is registered in the daemon. --- tests_interop/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests_interop/conftest.py b/tests_interop/conftest.py index ff4266f..12d8737 100644 --- a/tests_interop/conftest.py +++ b/tests_interop/conftest.py @@ -165,6 +165,8 @@ async def py_to_daemon_stream_pair(hosts, p2pds, is_to_fail_daemon_stream): event_stream_handled.set() await p2pd.control.stream_handler(protocol_id, daemon_stream_handler) + # Sleep for a while to wait for the handler being registered. + await asyncio.sleep(0.01) if is_to_fail_daemon_stream: # FIXME: This is a workaround to make daemon reset the stream. @@ -180,5 +182,5 @@ async def py_to_daemon_stream_pair(hosts, p2pds, is_to_fail_daemon_stream): stream_py = await host.new_stream(p2pd.peer_id, [protocol_id]) if not is_to_fail_daemon_stream: await event_stream_handled.wait() - # NOTE: If `is_to_fail_daemon_stream == True`, `stream_daemon == None`. + # NOTE: If `is_to_fail_daemon_stream == True`, then `stream_daemon == None`. yield stream_py, stream_daemon