Add utility functions for libp2p bindings

To prepare for pubsub interop test

libp2p/pubsub/floodsub.py

@@ -7,6 +7,8 @@ from .pb import rpc_pb2
 from .pubsub import Pubsub
 from .pubsub_router_interface import IPubsubRouter
 
+PROTOCOL_ID = TProtocol("/floodsub/1.0.0")
+
 
 class FloodSub(IPubsubRouter):
 

libp2p/pubsub/gossipsub.py

@@ -11,6 +11,8 @@ from .pb import rpc_pb2
 from .pubsub import Pubsub
 from .pubsub_router_interface import IPubsubRouter
 
+PROTOCOL_ID = TProtocol("/meshsub/1.0.0")
+
 
 class GossipSub(IPubsubRouter):
 

setup.py

@@ -9,6 +9,8 @@ extras_require = {
         "pytest>=4.6.3,<5.0.0",
         "pytest-asyncio>=0.10.0,<1.0.0",
         "pexpect>=4.6,<5",
+        # FIXME: Master branch. Use PyPI instead after it is released.
+        "p2pclient @ git+https://git@github.com/mhchia/py-libp2p-daemon-bindings@4777c62",
     ],
     "lint": [
         "mypy>=0.701,<1.0",

tests/interop/constants.py

@@ -1 +1,2 @@
+LOCALHOST_IP = "127.0.0.1"
 PEXPECT_NEW_LINE = "\r\n"

tests/interop/daemon.py

@@ -0,0 +1,81 @@
+from multiaddr import Multiaddr
+from pexpect.spawnbase import SpawnBase
+
+from p2pclient import Client
+
+from libp2p.peer.peerinfo import info_from_p2p_addr, PeerInfo
+from libp2p.peer.id import ID
+
+from .constants import PEXPECT_NEW_LINE, LOCALHOST_IP
+from .envs import GO_BIN_PATH
+
+P2PD_PATH = GO_BIN_PATH / "p2pd"
+
+
+class Daemon:
+    proc: SpawnBase
+    control: Client
+    peer_info: PeerInfo
+
+    def __init__(self, proc: SpawnBase, control: Client, peer_info: PeerInfo) -> None:
+        self.proc = proc
+        self.control = control
+        self.peer_info = peer_info
+
+    def __repr__(self) -> str:
+        return f"<Daemon {self.peer_id.to_string()[2:8]}>"
+
+    @property
+    def peer_id(self) -> ID:
+        return self.peer_info.peer_id
+
+    @property
+    def listen_maddr(self) -> Multiaddr:
+        return self.peer_info.addrs[0]
+
+
+async def make_p2pd(
+    proc_factory,
+    unused_tcp_port_factory,
+    is_secure: bool,
+    is_pubsub_enabled=True,
+    is_gossipsub=True,
+    is_pubsub_signing=False,
+    is_pubsub_signing_strict=False,
+) -> Daemon:
+    control_maddr = Multiaddr(f"/ip4/{LOCALHOST_IP}/tcp/{unused_tcp_port_factory()}")
+    args = [f"-listen={str(control_maddr)}"]
+    # NOTE: To support `-insecure`, we need to hack `go-libp2p-daemon`.
+    if not is_secure:
+        args.append("-insecure=true")
+    if is_pubsub_enabled:
+        args.append("-pubsub")
+        if is_gossipsub:
+            args.append("-pubsubRouter=gossipsub")
+        else:
+            args.append("-pubsubRouter=floodsub")
+        if not is_pubsub_signing:
+            args.append("-pubsubSign=false")
+        if not is_pubsub_signing_strict:
+            args.append("-pubsubSignStrict=false")
+        # NOTE:
+        #   Two other params are possibly what we want to configure:
+        #   - gossipsubHeartbeatInterval: GossipSubHeartbeatInitialDelay = 100 * time.Millisecond
+        #   - gossipsubHeartbeatInitialDelay: GossipSubHeartbeatInterval = 1 * time.Second
+        #   Referece: https://github.com/libp2p/go-libp2p-daemon/blob/b95e77dbfcd186ccf817f51e95f73f9fd5982600/p2pd/main.go#L348-L353  # noqa: E501
+    proc = proc_factory(str(P2PD_PATH), args)
+    await proc.expect(r"Peer ID:\s+(\w+)" + PEXPECT_NEW_LINE, async_=True)
+    peer_id_base58 = proc.match.group(1)
+    await proc.expect(r"Peer Addrs:", async_=True)
+    await proc.expect(
+        rf"(/ip4/{LOCALHOST_IP}/tcp/[\w]+)" + PEXPECT_NEW_LINE, async_=True
+    )
+    daemon_listener_maddr = Multiaddr(proc.match.group(1))
+    daemon_pinfo = info_from_p2p_addr(
+        daemon_listener_maddr.encapsulate(f"/p2p/{peer_id_base58}")
+    )
+    client_callback_maddr = Multiaddr(
+        f"/ip4/{LOCALHOST_IP}/tcp/{unused_tcp_port_factory()}"
+    )
+    p2pc = Client(control_maddr, client_callback_maddr)
+    return Daemon(proc, p2pc, daemon_pinfo)

tests/interop/envs.py

@@ -0,0 +1,4 @@
+import os
+import pathlib
+
+GO_BIN_PATH = pathlib.Path(os.environ["GOPATH"]) / "bin"

tests/interop/go_pkgs/echo/main.go

@@ -3,17 +3,13 @@ package main
 import (
 	"bufio"
 	"context"
-	"crypto/rand"
 	"flag"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"log"
-	mrand "math/rand"
 
-	"github.com/libp2p/go-libp2p"
+	utils "interop/utils"
-	"github.com/libp2p/go-libp2p-core/crypto"
+
-	"github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/libp2p/go-libp2p-core/peerstore"
@@ -23,59 +19,6 @@ import (
 	gologging "github.com/whyrusleeping/go-logging"
 )
 
-// makeBasicHost creates a LibP2P host with a random peer ID listening on the
-// given multiaddress. It won't encrypt the connection if insecure is true.
-func makeBasicHost(listenPort int, insecure bool, randseed int64) (host.Host, error) {
-
-	// If the seed is zero, use real cryptographic randomness. Otherwise, use a
-	// deterministic randomness source to make generated keys stay the same
-	// across multiple runs
-	var r io.Reader
-	if randseed == 0 {
-		r = rand.Reader
-	} else {
-		r = mrand.New(mrand.NewSource(randseed))
-	}
-
-	// Generate a key pair for this host. We will use it at least
-	// to obtain a valid host ID.
-	priv, _, err := crypto.GenerateKeyPairWithReader(crypto.RSA, 2048, r)
-	if err != nil {
-		return nil, err
-	}
-
-	opts := []libp2p.Option{
-		libp2p.ListenAddrStrings(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", listenPort)),
-		libp2p.Identity(priv),
-		libp2p.DisableRelay(),
-	}
-
-	if insecure {
-		opts = append(opts, libp2p.NoSecurity)
-	}
-
-	basicHost, err := libp2p.New(context.Background(), opts...)
-	if err != nil {
-		return nil, err
-	}
-
-	// Build host multiaddress
-	hostAddr, _ := ma.NewMultiaddr(fmt.Sprintf("/ipfs/%s", basicHost.ID().Pretty()))
-
-	// Now we can build a full multiaddress to reach this host
-	// by encapsulating both addresses:
-	addr := basicHost.Addrs()[0]
-	fullAddr := addr.Encapsulate(hostAddr)
-	log.Printf("I am %s\n", fullAddr)
-	if insecure {
-		log.Printf("Now run \"./echo -l %d -d %s -insecure\" on a different terminal\n", listenPort+1, fullAddr)
-	} else {
-		log.Printf("Now run \"./echo -l %d -d %s\" on a different terminal\n", listenPort+1, fullAddr)
-	}
-
-	return basicHost, nil
-}
-
 func main() {
 	// LibP2P code uses golog to log messages. They log with different
 	// string IDs (i.e. "swarm"). We can control the verbosity level for
@@ -94,7 +37,7 @@ func main() {
 	}
 
 	// Make a host that listens on the given multiaddress
-	ha, err := makeBasicHost(*listenF, *insecure, *seed)
+	ha, err := utils.MakeBasicHost(*listenF, *insecure, *seed)
 	if err != nil {
 		log.Fatal(err)
 	}

tests/interop/go_pkgs/utils/host.go

@@ -0,0 +1,69 @@
+package utils
+
+import (
+	"context"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"log"
+	mrand "math/rand"
+
+	"github.com/libp2p/go-libp2p"
+	"github.com/libp2p/go-libp2p-core/crypto"
+	"github.com/libp2p/go-libp2p-core/host"
+
+	ma "github.com/multiformats/go-multiaddr"
+)
+
+// MakeBasicHost creates a LibP2P host with a random peer ID listening on the
+// given multiaddress. It won't encrypt the connection if insecure is true.
+func MakeBasicHost(listenPort int, insecure bool, randseed int64) (host.Host, error) {
+
+	// If the seed is zero, use real cryptographic randomness. Otherwise, use a
+	// deterministic randomness source to make generated keys stay the same
+	// across multiple runs
+	var r io.Reader
+	if randseed == 0 {
+		r = rand.Reader
+	} else {
+		r = mrand.New(mrand.NewSource(randseed))
+	}
+
+	// Generate a key pair for this host. We will use it at least
+	// to obtain a valid host ID.
+	priv, _, err := crypto.GenerateKeyPairWithReader(crypto.RSA, 2048, r)
+	if err != nil {
+		return nil, err
+	}
+
+	opts := []libp2p.Option{
+		libp2p.ListenAddrStrings(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", listenPort)),
+		libp2p.Identity(priv),
+		libp2p.DisableRelay(),
+	}
+
+	if insecure {
+		opts = append(opts, libp2p.NoSecurity)
+	}
+
+	basicHost, err := libp2p.New(context.Background(), opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	// Build host multiaddress
+	hostAddr, _ := ma.NewMultiaddr(fmt.Sprintf("/ipfs/%s", basicHost.ID().Pretty()))
+
+	// Now we can build a full multiaddress to reach this host
+	// by encapsulating both addresses:
+	addr := basicHost.Addrs()[0]
+	fullAddr := addr.Encapsulate(hostAddr)
+	log.Printf("I am %s\n", fullAddr)
+	if insecure {
+		log.Printf("Now run \"./echo -l %d -d %s -insecure\" on a different terminal\n", listenPort+1, fullAddr)
+	} else {
+		log.Printf("Now run \"./echo -l %d -d %s\" on a different terminal\n", listenPort+1, fullAddr)
+	}
+
+	return basicHost, nil
+}

tests/interop/test_echo.py

@@ -1,6 +1,4 @@
 import asyncio
-import os
-import pathlib
 
 from multiaddr import Multiaddr
 import pytest
@@ -9,9 +7,9 @@ from libp2p.peer.peerinfo import info_from_p2p_addr
 from libp2p.typing import TProtocol
 
 from .constants import PEXPECT_NEW_LINE
+from .envs import GO_BIN_PATH
 
-GOPATH = pathlib.Path(os.environ["GOPATH"])
+ECHO_PATH = GO_BIN_PATH / "echo"
-ECHO_PATH = GOPATH / "bin" / "echo"
 ECHO_PROTOCOL_ID = TProtocol("/echo/1.0.0")
 
 

tests/interop/test_pubsub_bind.py

@@ -0,0 +1,11 @@
+import pytest
+
+from .daemon import make_p2pd
+
+
+@pytest.mark.parametrize("num_hosts", (1,))
+@pytest.mark.asyncio
+async def test_pubsub_init(
+    hosts, proc_factory, is_host_secure, unused_tcp_port_factory
+):
+    p2pd = await make_p2pd(proc_factory, unused_tcp_port_factory, is_host_secure)