diff --git a/toxcore/net_crypto.c b/toxcore/net_crypto.c
index b045ff8c..7fdf8dcc 100644
--- a/toxcore/net_crypto.c
+++ b/toxcore/net_crypto.c
@@ -384,6 +384,78 @@ static Crypto_Connection *get_crypto_connection(const Net_Crypto *c, int crypt_c
 }
 
 
+/* Associate an ip_port to a connection.
+ *
+ * return -1 on failure.
+ * return 0 on success.
+ */
+static int add_ip_port_connection(Net_Crypto *c, int crypt_connection_id, IP_Port ip_port)
+{
+    Crypto_Connection *conn = get_crypto_connection(c, crypt_connection_id);
+
+    if (conn == 0)
+        return -1;
+
+    if (ip_port.ip.family == AF_INET) {
+        if (!ipport_equal(&ip_port, &conn->ip_portv4) && LAN_ip(conn->ip_portv4.ip) != 0) {
+            if (!bs_list_add(&c->ip_port_list, (uint8_t *)&ip_port, crypt_connection_id))
+                return -1;
+
+            bs_list_remove(&c->ip_port_list, (uint8_t *)&conn->ip_portv4, crypt_connection_id);
+            conn->ip_portv4 = ip_port;
+            return 0;
+        }
+    } else if (ip_port.ip.family == AF_INET6) {
+        if (!ipport_equal(&ip_port, &conn->ip_portv6)) {
+            if (!bs_list_add(&c->ip_port_list, (uint8_t *)&ip_port, crypt_connection_id))
+                return -1;
+
+            bs_list_remove(&c->ip_port_list, (uint8_t *)&conn->ip_portv6, crypt_connection_id);
+            conn->ip_portv6 = ip_port;
+            return 0;
+        }
+    }
+
+    return -1;
+}
+
+/* Return the IP_Port that should be used to send packets to the other peer.
+ *
+ * return IP_Port with family 0 on failure.
+ * return IP_Port on success.
+ */
+IP_Port return_ip_port_connection(Net_Crypto *c, int crypt_connection_id)
+{
+    IP_Port empty;
+    empty.ip.family = 0;
+
+    Crypto_Connection *conn = get_crypto_connection(c, crypt_connection_id);
+
+    if (conn == 0)
+        return empty;
+
+    uint64_t current_time = unix_time();
+    _Bool v6 = 0, v4 = 0;
+
+    if ((UDP_DIRECT_TIMEOUT + conn->direct_lastrecv_timev4) > current_time) {
+        v4 = 1;
+    }
+
+    if ((UDP_DIRECT_TIMEOUT + conn->direct_lastrecv_timev6) > current_time) {
+        v6 = 1;
+    }
+
+    if (v4 && LAN_ip(conn->ip_portv4.ip) == 0) {
+        return conn->ip_portv4;
+    } else if (v6 && conn->ip_portv6.ip.family == AF_INET6) {
+        return conn->ip_portv6;
+    } else if (conn->ip_portv4.ip.family == AF_INET) {
+        return conn->ip_portv4;
+    } else {
+        return empty;
+    }
+}
+
 /* Sends a packet to the peer using the fastest route.
  *
  * return -1 on failure.
@@ -400,14 +472,15 @@ static int send_packet_to(Net_Crypto *c, int crypt_connection_id, const uint8_t
     int direct_send_attempt = 0;
 
     pthread_mutex_lock(&conn->mutex);
+    IP_Port ip_port = return_ip_port_connection(c, crypt_connection_id);
 
     //TODO: on bad networks, direct connections might not last indefinitely.
-    if (conn->ip_port.ip.family != 0) {
+    if (ip_port.ip.family != 0) {
         _Bool direct_connected = 0;
         crypto_connection_status(c, crypt_connection_id, &direct_connected, NULL);
 
         if (direct_connected) {
-            if ((uint32_t)sendpacket(c->dht->net, conn->ip_port, data, length) == length) {
+            if ((uint32_t)sendpacket(c->dht->net, ip_port, data, length) == length) {
                 pthread_mutex_unlock(&conn->mutex);
                 return 0;
             } else {
@@ -418,7 +491,7 @@ static int send_packet_to(Net_Crypto *c, int crypt_connection_id, const uint8_t
 
         //TODO: a better way of sending packets directly to confirm the others ip.
         if (length < 96 || data[0] == NET_PACKET_COOKIE_REQUEST || data[0] == NET_PACKET_CRYPTO_HS) {
-            if ((uint32_t)sendpacket(c->dht->net, conn->ip_port, data, length) == length)
+            if ((uint32_t)sendpacket(c->dht->net, ip_port, data, length) == length)
                 direct_send_attempt = 1;
         }
     }
@@ -1474,15 +1547,15 @@ static int crypto_connection_add_source(Net_Crypto *c, int crypt_connection_id,
         return -1;
 
     if (source.ip.family == AF_INET || source.ip.family == AF_INET6) {
-        if (!ipport_equal(&source, &conn->ip_port)) {
-            if (!bs_list_add(&c->ip_port_list, (uint8_t *)&source, crypt_connection_id))
-                return -1;
+        if (add_ip_port_connection(c, crypt_connection_id, source) != 0)
+            return -1;
 
-            bs_list_remove(&c->ip_port_list, (uint8_t *)&conn->ip_port, crypt_connection_id);
-            conn->ip_port = source;
+        if (source.ip.family == AF_INET) {
+            conn->direct_lastrecv_timev4 = unix_time();
+        } else {
+            conn->direct_lastrecv_timev6 = unix_time();
         }
 
-        conn->direct_lastrecv_time = unix_time();
         return 0;
     } else if (source.ip.family == TCP_FAMILY) {
         if (add_tcp_number_relay_connection(c->tcp_c, conn->connection_number_tcp, source.ip.ip6.uint32[0]) == 0)
@@ -1684,32 +1757,22 @@ int set_direct_ip_port(Net_Crypto *c, int crypt_connection_id, IP_Port ip_port,
     if (conn == 0)
         return -1;
 
-    if (ip_port.ip.family != AF_INET && ip_port.ip.family != AF_INET6)
-        return -1;
-
-    if (!ipport_equal(&ip_port, &conn->ip_port)) {
-        if ((UDP_DIRECT_TIMEOUT + conn->direct_lastrecv_time) > unix_time()) {
-            /* We already know a LAN ip, no need to switch. */
-            if (LAN_ip(conn->ip_port.ip) == 0)
-                return -1;
-
-            /* Prefer ipv6. */
-            if (conn->ip_port.ip.family == AF_INET6 && ip_port.ip.family == AF_INET)
-                return -1;
-        }
-
-        if (bs_list_add(&c->ip_port_list, (uint8_t *)&ip_port, crypt_connection_id)) {
-            bs_list_remove(&c->ip_port_list, (uint8_t *)&conn->ip_port, crypt_connection_id);
-            conn->ip_port = ip_port;
-
-            if (connected) {
-                conn->direct_lastrecv_time = unix_time();
+    if (add_ip_port_connection(c, crypt_connection_id, ip_port) == 0) {
+        if (connected) {
+            if (ip_port.ip.family == AF_INET) {
+                conn->direct_lastrecv_timev4 = unix_time();
             } else {
-                conn->direct_lastrecv_time = 0;
+                conn->direct_lastrecv_timev6 = unix_time();
+            }
+        } else {
+            if (ip_port.ip.family == AF_INET) {
+                conn->direct_lastrecv_timev4 = 0;
+            } else {
+                conn->direct_lastrecv_timev6 = 0;
             }
-
-            return 0;
         }
+
+        return 0;
     }
 
     return -1;
@@ -2019,7 +2082,13 @@ static int udp_handle_packet(void *object, IP_Port source, const uint8_t *packet
         return -1;
 
     pthread_mutex_lock(&conn->mutex);
-    conn->direct_lastrecv_time = unix_time();
+
+    if (source.ip.family == AF_INET) {
+        conn->direct_lastrecv_timev4 = unix_time();
+    } else {
+        conn->direct_lastrecv_timev6 = unix_time();
+    }
+
     pthread_mutex_unlock(&conn->mutex);
     return 0;
 }
@@ -2364,7 +2433,8 @@ int crypto_kill(Net_Crypto *c, int crypt_connection_id)
         kill_tcp_connection_to(c->tcp_c, conn->connection_number_tcp);
         pthread_mutex_unlock(&c->tcp_mutex);
 
-        bs_list_remove(&c->ip_port_list, (uint8_t *)&conn->ip_port, crypt_connection_id);
+        bs_list_remove(&c->ip_port_list, (uint8_t *)&conn->ip_portv4, crypt_connection_id);
+        bs_list_remove(&c->ip_port_list, (uint8_t *)&conn->ip_portv6, crypt_connection_id);
         clear_temp_packet(c, crypt_connection_id);
         clear_buffer(&conn->send_array);
         clear_buffer(&conn->recv_array);
@@ -2392,7 +2462,12 @@ unsigned int crypto_connection_status(const Net_Crypto *c, int crypt_connection_
     if (direct_connected) {
         *direct_connected = 0;
 
-        if ((UDP_DIRECT_TIMEOUT + conn->direct_lastrecv_time) > unix_time())
+        uint64_t current_time = unix_time();
+
+        if ((UDP_DIRECT_TIMEOUT + conn->direct_lastrecv_timev4) > current_time)
+            *direct_connected = 1;
+
+        if ((UDP_DIRECT_TIMEOUT + conn->direct_lastrecv_timev6) > current_time)
             *direct_connected = 1;
     }
 
diff --git a/toxcore/net_crypto.h b/toxcore/net_crypto.h
index 9e6ff4b5..cfeeaca5 100644
--- a/toxcore/net_crypto.h
+++ b/toxcore/net_crypto.h
@@ -119,8 +119,10 @@ typedef struct {
     uint64_t temp_packet_sent_time; /* The time at which the last temp_packet was sent in ms. */
     uint32_t temp_packet_num_sent;
 
-    IP_Port ip_port; /* The ip and port to contact this guy directly.*/
-    uint64_t direct_lastrecv_time; /* The Time at which we last received a direct packet in ms. */
+    IP_Port ip_portv4; /* The ip and port to contact this guy directly.*/
+    IP_Port ip_portv6;
+    uint64_t direct_lastrecv_timev4; /* The Time at which we last received a direct packet in ms. */
+    uint64_t direct_lastrecv_timev6;
 
     Packets_Array send_array;
     Packets_Array recv_array;