saminiir · saminiir · Mar 8, 2019 · Mar 8, 2019 · Mar 8, 2019 · Mar 9, 2019
diff --git a/.travis.yml b/.travis.yml
@@ -12,5 +12,7 @@ before_install:
 script:
   - sudo mknod /dev/net/tap c 10 200
   - sudo chmod 0666 /dev/net/tap
-  - sudo setcap cap_net_admin=ep /bin/ip
+  - sudo ip tuntap add mode tap tap0
+  - sudo ip link set dev tap0 up
+  - sudo ip addr add 10.0.0.5/24 dev tap0
   - make test
diff --git a/Documentation/getting-started.md b/Documentation/getting-started.md
@@ -10,34 +10,29 @@ DISCLAIMER: Level-IP is not a production-ready networking stack, and does not in
 
 Standard `make` stuff.
 
-    $ make all
+    make all
 
-This builds `lvl-ip` itself, but also the libc wrapper and provided example applications.
-
-When building, `sudo setcap ...` probably asks super user permissions from you. This is because `lvl-ip` needs the `CAP_NET_ADMIN` capability to setup itself. After the setup, it drops that capability.
-
-Currently, `lvl-ip` also configures the tap interface through the `ip` tool. Hence, give it permissions too:
-
-    $ which ip
-    /usr/bin/ip
-    $ sudo setcap cap_net_admin=ep /usr/bin/ip
+This builds `lvl-ip` and the libc wrapper (under `tools/`) and provided example applications (under `apps/`).
 
 # Setup
 
 Level-IP uses a Linux TAP device to communicate to the outside world. In short, the tap device is initialized in the host Linux' networking stack, and `lvl-ip` can then read the L2 frames:
 
-    $ sudo mknod /dev/net/tap c 10 200
-    $ sudo chmod 0666 /dev/net/tap
+    sudo mknod /dev/net/tap c 10 200
+    sudo chmod 0666 /dev/net/tap
+    sudo ip tuntap add mode tap tap0
+    sudo ip link set dev tap0 up
+    sudo ip addr add 10.0.0.5/24 dev tap0
 
 In essence, `lvl-ip` operates as a host inside the tap device's subnet. Therefore, in order to communicate with other hosts, the tap device needs to be set in a forwarding mode:
 
 An example from my (Arch) Linux machine, where `wlp2s0` is my outgoing interface, and `tap0` is the tap device for `lvl-ip`:
 
-    $ sysctl -w net.ipv4.ip_forward=1
-    $ iptables -I INPUT --source 10.0.0.0/24 -j ACCEPT
-    $ iptables -t nat -I POSTROUTING --out-interface wlp2s0 -j MASQUERADE
-    $ iptables -I FORWARD --in-interface wlp2s0 --out-interface tap0 -j ACCEPT
-    $ iptables -I FORWARD --in-interface tap0 --out-interface wlp2s0 -j ACCEPT
+    sysctl -w net.ipv4.ip_forward=1
+    iptables -I INPUT --source 10.0.0.0/24 -j ACCEPT
+    iptables -t nat -I POSTROUTING --out-interface wlp2s0 -j MASQUERADE
+    iptables -I FORWARD --in-interface wlp2s0 --out-interface tap0 -j ACCEPT
+    iptables -I FORWARD --in-interface tap0 --out-interface wlp2s0 -j ACCEPT
 
 Now, packets coming from `lvl-ip` (10.0.0.4/24 in this case) should be NATed by the host Linux interfaces and traverse the FORWARD chain correctly to the host's outgoing gateway.
 
@@ -47,21 +42,19 @@ See http://www.netfilter.org/documentation/HOWTO/packet-filtering-HOWTO-9.html f
 
 When you've built lvl-ip and setup your host stack to forward packets, you can try communicating to the Internet:
 
-    $ ./lvl-ip
+    ./lvl-ip
 
-The userspace TCP/IP stack should start. Now, first test communications with the provided applications:
+The userspace TCP/IP stack should start. 
 
-    $ cd tools
-    $ ./level-ip ../apps/curl/curl google.com 80
+Now, first test communications with the provided applications:
+
+    cd tools
+    ./level-ip ../apps/curl/curl google.com 80
 
 `./level-ip` is just a bash-script that allows `liblevelip.so` to take precedence over the libc socket API calls. 
 
 The important point is that `./level-ip` aims to be usable against any existing dynamically-linked application. Let's try the _real_ `curl`:
 
-    [saminiir@localhost tools]$ curl --version
-    curl 7.50.0 (x86_64-pc-linux-gnu) libcurl/7.50.0 OpenSSL/1.0.2h zlib/1.2.8 libidn/1.33 libssh2/1.7.0
-    Protocols: dict file ftp ftps gopher http https imap imaps pop3 pop3s rtsp scp sftp smb smbs smtp smtps telnet tftp
-    Features: AsynchDNS IDN IPv6 Largefile GSS-API Kerberos SPNEGO NTLM NTLM_WB SSL libz TLS-SRP UnixSockets
     [saminiir@localhost tools]$ curl google.com
     <HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">
     <TITLE>302 Moved</TITLE></HEAD><BODY>
@@ -89,3 +82,23 @@ Try browsing the Web, with Level-IP doing the packet transfer:
     [saminiir@localhost tools]$ ./level-ip firefox google.com
 
 That's it!
+
+# Troubleshooting
+
+Common errors:
+
+## Tun module not loaded/available
+
+`tun` is required to be loaded:
+
+```
+$ lsmod | grep tun
+tun                    57344  2
+```
+
+Try loading it
+```
+$ modprobe tun
+```
+
+Otherwise, consult your distro's documentation on setting it up.
diff --git a/Makefile b/Makefile
@@ -7,9 +7,6 @@ apps = apps/curl/curl
 
 lvl-ip: $(obj)
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(obj) -o lvl-ip
-	@echo
-	@echo "lvl-ip needs CAP_NET_ADMIN:"
-	sudo setcap cap_setpcap,cap_net_admin=ep lvl-ip
 
 build/%.o: src/%.c ${headers}
 	$(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@

diff --git a/include/ipc.h b/include/ipc.h
@@ -7,7 +7,7 @@
 #define ipc_dbg(msg, th)                                                \
     do {                                                                \
         print_debug("IPC sockets count %d, current sock %d, tid %lu: %s", \
-                    socket_count, th->sock, th->id, msg);             \
+                socket_count, th->sock, th->id, msg);             \
     } while (0)
 #else
 #define ipc_dbg(msg, th)
@@ -26,6 +26,8 @@ void *start_ipc_listener();
 #define IPC_SETSOCKOPT  0x0009
 #define IPC_GETPEERNAME 0x000A
 #define IPC_GETSOCKNAME 0x000B
+#define IPC_SENDMSG     0x000C
+#define IPC_RECVMSG     0x000D
 
 struct ipc_thread {
     struct list_head list;
@@ -105,4 +107,18 @@ struct ipc_sockname {
     uint8_t sa_data[128];
 };
 
+struct ipc_msghdr {
+    int sockfd;
+    socklen_t msg_namelen;
+    int msg_iovlen;
+    socklen_t msg_controllen;
+    int flags;
+    uint8_t data[];
+} __attribute__((packed));
+
+struct ipc_iovec {
+    size_t iov_len;
+    uint8_t iov_base[];
+} __attribute__((packed));
+
 #endif
diff --git a/include/netlink.h b/include/netlink.h
@@ -0,0 +1,40 @@
+#ifndef _NETLINK_H
+#define _NETLINK_H
+
+#include "syshead.h"
+#include "socket.h"
+#include "skbuff.h"
+
+#ifdef DEBUG_SOCKET
+#define netlink_dbg(sock, msg, ...)                                            \
+    do {                                                                \
+        socket_dbg(sock, "NETLINK "msg, ##__VA_ARGS__);                    \
+    } while (0)
+#else
+#define netlink_dbg(msg, th, ...)
+#endif
+
+struct nl_message {
+    struct list_head list;
+    struct socket *sock;
+    struct nlmsghdr nl;
+    uint8_t data[];
+};
+
+int netlink_create(struct socket *sock, int protocol);
+int netlink_socket(struct socket *sock, int protocol);
+int netlink_connect(struct socket *sock, struct sockaddr *addr, int addr_len, int flags);
+int netlink_write(struct socket *sock, const void *buf, int len);
+int netlink_read(struct socket *sock, void *buf, int len);
+int netlink_close(struct socket *sock);
+int netlink_free(struct socket *sock);
+int netlink_abort(struct socket *sock);
+int netlink_getpeername(struct socket *sock, struct sockaddr *restrict address,
+                     socklen_t *restrict address_len);
+int netlink_getsockname(struct socket *sock, struct sockaddr *restrict address,
+                     socklen_t *restrict address_len);
+int netlink_sendmsg(struct socket *sock, const struct msghdr *message, int flags);
+int netlink_recvmsg(struct socket *sock, struct msghdr *message, int flags);
+
+struct sock *netlink_lookup(struct sk_buff *skb, uint16_t sport, uint16_t dport);
+#endif
diff --git a/include/socket.h b/include/socket.h
@@ -50,6 +50,8 @@ struct sock_ops {
                         socklen_t *restrict address_len);
     int (*getsockname) (struct socket *sock, struct sockaddr *restrict addr,
                         socklen_t *restrict address_len);
+    int (*sendmsg) (struct socket *sock, const struct msghdr *message, int flags);
+    int (*recvmsg) (struct socket *sock, struct msghdr *message, int flags);
 };
 
 struct net_family {
@@ -62,6 +64,7 @@ struct socket {
     pid_t pid;
     int refcnt;
     enum socket_state state;
+    int family;
     short type;
     int flags;
     struct sock *sk;
@@ -83,9 +86,13 @@ int _getpeername(pid_t pid, int socket, struct sockaddr *restrict address,
                  socklen_t *restrict address_len);
 int _getsockname(pid_t pid, int socket, struct sockaddr *restrict address,
                  socklen_t *restrict address_len);
+int _sendmsg (pid_t pid, int socket, const struct msghdr *message, int flags);
+int _recvmsg (pid_t pid, int socket, struct msghdr *message, int flags);
 
 struct socket *socket_lookup(uint16_t sport, uint16_t dport);
 struct socket *socket_find(struct socket *sock);
+int filter_sockets(int family, int proto, uint8_t **store,
+                   int (*f)(struct socket *s, uint8_t *ptr), int size);
 int socket_rd_acquire(struct socket *sock);
 int socket_wr_acquire(struct socket *sock);
 int socket_release(struct socket *sock);

diff --git a/include/syshead.h b/include/syshead.h
@@ -1,6 +1,7 @@
 #ifndef SYSHEAD_H
 #define SYSHEAD_H
 
+#include <assert.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdint.h>
@@ -13,6 +14,9 @@
 #include <arpa/inet.h>
 #include <linux/if.h>
 #include <linux/if_tun.h>
+#include <linux/netlink.h>
+#include <linux/sock_diag.h>
+#include <linux/inet_diag.h>
 #include <sys/ioctl.h>
 #include <fcntl.h>
 #include <signal.h>

diff --git a/include/tcp.h b/include/tcp.h
@@ -142,33 +142,33 @@ struct tcpiphdr {
 } __attribute__((packed));
 
 enum tcp_states {
-    TCP_LISTEN, /* represents waiting for a connection request from any remote
-                   TCP and port. */
+    TCP_ESTABLISHED = 1, /* represents an open connection, data received can be
+                        delivered to the user.  The normal state for the data transfer phase
+                        of the connection. */
     TCP_SYN_SENT, /* represents waiting for a matching connection request
                      after having sent a connection request. */
     TCP_SYN_RECEIVED, /* represents waiting for a confirming connection
                          request acknowledgment after having both received and sent a
                          connection request. */
-    TCP_ESTABLISHED, /* represents an open connection, data received can be
-                        delivered to the user.  The normal state for the data transfer phase
-                        of the connection. */
     TCP_FIN_WAIT_1, /* represents waiting for a connection termination request
                        from the remote TCP, or an acknowledgment of the connection
                        termination request previously sent. */
     TCP_FIN_WAIT_2, /* represents waiting for a connection termination request
                        from the remote TCP. */
+    TCP_TIME_WAIT, /* represents waiting for enough time to pass to be sure
+                      the remote TCP received the acknowledgment of its connection
+                      termination request. */
     TCP_CLOSE, /* represents no connection state at all. */
     TCP_CLOSE_WAIT, /* represents waiting for a connection termination request
                        from the local user. */
-    TCP_CLOSING, /* represents waiting for a connection termination request
-                    acknowledgment from the remote TCP. */
     TCP_LAST_ACK, /* represents waiting for an acknowledgment of the
                      connection termination request previously sent to the remote TCP
                      (which includes an acknowledgment of its connection termination
                      request). */
-    TCP_TIME_WAIT, /* represents waiting for enough time to pass to be sure
-                      the remote TCP received the acknowledgment of its connection
-                      termination request. */
+    TCP_LISTEN, /* represents waiting for a connection request from any remote
+                   TCP and port. */
+    TCP_CLOSING, /* represents waiting for a connection termination request
+                    acknowledgment from the remote TCP. */
 };
 
 struct tcb {

diff --git a/include/utils.h b/include/utils.h
@@ -9,6 +9,9 @@
 #define print_err(str, ...)                     \
     fprintf(stderr, str, ##__VA_ARGS__);
 
+#define exit_with_error(en, msg)                                        \
+    do { errno = en; perror(msg); exit(EXIT_FAILURE); } while (0)
+
 int run_cmd(char *cmd, ...);
 uint32_t sum_every_16bits(void *addr, int count);
 uint16_t checksum(void *addr, int count, int start_sum);

diff --git a/src/cli.c b/src/cli.c
@@ -9,7 +9,6 @@ static void usage(char *app)
     print_err("Usage: %s\n", app);
     print_err("\n");
     print_err("Linux TCP/IP stack implemented with TUN/TAP devices.\n");
-    print_err("Requires the CAP_NET_ADMIN capability. See capabilities(7).\n");
     print_err("See https://www.kernel.org/doc/Documentation/networking/tuntap.txt\n");
     print_err("\n");
     print_err("Options:\n");

diff --git a/src/inet.c b/src/inet.c
@@ -113,14 +113,15 @@ static int inet_stream_connect(struct socket *sock, const struct sockaddr *addr,
             goto out;
         }
 
+        // False thread deadlock detected here by TSan?
+        // See commit c60d98322964976439ad4f09344de2989131d091
         pthread_mutex_lock(&sock->sleep.lock);
         while (sock->state == SS_CONNECTING && sk->err == -EINPROGRESS) {
             socket_release(sock);
             wait_sleep(&sock->sleep);
             socket_wr_acquire(sock);
         }
         pthread_mutex_unlock(&sock->sleep.lock);
-        socket_wr_acquire(sock);
 
         switch (sk->err) {
         case -ETIMEDOUT: