io: always make fds O_NONBLOCK.

Debugging an issue where pettycoin would become unresponsive, I discovered
this:

poll([{fd=5, events=POLLIN}, {fd=19, events=POLLIN}, {fd=6, events=POLLIN}, {fd=15, events=POLLIN}, {fd=8, events=POLLIN}, {fd=-11}, {fd=7, events=POLL
OUT}], 7, -1) = 1 ([{fd=7, revents=POLLOUT}]) <0.000014>
write(7, "\224]\4\0\4\0\0\0\200\203\16\234\v\262\276\321h\357\217Y\0\204\21\31\253\304#U\0206}\20"..., 286100) = 159280 <98.894019>

Despite poll saying the (TCP socket) fd was ready, the write took 98 seconds!
The results were far more reasonable with O_NONBLOCK:

write(9, "%\247l0\337^\216\24\323\2705\203Y\340h\2767/bM\373?dM\254\22g\310\v\0\0\0"..., 206460) = 40544 <0.000052>

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

diff --git a/ccan/io/io.c b/ccan/io/io.c
index 7dbb9c5a3962d3a802bf1aae502ceeda4bb1b685..0e716fced062c594d42955603e3fcfd2af67d3bb 100644 (file)
--- a/ccan/io/io.c
+++ b/ccan/io/io.c
@@ -67,6 +67,18 @@ static void next_plan(struct io_conn *conn, struct io_plan *plan)
        backend_new_plan(conn);
 }
 
+static void set_blocking(int fd, bool block)
+{
+       int flags = fcntl(fd, F_GETFL);
+
+       if (block)
+               flags &= ~O_NONBLOCK;
+       else
+               flags |= O_NONBLOCK;
+
+       fcntl(fd, F_SETFL, flags);
+}
+
 struct io_conn *io_new_conn_(const tal_t *ctx, int fd,
                             struct io_plan *(*init)(struct io_conn *, void *),
                             void *arg)
@@ -86,6 +98,9 @@ struct io_conn *io_new_conn_(const tal_t *ctx, int fd,
        if (!add_conn(conn))
                return tal_free(conn);
 
+       /* Keep our I/O async. */
+       set_blocking(fd, false);
+
        /* We start with out doing nothing, and in doing our init. */
        conn->plan[IO_OUT].status = IO_UNSET;
 
@@ -275,8 +290,6 @@ static int do_connect(int fd, struct io_plan_arg *arg)
                return -1;
 
        if (err == 0) {
-               /* Restore blocking if it was initially. */
-               fcntl(fd, F_SETFL, arg->u1.s);
                return 1;
        } else if (err == EINPROGRESS)
                return 0;
@@ -289,21 +302,21 @@ struct io_plan *io_connect_(struct io_conn *conn, const struct addrinfo *addr,
                            struct io_plan *(*next)(struct io_conn *, void *),
                            void *next_arg)
 {
-       struct io_plan_arg *arg = io_plan_arg(conn, IO_IN);
        int fd = io_conn_fd(conn);
 
-       /* Save old flags, set nonblock if not already. */
-       arg->u1.s = fcntl(fd, F_GETFL);
-       fcntl(fd, F_SETFL, arg->u1.s | O_NONBLOCK);
+       /* We don't actually need the arg, but we need it polling. */
+       io_plan_arg(conn, IO_OUT);
+
+       /* Note that io_new_conn() will make fd O_NONBLOCK */
 
        /* Immediate connect can happen. */
        if (connect(fd, addr->ai_addr, addr->ai_addrlen) == 0)
-               return set_always(conn, IO_IN, next, next_arg);
+               return set_always(conn, IO_OUT, next, next_arg);
 
        if (errno != EINPROGRESS)
                return io_close(conn);
 
-       return io_set_plan(conn, IO_IN, do_connect, next, next_arg);
+       return io_set_plan(conn, IO_OUT, do_connect, next, next_arg);
 }
 
 static struct io_plan *io_wait_dir(struct io_conn *conn,
@@ -437,7 +450,7 @@ void io_duplex_prepare(struct io_conn *conn)
        /* We can't sync debug until we've set both: io_wait() and io_always
         * can't handle it. */
        conn->debug_saved = conn->debug;
-       conn->debug = false;
+       io_set_debug(conn, false);
 }
 
 struct io_plan *io_duplex_(struct io_plan *in_plan, struct io_plan *out_plan)
@@ -449,7 +462,7 @@ struct io_plan *io_duplex_(struct io_plan *in_plan, struct io_plan *out_plan)
 
        /* Restore debug. */
        conn = container_of(in_plan, struct io_conn, plan[IO_IN]);
-       conn->debug = conn->debug_saved;
+       io_set_debug(conn, conn->debug_saved);
 
        /* Now set the plans again, to invoke sync debug. */
        io_set_plan(conn, IO_OUT,
@@ -522,6 +535,9 @@ struct io_plan *io_set_plan(struct io_conn *conn, enum io_direction dir,
 void io_set_debug(struct io_conn *conn, bool debug)
 {
        conn->debug = debug;
+
+       /* Debugging means fds must block. */
+       set_blocking(io_conn_fd(conn), debug);
 }
 
 void io_debug_complete(struct io_conn *conn)