rbuf: new module.
authorRusty Russell <rusty@rustcorp.com.au>
Mon, 3 Dec 2012 11:36:40 +0000 (22:06 +1030)
committerRusty Russell <rusty@rustcorp.com.au>
Mon, 3 Dec 2012 11:36:40 +0000 (22:06 +1030)
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Makefile-ccan
ccan/rbuf/LICENSE [new symlink]
ccan/rbuf/_info [new file with mode: 0644]
ccan/rbuf/rbuf.c [new file with mode: 0644]
ccan/rbuf/rbuf.h [new file with mode: 0644]
ccan/rbuf/test/run-all.c [new file with mode: 0644]
ccan/rbuf/test/run-open.c [new file with mode: 0644]
ccan/rbuf/test/run-partial-read.c [new file with mode: 0644]
ccan/rbuf/test/run-term-eof.c [new file with mode: 0644]
ccan/rbuf/test/run.c [new file with mode: 0644]

index ef5f82897d1c1a9d1985bcc86b121deab0244b32..249d0bb0e752815292536e11c2796f9be8afe29c 100644 (file)
@@ -62,6 +62,7 @@ MODS_NORMAL_WITH_SRC := antithread \
        opt \
        ptr_valid \
        rbtree \
+       rbuf \
        read_write_all \
        rfc822 \
        sparse_bsearch \
diff --git a/ccan/rbuf/LICENSE b/ccan/rbuf/LICENSE
new file mode 120000 (symlink)
index 0000000..2354d12
--- /dev/null
@@ -0,0 +1 @@
+../../licenses/BSD-MIT
\ No newline at end of file
diff --git a/ccan/rbuf/_info b/ccan/rbuf/_info
new file mode 100644 (file)
index 0000000..30232db
--- /dev/null
@@ -0,0 +1,52 @@
+#include <string.h>
+#include "config.h"
+
+/**
+ * rbuf - buffered I/O input primitive.
+ *
+ * This code is like stdio, only simpler and more transparent to the user.
+ *
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ * License: BSD-MIT
+ *
+ * Example:
+ *     #include <ccan/rbuf/rbuf.h>
+ *     #include <ccan/err/err.h>
+ *     #include <stdlib.h>
+ *     #include <unistd.h>
+ *
+ *     // Dumb demo program to replace ' ' with '*'.
+ *     int main(int argc, char *argv[])
+ *     {
+ *             struct rbuf in;
+ *             char *word;
+ *
+ *             if (argv[1]) {
+ *                     if (!rbuf_open(&in, argv[1], NULL, 0))
+ *                             err(1, "Failed opening %s", argv[1]);
+ *             } else
+ *                     rbuf_init(&in, STDIN_FILENO, NULL, 0);
+ *
+ *             while ((word = rbuf_read_str(&in, ' ', realloc)) != NULL)
+ *                     printf("%s*", word);
+ *
+ *             if (errno)
+ *                     err(1, "Reading %s", argv[1] ? argv[1] : "<stdin>");
+ *
+ *             // Free the buffer, just because we can.
+ *             free(in.buf);
+ *             return 0;
+ *     }
+ */
+int main(int argc, char *argv[])
+{
+       /* Expect exactly one argument */
+       if (argc != 2)
+               return 1;
+
+       if (strcmp(argv[1], "depends") == 0) {
+               return 0;
+       }
+
+       return 1;
+}
diff --git a/ccan/rbuf/rbuf.c b/ccan/rbuf/rbuf.c
new file mode 100644 (file)
index 0000000..0e210a9
--- /dev/null
@@ -0,0 +1,130 @@
+/* Licensed under BSD-MIT - see LICENSE file for details */
+#include <ccan/rbuf/rbuf.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <fcntl.h>
+
+bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max)
+{
+       int fd = open(name, O_RDONLY);
+       if (fd >= 0) {
+               rbuf_init(rbuf, fd, buf, buf_max);
+               return true;
+       }
+       return false;
+}
+
+static size_t rem(const struct rbuf *buf)
+{
+       return buf->buf_end - (buf->start + buf->len);
+}
+
+size_t rbuf_good_size(int fd)
+{
+       struct stat st;
+
+       if (fstat(fd, &st) == 0 && st.st_blksize >= 4096)
+               return st.st_blksize;
+       return 4096;
+}
+
+static bool enlarge_buf(struct rbuf *buf, size_t len,
+                       void *(*resize)(void *buf, size_t len))
+{
+       char *new;
+       if (!resize) {
+               errno = ENOMEM;
+               return false;
+       }
+       if (!len)
+               len = rbuf_good_size(buf->fd);
+       new = resize(buf->buf, len);
+       if (!new)
+               return false;
+       buf->start += (new - buf->buf);
+       buf->buf = new;
+       buf->buf_end = new + len;
+       return true;
+}
+
+static ssize_t get_more(struct rbuf *rbuf,
+                       void *(*resize)(void *buf, size_t len))
+{
+       size_t r;
+
+       if (rbuf->start + rbuf->len == rbuf->buf_end) {
+               if (!enlarge_buf(rbuf, (rbuf->buf_end - rbuf->buf) * 2, resize))
+                       return -1;
+       }
+
+       r = read(rbuf->fd, rbuf->start + rbuf->len, rem(rbuf));
+       if (r <= 0)
+               return r;
+
+       rbuf->len += r;
+       return r;
+}
+
+void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len))
+{
+       ssize_t r;
+
+       /* Move back to start of buffer if we're empty. */
+       if (!rbuf->len)
+               rbuf->start = rbuf->buf;
+
+       while ((r = get_more(rbuf, resize)) != 0)
+               if (r < 0)
+                       return NULL;
+       return rbuf->start;
+}
+
+void *rbuf_fill(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len))
+{
+       if (!rbuf->len) {
+               rbuf->start = rbuf->buf;
+               if (get_more(rbuf, resize) < 0)
+                       return NULL;
+       }
+       return rbuf->start;
+}
+
+char *rbuf_read_str(struct rbuf *rbuf, char term,
+                   void *(*resize)(void *buf, size_t len))
+{
+       char *p, *ret;
+       ssize_t r = 0;
+       size_t prev = 0;
+
+       /* Move back to start of buffer if we're empty. */
+       if (!rbuf->len)
+               rbuf->start = rbuf->buf;
+
+       while (!(p = memchr(rbuf->start + prev, term, rbuf->len - prev))) {
+               prev += r;
+               r = get_more(rbuf, resize);
+               if (r < 0)
+                       return NULL;
+               /* EOF with no term. */
+               if (r == 0) {
+                       /* Nothing read at all? */
+                       if (!rbuf->len && term) {
+                               errno = 0;
+                               return NULL;
+                       }
+                       /* Put term after input (get_more made room). */
+                       assert(rbuf->start + rbuf->len < rbuf->buf_end);
+                       rbuf->start[rbuf->len] = '\0';
+                       ret = rbuf->start;
+                       rbuf_consume(rbuf, rbuf->len);
+                       return ret;
+               }
+       }
+       *p = '\0';
+       ret = rbuf->start;
+       rbuf_consume(rbuf, p + 1 - ret);
+       return ret;
+}
diff --git a/ccan/rbuf/rbuf.h b/ccan/rbuf/rbuf.h
new file mode 100644 (file)
index 0000000..ab1504a
--- /dev/null
@@ -0,0 +1,156 @@
+/* Licensed under BSD-MIT - see LICENSE file for details */
+#ifndef CCAN_RBUF_H
+#define CCAN_RBUF_H
+#include <stdio.h> // For size_t
+#include <limits.h> // For UCHAR_MAX
+#include <assert.h>
+#include <stdbool.h>
+
+struct rbuf {
+       int fd;
+
+       /* Where to read next. */
+       char *start;
+       /* How much of what is there is valid. */
+       size_t len;
+
+       /* The entire buffer memory we have to work with. */
+       char *buf, *buf_end;
+};
+
+/**
+ * rbuf_init - set up a buffer.
+ * @buf: the struct rbuf.
+ * @fd: the file descriptor.
+ * @buf: the buffer to use.
+ * @buf_max: the size of the buffer.
+ */
+static inline void rbuf_init(struct rbuf *buf,
+                            int fd, char *buffer, size_t buf_max)
+{
+       buf->fd = fd;
+       buf->start = buf->buf = buffer;
+       buf->len = 0;
+       buf->buf_end = buffer + buf_max;
+}
+
+/**
+ * rbuf_open - set up a buffer by opening a file.
+ * @buf: the struct rbuf.
+ * @filename: the filename
+ * @buf: the buffer to use.
+ * @buf_max: the size of the buffer.
+ *
+ * Returns false if the open fails.  If @buf_max is 0, then the buffer
+ * will be resized to rbuf_good_size() on first rbuf_fill.
+ *
+ * Example:
+ *     struct rbuf in;
+ *
+ *     if (!rbuf_open(&in, "foo", NULL, 0))
+ *             err(1, "Could not open foo");
+ */
+bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max);
+
+/**
+ * rbuf_good_size - get a good buffer size for this fd.
+ * @fd: the file descriptor.
+ *
+ * If you don't know what size you want, try this.
+ */
+size_t rbuf_good_size(int fd);
+
+/**
+ * rbuf_fill - read into a buffer if it's empty.
+ * @buf: the struct rbuf
+ * @resize: the call to resize the buffer.
+ *
+ * If @resize is needed and is NULL, or returns false, rbuf_read will
+ * return NULL (with errno set to ENOMEM).  If a read fails, then NULL
+ * is also returned.  If there is nothing more to read, it will return
+ * NULL with errno set to 0.  Otherwise, returns @buf->start; @buf->len
+ * is the valid length of the buffer.
+ *
+ * You need to call rbuf_consume() to mark data in the buffer as
+ * consumed.
+ *
+ * Example:
+ *     while (rbuf_fill(&in, realloc)) {
+ *             printf("%.*s\n", (int)in.len, in.start);
+ *             rbuf_consume(&in, in.len);
+ *     }
+ *     if (errno)
+ *             err(1, "reading foo");
+ */
+void *rbuf_fill(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len));
+
+/**
+ * rbuf_consume - helper to use up data in a buffer.
+ * @buf: the struct rbuf
+ * @len: the length (from @buf->start) you used.
+ *
+ * After rbuf_fill() you should indicate the data you've used with
+ * rbuf_consume().  That way rbuf_fill() will know if it has anything
+ * to do.
+ */
+static inline void rbuf_consume(struct rbuf *buf, size_t len)
+{
+       buf->len -= len;
+       buf->start += len;
+}
+
+/**
+ * rbuf_fill_all - read rest of file into a buffer.
+ * @buf: the struct rbuf
+ * @resize: the call to resize the buffer.
+ *
+ * If @resize is needed and is NULL, or returns false, rbuf_read_all
+ * will return NULL (with errno set to ENOMEM).  If a read fails,
+ * then NULL is also returned, otherwise returns @buf->start.
+ *
+ * Example:
+ *     if (!rbuf_fill_all(&in, realloc)) {
+ *             if (errno)
+ *                     err(1, "reading foo");
+ *     }
+ */
+void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len));
+
+/**
+ * rbuf_read_str - fill into a buffer up to a terminator, and consume string.
+ * @buf: the struct rbuf
+ * @term: the character to terminate the read.
+ * @resize: the call to resize the buffer.
+ *
+ * If @resize is needed and is NULL, or returns false, rbuf_read_str
+ * will return NULL (with errno set to ENOMEM).  If a read fails,
+ * then NULL is also returned, otherwise the next string.  It
+ * replaces the terminator @term (if any) with NUL, otherwise NUL
+ * is placed after EOF.  If you need to, you can tell this has happened
+ * because the nul terminator will be at @buf->start (normally it will
+ * be at @buf->start - 1).
+ *
+ * If there is nothing remaining to be read, NULL is returned with
+ * errno set to 0, unless @term is NUL, in which case it returns the
+ * empty string.
+ *
+ * Note: using @term set to NUL is a cheap way of getting an entire
+ * file into a C string, as long as the file doesn't contain NUL.
+ *
+ * Example:
+ *     char *line;
+ *
+ *     line = rbuf_read_str(&in, '\n', realloc);
+ *     if (!line) {
+ *             if (errno)
+ *                     err(1, "reading foo");
+ *             else
+ *                     printf("Empty file\n");
+ *     } else
+ *             printf("First line is %s\n", line);
+ *
+ */
+char *rbuf_read_str(struct rbuf *rbuf, char term,
+                   void *(*resize)(void *buf, size_t len));
+
+#endif /* CCAN_RBUF_H */
diff --git a/ccan/rbuf/test/run-all.c b/ccan/rbuf/test/run-all.c
new file mode 100644 (file)
index 0000000..2cd0b42
--- /dev/null
@@ -0,0 +1,49 @@
+#include <ccan/rbuf/rbuf.h>
+/* Include the C files directly. */
+#include <ccan/rbuf/rbuf.c>
+#include <ccan/tap/tap.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+
+int main(void)
+{
+       struct rbuf in;
+       char buf[4096];
+       int i, size, fd = open("run-all-file", O_WRONLY|O_CREAT, 0600);
+
+       /* This is how many tests you plan to run */
+       plan_tests(8);
+
+       /* Make sure we're bigger than a single buffer! */
+       size = rbuf_good_size(fd)*2;
+       for (i = 0; i * sizeof(buf) < size; i++) {
+               memset(buf, 0x42 + i, sizeof(buf));
+               write(fd, buf, sizeof(buf));
+       }
+       close(fd);
+
+       ok1(rbuf_open(&in, "run-all-file", NULL, 0));
+       /* Can't fill without realloc. */
+       ok1(!rbuf_fill(&in, NULL));
+       ok1(errno == ENOMEM);
+       ok1(rbuf_fill(&in, realloc));
+       /* But can't load in whole file. */
+       ok1(!rbuf_fill_all(&in, NULL));
+       ok1(errno == ENOMEM);
+       ok1(rbuf_fill_all(&in, realloc));
+       ok1(in.len == size);
+       for (i = 0; i * sizeof(buf) < size; i++) {
+               memset(buf, 0x42 + i, sizeof(buf));
+               if (memcmp(buf, in.start, sizeof(buf)) != 0) {
+                       fail("Bad buffer contents");
+                       break;
+               }
+               rbuf_consume(&in, sizeof(buf));
+       }
+       free(in.buf);
+
+       /* This exits depending on whether all tests passed */
+       return exit_status();
+}
diff --git a/ccan/rbuf/test/run-open.c b/ccan/rbuf/test/run-open.c
new file mode 100644 (file)
index 0000000..5678e42
--- /dev/null
@@ -0,0 +1,25 @@
+#include <ccan/rbuf/rbuf.h>
+/* Include the C files directly. */
+#include <ccan/rbuf/rbuf.c>
+#include <ccan/tap/tap.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+
+int main(void)
+{
+       struct rbuf in;
+
+       /* This is how many tests you plan to run */
+       plan_tests(5);
+
+       ok1(!rbuf_open(&in, "nonexistent-file", NULL, 0));
+       ok1(errno == ENOENT);
+       ok1(rbuf_open(&in, "test/run-open.c", NULL, 0));
+       ok1(close(in.fd) == 0);
+       /* If this fails to stat, it should fall back */
+       ok1(rbuf_good_size(in.fd) == 4096);
+
+       return exit_status();
+}
diff --git a/ccan/rbuf/test/run-partial-read.c b/ccan/rbuf/test/run-partial-read.c
new file mode 100644 (file)
index 0000000..362e6b6
--- /dev/null
@@ -0,0 +1,67 @@
+#include <unistd.h>
+
+static ssize_t partial_read(int fd, void *buf, size_t count)
+{
+       return read(fd, buf, 1);
+}
+static ssize_t full_read(int fd, void *buf, size_t count)
+{
+       return read(fd, buf, count);
+}
+#define read partial_read
+
+#include <ccan/rbuf/rbuf.h>
+/* Include the C files directly. */
+#include <ccan/rbuf/rbuf.c>
+#include <ccan/tap/tap.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+
+int main(void)
+{
+       struct rbuf in;
+       char buf[4096];
+       char *lines[100], *p;
+       int i, fd = open("test/run.c", O_RDONLY);
+
+       /* This is how many tests you plan to run */
+       plan_tests(140);
+
+       /* Grab ourselves for comparison. */
+       buf[full_read(fd, buf, sizeof(buf))] = '\0';
+       lseek(fd, SEEK_SET, 0);
+
+       for (i = 0, p = buf; *p; i++) {
+               lines[i] = p;
+               p = strchr(p, '\n');
+               *p = '\0';
+               p++;
+       }
+       lines[i] = NULL;
+
+       rbuf_init(&in, fd, malloc(31), 31);
+       ok1(in.fd == fd);
+       ok1(in.buf_end - in.buf == 31);
+       p = rbuf_read_str(&in, '\n', NULL);
+       ok1(p);
+       ok1(strcmp(p, lines[0]) == 0);
+
+       p = rbuf_read_str(&in, '\n', realloc);
+       ok1(p);
+       ok1(strcmp(p, lines[1]) == 0);
+
+       for (i = 2; lines[i]; i++) {
+               ok1(p = rbuf_read_str(&in, '\n', realloc));
+               ok1(strcmp(p, lines[i]) == 0);
+       }
+
+       p = rbuf_read_str(&in, '\n', realloc);
+       ok1(errno == 0);
+       ok1(p == NULL);
+       free(in.buf);
+
+       /* This exits depending on whether all tests passed */
+       return exit_status();
+}
diff --git a/ccan/rbuf/test/run-term-eof.c b/ccan/rbuf/test/run-term-eof.c
new file mode 100644 (file)
index 0000000..097dcbb
--- /dev/null
@@ -0,0 +1,44 @@
+#include <ccan/rbuf/rbuf.h>
+/* Include the C files directly. */
+#include <ccan/rbuf/rbuf.c>
+#include <ccan/tap/tap.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+
+int main(void)
+{
+       struct rbuf in;
+       char buf[4096], *p;
+       int fd = open("test/run-term-eof.c", O_RDONLY), len;
+
+       /* This is how many tests you plan to run */
+       plan_tests(6);
+
+       /* Grab ourselves for comparison. */
+       len = read(fd, buf, sizeof(buf));
+       buf[len] = '\0';
+       lseek(fd, SEEK_SET, 0);
+
+       /* We have exact-size buffer, which causes problems adding term. */
+       rbuf_init(&in, fd, malloc(len), len);
+       p = rbuf_read_str(&in, 64, NULL); /* At symbol does not appear. */
+       ok1(errno == ENOMEM);
+       ok1(!p);
+       /* This should succeed... */
+       p = rbuf_read_str(&in, 64, realloc);
+       ok1(p);
+       ok1(strcmp(p, buf) == 0);
+       free(in.buf);
+
+       /* Try again. */
+       lseek(fd, SEEK_SET, 0);
+       rbuf_init(&in, fd, malloc(len), len);
+       p = rbuf_read_str(&in, 64, realloc);
+       ok1(p);
+       ok1(strcmp(p, buf) == 0);
+       free(in.buf);
+
+       return exit_status();
+}
diff --git a/ccan/rbuf/test/run.c b/ccan/rbuf/test/run.c
new file mode 100644 (file)
index 0000000..e576188
--- /dev/null
@@ -0,0 +1,68 @@
+#include <ccan/rbuf/rbuf.h>
+/* Include the C files directly. */
+#include <ccan/rbuf/rbuf.c>
+#include <ccan/tap/tap.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+
+int main(void)
+{
+       struct rbuf in;
+       char buf[4096];
+       char *lines[100], *p;
+       int i, fd = open("test/run.c", O_RDONLY), len;
+
+       /* This is how many tests you plan to run */
+       plan_tests(144);
+
+       /* Grab ourselves for comparison. */
+       len = read(fd, buf, sizeof(buf));
+       buf[len] = '\0';
+       lseek(fd, SEEK_SET, 0);
+
+       for (i = 0, p = buf; *p; i++) {
+               lines[i] = p;
+               p = strchr(p, '\n');
+               *p = '\0';
+               p++;
+       }
+       lines[i] = NULL;
+
+       rbuf_init(&in, fd, malloc(31), 31);
+       ok1(in.fd == fd);
+       ok1(in.buf_end - in.buf == 31);
+       p = rbuf_read_str(&in, '\n', NULL);
+       ok1(p);
+       ok1(strcmp(p, lines[0]) == 0);
+
+       p = rbuf_read_str(&in, '\n', realloc);
+       ok1(p);
+       ok1(strcmp(p, lines[1]) == 0);
+
+       for (i = 2; lines[i]; i++) {
+               ok1(p = rbuf_read_str(&in, '\n', realloc));
+               ok1(strcmp(p, lines[i]) == 0);
+       }
+
+       p = rbuf_read_str(&in, '\n', realloc);
+       ok1(errno == 0);
+       ok1(p == NULL);
+       free(in.buf);
+
+       /* Another way of reading the entire (text) file. */
+       lseek(fd, SEEK_SET, 0);
+       rbuf_init(&in, fd, NULL, 0);
+       p = rbuf_read_str(&in, 0, realloc);
+       ok1(p);
+       ok1(strlen(p) == len);
+
+       close(fd);
+       p = rbuf_read_str(&in, 0, realloc);
+       ok1(errno == EBADF);
+       ok1(!p);
+       free(in.buf);
+
+       return exit_status();
+}