From 44b37ee8aeb185f6837378fef17d887bda6ff89b Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 3 Dec 2012 22:06:40 +1030 Subject: [PATCH] rbuf: new module. Signed-off-by: Rusty Russell --- Makefile-ccan | 1 + ccan/rbuf/LICENSE | 1 + ccan/rbuf/_info | 52 ++++++++++ ccan/rbuf/rbuf.c | 130 +++++++++++++++++++++++++ ccan/rbuf/rbuf.h | 156 ++++++++++++++++++++++++++++++ ccan/rbuf/test/run-all.c | 49 ++++++++++ ccan/rbuf/test/run-open.c | 25 +++++ ccan/rbuf/test/run-partial-read.c | 67 +++++++++++++ ccan/rbuf/test/run-term-eof.c | 44 +++++++++ ccan/rbuf/test/run.c | 68 +++++++++++++ 10 files changed, 593 insertions(+) create mode 120000 ccan/rbuf/LICENSE create mode 100644 ccan/rbuf/_info create mode 100644 ccan/rbuf/rbuf.c create mode 100644 ccan/rbuf/rbuf.h create mode 100644 ccan/rbuf/test/run-all.c create mode 100644 ccan/rbuf/test/run-open.c create mode 100644 ccan/rbuf/test/run-partial-read.c create mode 100644 ccan/rbuf/test/run-term-eof.c create mode 100644 ccan/rbuf/test/run.c diff --git a/Makefile-ccan b/Makefile-ccan index ef5f8289..249d0bb0 100644 --- a/Makefile-ccan +++ b/Makefile-ccan @@ -62,6 +62,7 @@ MODS_NORMAL_WITH_SRC := antithread \ opt \ ptr_valid \ rbtree \ + rbuf \ read_write_all \ rfc822 \ sparse_bsearch \ diff --git a/ccan/rbuf/LICENSE b/ccan/rbuf/LICENSE new file mode 120000 index 00000000..2354d129 --- /dev/null +++ b/ccan/rbuf/LICENSE @@ -0,0 +1 @@ +../../licenses/BSD-MIT \ No newline at end of file diff --git a/ccan/rbuf/_info b/ccan/rbuf/_info new file mode 100644 index 00000000..30232db2 --- /dev/null +++ b/ccan/rbuf/_info @@ -0,0 +1,52 @@ +#include +#include "config.h" + +/** + * rbuf - buffered I/O input primitive. + * + * This code is like stdio, only simpler and more transparent to the user. + * + * Author: Rusty Russell + * License: BSD-MIT + * + * Example: + * #include + * #include + * #include + * #include + * + * // Dumb demo program to replace ' ' with '*'. + * int main(int argc, char *argv[]) + * { + * struct rbuf in; + * char *word; + * + * if (argv[1]) { + * if (!rbuf_open(&in, argv[1], NULL, 0)) + * err(1, "Failed opening %s", argv[1]); + * } else + * rbuf_init(&in, STDIN_FILENO, NULL, 0); + * + * while ((word = rbuf_read_str(&in, ' ', realloc)) != NULL) + * printf("%s*", word); + * + * if (errno) + * err(1, "Reading %s", argv[1] ? argv[1] : ""); + * + * // Free the buffer, just because we can. + * free(in.buf); + * return 0; + * } + */ +int main(int argc, char *argv[]) +{ + /* Expect exactly one argument */ + if (argc != 2) + return 1; + + if (strcmp(argv[1], "depends") == 0) { + return 0; + } + + return 1; +} diff --git a/ccan/rbuf/rbuf.c b/ccan/rbuf/rbuf.c new file mode 100644 index 00000000..0e210a9e --- /dev/null +++ b/ccan/rbuf/rbuf.c @@ -0,0 +1,130 @@ +/* Licensed under BSD-MIT - see LICENSE file for details */ +#include +#include +#include +#include +#include +#include +#include + +bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max) +{ + int fd = open(name, O_RDONLY); + if (fd >= 0) { + rbuf_init(rbuf, fd, buf, buf_max); + return true; + } + return false; +} + +static size_t rem(const struct rbuf *buf) +{ + return buf->buf_end - (buf->start + buf->len); +} + +size_t rbuf_good_size(int fd) +{ + struct stat st; + + if (fstat(fd, &st) == 0 && st.st_blksize >= 4096) + return st.st_blksize; + return 4096; +} + +static bool enlarge_buf(struct rbuf *buf, size_t len, + void *(*resize)(void *buf, size_t len)) +{ + char *new; + if (!resize) { + errno = ENOMEM; + return false; + } + if (!len) + len = rbuf_good_size(buf->fd); + new = resize(buf->buf, len); + if (!new) + return false; + buf->start += (new - buf->buf); + buf->buf = new; + buf->buf_end = new + len; + return true; +} + +static ssize_t get_more(struct rbuf *rbuf, + void *(*resize)(void *buf, size_t len)) +{ + size_t r; + + if (rbuf->start + rbuf->len == rbuf->buf_end) { + if (!enlarge_buf(rbuf, (rbuf->buf_end - rbuf->buf) * 2, resize)) + return -1; + } + + r = read(rbuf->fd, rbuf->start + rbuf->len, rem(rbuf)); + if (r <= 0) + return r; + + rbuf->len += r; + return r; +} + +void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)) +{ + ssize_t r; + + /* Move back to start of buffer if we're empty. */ + if (!rbuf->len) + rbuf->start = rbuf->buf; + + while ((r = get_more(rbuf, resize)) != 0) + if (r < 0) + return NULL; + return rbuf->start; +} + +void *rbuf_fill(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)) +{ + if (!rbuf->len) { + rbuf->start = rbuf->buf; + if (get_more(rbuf, resize) < 0) + return NULL; + } + return rbuf->start; +} + +char *rbuf_read_str(struct rbuf *rbuf, char term, + void *(*resize)(void *buf, size_t len)) +{ + char *p, *ret; + ssize_t r = 0; + size_t prev = 0; + + /* Move back to start of buffer if we're empty. */ + if (!rbuf->len) + rbuf->start = rbuf->buf; + + while (!(p = memchr(rbuf->start + prev, term, rbuf->len - prev))) { + prev += r; + r = get_more(rbuf, resize); + if (r < 0) + return NULL; + /* EOF with no term. */ + if (r == 0) { + /* Nothing read at all? */ + if (!rbuf->len && term) { + errno = 0; + return NULL; + } + /* Put term after input (get_more made room). */ + assert(rbuf->start + rbuf->len < rbuf->buf_end); + rbuf->start[rbuf->len] = '\0'; + ret = rbuf->start; + rbuf_consume(rbuf, rbuf->len); + return ret; + } + } + *p = '\0'; + ret = rbuf->start; + rbuf_consume(rbuf, p + 1 - ret); + return ret; +} diff --git a/ccan/rbuf/rbuf.h b/ccan/rbuf/rbuf.h new file mode 100644 index 00000000..ab1504a7 --- /dev/null +++ b/ccan/rbuf/rbuf.h @@ -0,0 +1,156 @@ +/* Licensed under BSD-MIT - see LICENSE file for details */ +#ifndef CCAN_RBUF_H +#define CCAN_RBUF_H +#include // For size_t +#include // For UCHAR_MAX +#include +#include + +struct rbuf { + int fd; + + /* Where to read next. */ + char *start; + /* How much of what is there is valid. */ + size_t len; + + /* The entire buffer memory we have to work with. */ + char *buf, *buf_end; +}; + +/** + * rbuf_init - set up a buffer. + * @buf: the struct rbuf. + * @fd: the file descriptor. + * @buf: the buffer to use. + * @buf_max: the size of the buffer. + */ +static inline void rbuf_init(struct rbuf *buf, + int fd, char *buffer, size_t buf_max) +{ + buf->fd = fd; + buf->start = buf->buf = buffer; + buf->len = 0; + buf->buf_end = buffer + buf_max; +} + +/** + * rbuf_open - set up a buffer by opening a file. + * @buf: the struct rbuf. + * @filename: the filename + * @buf: the buffer to use. + * @buf_max: the size of the buffer. + * + * Returns false if the open fails. If @buf_max is 0, then the buffer + * will be resized to rbuf_good_size() on first rbuf_fill. + * + * Example: + * struct rbuf in; + * + * if (!rbuf_open(&in, "foo", NULL, 0)) + * err(1, "Could not open foo"); + */ +bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max); + +/** + * rbuf_good_size - get a good buffer size for this fd. + * @fd: the file descriptor. + * + * If you don't know what size you want, try this. + */ +size_t rbuf_good_size(int fd); + +/** + * rbuf_fill - read into a buffer if it's empty. + * @buf: the struct rbuf + * @resize: the call to resize the buffer. + * + * If @resize is needed and is NULL, or returns false, rbuf_read will + * return NULL (with errno set to ENOMEM). If a read fails, then NULL + * is also returned. If there is nothing more to read, it will return + * NULL with errno set to 0. Otherwise, returns @buf->start; @buf->len + * is the valid length of the buffer. + * + * You need to call rbuf_consume() to mark data in the buffer as + * consumed. + * + * Example: + * while (rbuf_fill(&in, realloc)) { + * printf("%.*s\n", (int)in.len, in.start); + * rbuf_consume(&in, in.len); + * } + * if (errno) + * err(1, "reading foo"); + */ +void *rbuf_fill(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)); + +/** + * rbuf_consume - helper to use up data in a buffer. + * @buf: the struct rbuf + * @len: the length (from @buf->start) you used. + * + * After rbuf_fill() you should indicate the data you've used with + * rbuf_consume(). That way rbuf_fill() will know if it has anything + * to do. + */ +static inline void rbuf_consume(struct rbuf *buf, size_t len) +{ + buf->len -= len; + buf->start += len; +} + +/** + * rbuf_fill_all - read rest of file into a buffer. + * @buf: the struct rbuf + * @resize: the call to resize the buffer. + * + * If @resize is needed and is NULL, or returns false, rbuf_read_all + * will return NULL (with errno set to ENOMEM). If a read fails, + * then NULL is also returned, otherwise returns @buf->start. + * + * Example: + * if (!rbuf_fill_all(&in, realloc)) { + * if (errno) + * err(1, "reading foo"); + * } + */ +void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)); + +/** + * rbuf_read_str - fill into a buffer up to a terminator, and consume string. + * @buf: the struct rbuf + * @term: the character to terminate the read. + * @resize: the call to resize the buffer. + * + * If @resize is needed and is NULL, or returns false, rbuf_read_str + * will return NULL (with errno set to ENOMEM). If a read fails, + * then NULL is also returned, otherwise the next string. It + * replaces the terminator @term (if any) with NUL, otherwise NUL + * is placed after EOF. If you need to, you can tell this has happened + * because the nul terminator will be at @buf->start (normally it will + * be at @buf->start - 1). + * + * If there is nothing remaining to be read, NULL is returned with + * errno set to 0, unless @term is NUL, in which case it returns the + * empty string. + * + * Note: using @term set to NUL is a cheap way of getting an entire + * file into a C string, as long as the file doesn't contain NUL. + * + * Example: + * char *line; + * + * line = rbuf_read_str(&in, '\n', realloc); + * if (!line) { + * if (errno) + * err(1, "reading foo"); + * else + * printf("Empty file\n"); + * } else + * printf("First line is %s\n", line); + * + */ +char *rbuf_read_str(struct rbuf *rbuf, char term, + void *(*resize)(void *buf, size_t len)); + +#endif /* CCAN_RBUF_H */ diff --git a/ccan/rbuf/test/run-all.c b/ccan/rbuf/test/run-all.c new file mode 100644 index 00000000..2cd0b424 --- /dev/null +++ b/ccan/rbuf/test/run-all.c @@ -0,0 +1,49 @@ +#include +/* Include the C files directly. */ +#include +#include +#include +#include +#include +#include + +int main(void) +{ + struct rbuf in; + char buf[4096]; + int i, size, fd = open("run-all-file", O_WRONLY|O_CREAT, 0600); + + /* This is how many tests you plan to run */ + plan_tests(8); + + /* Make sure we're bigger than a single buffer! */ + size = rbuf_good_size(fd)*2; + for (i = 0; i * sizeof(buf) < size; i++) { + memset(buf, 0x42 + i, sizeof(buf)); + write(fd, buf, sizeof(buf)); + } + close(fd); + + ok1(rbuf_open(&in, "run-all-file", NULL, 0)); + /* Can't fill without realloc. */ + ok1(!rbuf_fill(&in, NULL)); + ok1(errno == ENOMEM); + ok1(rbuf_fill(&in, realloc)); + /* But can't load in whole file. */ + ok1(!rbuf_fill_all(&in, NULL)); + ok1(errno == ENOMEM); + ok1(rbuf_fill_all(&in, realloc)); + ok1(in.len == size); + for (i = 0; i * sizeof(buf) < size; i++) { + memset(buf, 0x42 + i, sizeof(buf)); + if (memcmp(buf, in.start, sizeof(buf)) != 0) { + fail("Bad buffer contents"); + break; + } + rbuf_consume(&in, sizeof(buf)); + } + free(in.buf); + + /* This exits depending on whether all tests passed */ + return exit_status(); +} diff --git a/ccan/rbuf/test/run-open.c b/ccan/rbuf/test/run-open.c new file mode 100644 index 00000000..5678e42e --- /dev/null +++ b/ccan/rbuf/test/run-open.c @@ -0,0 +1,25 @@ +#include +/* Include the C files directly. */ +#include +#include +#include +#include +#include +#include + +int main(void) +{ + struct rbuf in; + + /* This is how many tests you plan to run */ + plan_tests(5); + + ok1(!rbuf_open(&in, "nonexistent-file", NULL, 0)); + ok1(errno == ENOENT); + ok1(rbuf_open(&in, "test/run-open.c", NULL, 0)); + ok1(close(in.fd) == 0); + /* If this fails to stat, it should fall back */ + ok1(rbuf_good_size(in.fd) == 4096); + + return exit_status(); +} diff --git a/ccan/rbuf/test/run-partial-read.c b/ccan/rbuf/test/run-partial-read.c new file mode 100644 index 00000000..362e6b60 --- /dev/null +++ b/ccan/rbuf/test/run-partial-read.c @@ -0,0 +1,67 @@ +#include + +static ssize_t partial_read(int fd, void *buf, size_t count) +{ + return read(fd, buf, 1); +} +static ssize_t full_read(int fd, void *buf, size_t count) +{ + return read(fd, buf, count); +} +#define read partial_read + +#include +/* Include the C files directly. */ +#include +#include +#include +#include +#include +#include + +int main(void) +{ + struct rbuf in; + char buf[4096]; + char *lines[100], *p; + int i, fd = open("test/run.c", O_RDONLY); + + /* This is how many tests you plan to run */ + plan_tests(140); + + /* Grab ourselves for comparison. */ + buf[full_read(fd, buf, sizeof(buf))] = '\0'; + lseek(fd, SEEK_SET, 0); + + for (i = 0, p = buf; *p; i++) { + lines[i] = p; + p = strchr(p, '\n'); + *p = '\0'; + p++; + } + lines[i] = NULL; + + rbuf_init(&in, fd, malloc(31), 31); + ok1(in.fd == fd); + ok1(in.buf_end - in.buf == 31); + p = rbuf_read_str(&in, '\n', NULL); + ok1(p); + ok1(strcmp(p, lines[0]) == 0); + + p = rbuf_read_str(&in, '\n', realloc); + ok1(p); + ok1(strcmp(p, lines[1]) == 0); + + for (i = 2; lines[i]; i++) { + ok1(p = rbuf_read_str(&in, '\n', realloc)); + ok1(strcmp(p, lines[i]) == 0); + } + + p = rbuf_read_str(&in, '\n', realloc); + ok1(errno == 0); + ok1(p == NULL); + free(in.buf); + + /* This exits depending on whether all tests passed */ + return exit_status(); +} diff --git a/ccan/rbuf/test/run-term-eof.c b/ccan/rbuf/test/run-term-eof.c new file mode 100644 index 00000000..097dcbbd --- /dev/null +++ b/ccan/rbuf/test/run-term-eof.c @@ -0,0 +1,44 @@ +#include +/* Include the C files directly. */ +#include +#include +#include +#include +#include +#include + +int main(void) +{ + struct rbuf in; + char buf[4096], *p; + int fd = open("test/run-term-eof.c", O_RDONLY), len; + + /* This is how many tests you plan to run */ + plan_tests(6); + + /* Grab ourselves for comparison. */ + len = read(fd, buf, sizeof(buf)); + buf[len] = '\0'; + lseek(fd, SEEK_SET, 0); + + /* We have exact-size buffer, which causes problems adding term. */ + rbuf_init(&in, fd, malloc(len), len); + p = rbuf_read_str(&in, 64, NULL); /* At symbol does not appear. */ + ok1(errno == ENOMEM); + ok1(!p); + /* This should succeed... */ + p = rbuf_read_str(&in, 64, realloc); + ok1(p); + ok1(strcmp(p, buf) == 0); + free(in.buf); + + /* Try again. */ + lseek(fd, SEEK_SET, 0); + rbuf_init(&in, fd, malloc(len), len); + p = rbuf_read_str(&in, 64, realloc); + ok1(p); + ok1(strcmp(p, buf) == 0); + free(in.buf); + + return exit_status(); +} diff --git a/ccan/rbuf/test/run.c b/ccan/rbuf/test/run.c new file mode 100644 index 00000000..e5761881 --- /dev/null +++ b/ccan/rbuf/test/run.c @@ -0,0 +1,68 @@ +#include +/* Include the C files directly. */ +#include +#include +#include +#include +#include +#include + +int main(void) +{ + struct rbuf in; + char buf[4096]; + char *lines[100], *p; + int i, fd = open("test/run.c", O_RDONLY), len; + + /* This is how many tests you plan to run */ + plan_tests(144); + + /* Grab ourselves for comparison. */ + len = read(fd, buf, sizeof(buf)); + buf[len] = '\0'; + lseek(fd, SEEK_SET, 0); + + for (i = 0, p = buf; *p; i++) { + lines[i] = p; + p = strchr(p, '\n'); + *p = '\0'; + p++; + } + lines[i] = NULL; + + rbuf_init(&in, fd, malloc(31), 31); + ok1(in.fd == fd); + ok1(in.buf_end - in.buf == 31); + p = rbuf_read_str(&in, '\n', NULL); + ok1(p); + ok1(strcmp(p, lines[0]) == 0); + + p = rbuf_read_str(&in, '\n', realloc); + ok1(p); + ok1(strcmp(p, lines[1]) == 0); + + for (i = 2; lines[i]; i++) { + ok1(p = rbuf_read_str(&in, '\n', realloc)); + ok1(strcmp(p, lines[i]) == 0); + } + + p = rbuf_read_str(&in, '\n', realloc); + ok1(errno == 0); + ok1(p == NULL); + free(in.buf); + + /* Another way of reading the entire (text) file. */ + lseek(fd, SEEK_SET, 0); + rbuf_init(&in, fd, NULL, 0); + p = rbuf_read_str(&in, 0, realloc); + ok1(p); + ok1(strlen(p) == len); + + close(fd); + p = rbuf_read_str(&in, 0, realloc); + ok1(errno == EBADF); + ok1(!p); + free(in.buf); + + return exit_status(); +} -- 2.39.2