From d8a270fdfc57ffe7603028ecda99eeab3889ca70 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 17 Sep 2018 10:54:45 +0930 Subject: [PATCH] rbuf: adapt to work on ccan/membuf. Signed-off-by: Rusty Russell --- ccan/rbuf/_info | 9 ++- ccan/rbuf/rbuf.c | 100 +++++++++--------------- ccan/rbuf/rbuf.h | 126 +++++++++++++++++------------- ccan/rbuf/test/run-all.c | 30 ++++--- ccan/rbuf/test/run-open.c | 4 +- ccan/rbuf/test/run-partial-read.c | 16 ++-- ccan/rbuf/test/run-term-eof.c | 36 +++++++-- ccan/rbuf/test/run.c | 34 +++++--- 8 files changed, 195 insertions(+), 160 deletions(-) diff --git a/ccan/rbuf/_info b/ccan/rbuf/_info index 7c9224d0..8a7d46e8 100644 --- a/ccan/rbuf/_info +++ b/ccan/rbuf/_info @@ -23,19 +23,19 @@ * char *word; * * if (argv[1]) { - * if (!rbuf_open(&in, argv[1], NULL, 0)) + * if (!rbuf_open(&in, argv[1], NULL, 0, membuf_realloc)) * err(1, "Failed opening %s", argv[1]); * } else - * rbuf_init(&in, STDIN_FILENO, NULL, 0); + * rbuf_init(&in, STDIN_FILENO, NULL, 0, membuf_realloc); * - * while ((word = rbuf_read_str(&in, ' ', realloc)) != NULL) + * while ((word = rbuf_read_str(&in, ' ')) != NULL) * printf("%s*", word); * * if (errno) * err(1, "Reading %s", argv[1] ? argv[1] : ""); * * // Free the buffer, just because we can. - * free(in.buf); + * free(rbuf_cleanup(&in)); * return 0; * } */ @@ -46,6 +46,7 @@ int main(int argc, char *argv[]) return 1; if (strcmp(argv[1], "depends") == 0) { + printf("ccan/membuf\n"); return 0; } diff --git a/ccan/rbuf/rbuf.c b/ccan/rbuf/rbuf.c index 0e210a9e..d8d658d3 100644 --- a/ccan/rbuf/rbuf.c +++ b/ccan/rbuf/rbuf.c @@ -7,21 +7,17 @@ #include #include -bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max) +bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max, + void *(*expandfn)(struct membuf *, void *, size_t)) { int fd = open(name, O_RDONLY); if (fd >= 0) { - rbuf_init(rbuf, fd, buf, buf_max); + rbuf_init(rbuf, fd, buf, buf_max, expandfn); return true; } return false; } -static size_t rem(const struct rbuf *buf) -{ - return buf->buf_end - (buf->start + buf->len); -} - size_t rbuf_good_size(int fd) { struct stat st; @@ -31,100 +27,78 @@ size_t rbuf_good_size(int fd) return 4096; } -static bool enlarge_buf(struct rbuf *buf, size_t len, - void *(*resize)(void *buf, size_t len)) +static ssize_t get_more(struct rbuf *rbuf) { - char *new; - if (!resize) { - errno = ENOMEM; - return false; - } - if (!len) - len = rbuf_good_size(buf->fd); - new = resize(buf->buf, len); - if (!new) - return false; - buf->start += (new - buf->buf); - buf->buf = new; - buf->buf_end = new + len; - return true; -} + ssize_t r; -static ssize_t get_more(struct rbuf *rbuf, - void *(*resize)(void *buf, size_t len)) -{ - size_t r; + /* This is so we only call rbuf_good_size once. */ + if (tcon_unwrap(&rbuf->m)->max_elems == 0) + membuf_prepare_space(&rbuf->m, rbuf_good_size(rbuf->fd)); + else /* membuf doubles internally, so just ask for anything. */ + membuf_prepare_space(&rbuf->m, 1); - if (rbuf->start + rbuf->len == rbuf->buf_end) { - if (!enlarge_buf(rbuf, (rbuf->buf_end - rbuf->buf) * 2, resize)) - return -1; - } + /* This happens if realloc fails (errno already ENOMEM) */ + if (!membuf_num_space(&rbuf->m)) + return -1; - r = read(rbuf->fd, rbuf->start + rbuf->len, rem(rbuf)); + r = read(rbuf->fd, membuf_space(&rbuf->m), membuf_num_space(&rbuf->m)); if (r <= 0) return r; - rbuf->len += r; + membuf_add(&rbuf->m, r); return r; } -void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)) +void *rbuf_fill_all(struct rbuf *rbuf) { ssize_t r; - /* Move back to start of buffer if we're empty. */ - if (!rbuf->len) - rbuf->start = rbuf->buf; - - while ((r = get_more(rbuf, resize)) != 0) + while ((r = get_more(rbuf)) != 0) if (r < 0) return NULL; - return rbuf->start; + return rbuf_start(rbuf); } -void *rbuf_fill(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)) +void *rbuf_fill(struct rbuf *rbuf) { - if (!rbuf->len) { - rbuf->start = rbuf->buf; - if (get_more(rbuf, resize) < 0) + if (!rbuf_len(rbuf)) { + if (get_more(rbuf) < 0) return NULL; } - return rbuf->start; + return rbuf_start(rbuf); } -char *rbuf_read_str(struct rbuf *rbuf, char term, - void *(*resize)(void *buf, size_t len)) +char *rbuf_read_str(struct rbuf *rbuf, char term) { - char *p, *ret; + char *p; ssize_t r = 0; size_t prev = 0; - /* Move back to start of buffer if we're empty. */ - if (!rbuf->len) - rbuf->start = rbuf->buf; - - while (!(p = memchr(rbuf->start + prev, term, rbuf->len - prev))) { + while (!(p = memchr(membuf_elems(&rbuf->m) + prev, + term, + membuf_num_elems(&rbuf->m) - prev))) { prev += r; - r = get_more(rbuf, resize); + r = get_more(rbuf); if (r < 0) return NULL; /* EOF with no term. */ if (r == 0) { + char *ret; + size_t len = rbuf_len(rbuf); + /* Nothing read at all? */ - if (!rbuf->len && term) { + if (!len && term) { errno = 0; return NULL; } + /* Put term after input (get_more made room). */ - assert(rbuf->start + rbuf->len < rbuf->buf_end); - rbuf->start[rbuf->len] = '\0'; - ret = rbuf->start; - rbuf_consume(rbuf, rbuf->len); + assert(membuf_num_space(&rbuf->m) > 0); + ret = membuf_consume(&rbuf->m, len); + ret[len] = '\0'; return ret; } } *p = '\0'; - ret = rbuf->start; - rbuf_consume(rbuf, p + 1 - ret); - return ret; + return membuf_consume(&rbuf->m, p + 1 - (char *)rbuf_start(rbuf)); } diff --git a/ccan/rbuf/rbuf.h b/ccan/rbuf/rbuf.h index ab1504a7..7b5306c4 100644 --- a/ccan/rbuf/rbuf.h +++ b/ccan/rbuf/rbuf.h @@ -5,41 +5,36 @@ #include // For UCHAR_MAX #include #include +#include struct rbuf { int fd; - - /* Where to read next. */ - char *start; - /* How much of what is there is valid. */ - size_t len; - - /* The entire buffer memory we have to work with. */ - char *buf, *buf_end; + MEMBUF(char) m; }; /** * rbuf_init - set up a buffer. - * @buf: the struct rbuf. + * @rbuf: the struct rbuf. * @fd: the file descriptor. * @buf: the buffer to use. * @buf_max: the size of the buffer. + * @expandfn: usually membuf_realloc. */ -static inline void rbuf_init(struct rbuf *buf, - int fd, char *buffer, size_t buf_max) +static inline void rbuf_init(struct rbuf *rbuf, + int fd, char *buffer, size_t buf_max, + void *(*expandfn)(struct membuf *, void *, size_t)) { - buf->fd = fd; - buf->start = buf->buf = buffer; - buf->len = 0; - buf->buf_end = buffer + buf_max; + rbuf->fd = fd; + membuf_init(&rbuf->m, buffer, buf_max, expandfn); } /** * rbuf_open - set up a buffer by opening a file. - * @buf: the struct rbuf. + * @rbuf: the struct rbuf. * @filename: the filename * @buf: the buffer to use. * @buf_max: the size of the buffer. + * @expandfn: usually membuf_realloc. * * Returns false if the open fails. If @buf_max is 0, then the buffer * will be resized to rbuf_good_size() on first rbuf_fill. @@ -47,10 +42,11 @@ static inline void rbuf_init(struct rbuf *buf, * Example: * struct rbuf in; * - * if (!rbuf_open(&in, "foo", NULL, 0)) + * if (!rbuf_open(&in, "foo", NULL, 0, membuf_realloc)) * err(1, "Could not open foo"); */ -bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max); +bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max, + void *(*expandfn)(struct membuf *, void *, size_t)); /** * rbuf_good_size - get a good buffer size for this fd. @@ -62,73 +58,82 @@ size_t rbuf_good_size(int fd); /** * rbuf_fill - read into a buffer if it's empty. - * @buf: the struct rbuf - * @resize: the call to resize the buffer. + * @rbuf: the struct rbuf * - * If @resize is needed and is NULL, or returns false, rbuf_read will - * return NULL (with errno set to ENOMEM). If a read fails, then NULL - * is also returned. If there is nothing more to read, it will return - * NULL with errno set to 0. Otherwise, returns @buf->start; @buf->len - * is the valid length of the buffer. + * If @expandfn fails, rbuf_fill will return NULL (with errno set to ENOMEM). + * If a read fails, then NULL is also returned. If there is nothing more to + * read, it will return NULL with errno set to 0. Otherwise, returns first + * populated bytes (aka. rbuf_start()); rbuf_len() is the valid length of the + * buffer. * * You need to call rbuf_consume() to mark data in the buffer as * consumed. * * Example: - * while (rbuf_fill(&in, realloc)) { - * printf("%.*s\n", (int)in.len, in.start); - * rbuf_consume(&in, in.len); + * while (rbuf_fill(&in)) { + * printf("%.*s\n", (int)rbuf_len(&in), rbuf_start(&in)); + * rbuf_consume(&in, rbuf_len(&in)); * } * if (errno) * err(1, "reading foo"); */ -void *rbuf_fill(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)); +void *rbuf_fill(struct rbuf *rbuf); /** * rbuf_consume - helper to use up data in a buffer. - * @buf: the struct rbuf + * @rbuf: the struct rbuf * @len: the length (from @buf->start) you used. * * After rbuf_fill() you should indicate the data you've used with * rbuf_consume(). That way rbuf_fill() will know if it has anything * to do. */ -static inline void rbuf_consume(struct rbuf *buf, size_t len) +static inline void rbuf_consume(struct rbuf *rbuf, size_t len) +{ + membuf_consume(&rbuf->m, len); +} + +/** + * rbuf_len - helper to determine how many bytes in rbuf + * @rbuf: the struct rbuf + */ +static inline size_t rbuf_len(const struct rbuf *rbuf) +{ + return membuf_num_elems(&rbuf->m); +} + +/** + * rbuf_start - helper to get pointert to unconsumed bytes in rbuf + * @rbuf: the struct rbuf + */ +static inline char *rbuf_start(const struct rbuf *rbuf) { - buf->len -= len; - buf->start += len; + return membuf_elems(&rbuf->m); } /** * rbuf_fill_all - read rest of file into a buffer. - * @buf: the struct rbuf - * @resize: the call to resize the buffer. + * @rbuf: the struct rbuf * - * If @resize is needed and is NULL, or returns false, rbuf_read_all - * will return NULL (with errno set to ENOMEM). If a read fails, - * then NULL is also returned, otherwise returns @buf->start. + * If a read or @expandfn fails then NULL returned, otherwise returns + * @rbuf->start. * * Example: - * if (!rbuf_fill_all(&in, realloc)) { - * if (errno) - * err(1, "reading foo"); - * } + * if (!rbuf_fill_all(&in)) + * err(1, "reading foo"); */ -void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)); +void *rbuf_fill_all(struct rbuf *rbuf); /** * rbuf_read_str - fill into a buffer up to a terminator, and consume string. - * @buf: the struct rbuf + * @rbuf: the struct rbuf * @term: the character to terminate the read. - * @resize: the call to resize the buffer. * - * If @resize is needed and is NULL, or returns false, rbuf_read_str - * will return NULL (with errno set to ENOMEM). If a read fails, - * then NULL is also returned, otherwise the next string. It - * replaces the terminator @term (if any) with NUL, otherwise NUL + * If a read or @expandfn fails, then NULL is returned, otherwise the next + * string. It replaces the terminator @term (if any) with NUL, otherwise NUL * is placed after EOF. If you need to, you can tell this has happened - * because the nul terminator will be at @buf->start (normally it will - * be at @buf->start - 1). + * because the nul terminator will be at rbuf_start(@rbuf) (normally it will be + * at rbuf_start(@rbuf) - 1). * * If there is nothing remaining to be read, NULL is returned with * errno set to 0, unless @term is NUL, in which case it returns the @@ -140,7 +145,7 @@ void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)); * Example: * char *line; * - * line = rbuf_read_str(&in, '\n', realloc); + * line = rbuf_read_str(&in, '\n'); * if (!line) { * if (errno) * err(1, "reading foo"); @@ -150,7 +155,20 @@ void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)); * printf("First line is %s\n", line); * */ -char *rbuf_read_str(struct rbuf *rbuf, char term, - void *(*resize)(void *buf, size_t len)); +char *rbuf_read_str(struct rbuf *rbuf, char term); +/** + * rbuf_cleanup - reset rbuf, return buffer for freeing. + * @rbuf: the struct rbuf + * + * The rbuf will be empty after this, and crash if you try to use it. + * You can rbuf_init() it again, however. + * + * Example: + * free(rbuf_cleanup(&in)); + */ +static inline char *rbuf_cleanup(struct rbuf *rbuf) +{ + return membuf_cleanup(&rbuf->m); +} #endif /* CCAN_RBUF_H */ diff --git a/ccan/rbuf/test/run-all.c b/ccan/rbuf/test/run-all.c index 2cd0b424..d7c1a3f1 100644 --- a/ccan/rbuf/test/run-all.c +++ b/ccan/rbuf/test/run-all.c @@ -7,6 +7,14 @@ #include #include +static bool test_realloc_fail; +static void *test_realloc(struct membuf *mb, void *buf, size_t n) +{ + if (test_realloc_fail) + return NULL; + return realloc(buf, n); +} + int main(void) { struct rbuf in; @@ -24,25 +32,29 @@ int main(void) } close(fd); - ok1(rbuf_open(&in, "run-all-file", NULL, 0)); - /* Can't fill without realloc. */ - ok1(!rbuf_fill(&in, NULL)); + ok1(rbuf_open(&in, "run-all-file", NULL, 0, test_realloc)); + /* Can't fill if realloc fails. */ + test_realloc_fail = true; + ok1(!rbuf_fill(&in)); ok1(errno == ENOMEM); - ok1(rbuf_fill(&in, realloc)); + test_realloc_fail = false; + ok1(rbuf_fill(&in)); /* But can't load in whole file. */ - ok1(!rbuf_fill_all(&in, NULL)); + test_realloc_fail = true; + ok1(!rbuf_fill_all(&in)); ok1(errno == ENOMEM); - ok1(rbuf_fill_all(&in, realloc)); - ok1(in.len == size); + test_realloc_fail = false; + ok1(rbuf_fill_all(&in)); + ok1(rbuf_len(&in) == size); for (i = 0; i * sizeof(buf) < size; i++) { memset(buf, 0x42 + i, sizeof(buf)); - if (memcmp(buf, in.start, sizeof(buf)) != 0) { + if (memcmp(buf, rbuf_start(&in), sizeof(buf)) != 0) { fail("Bad buffer contents"); break; } rbuf_consume(&in, sizeof(buf)); } - free(in.buf); + free(membuf_cleanup(&in.m)); /* This exits depending on whether all tests passed */ return exit_status(); diff --git a/ccan/rbuf/test/run-open.c b/ccan/rbuf/test/run-open.c index 5678e42e..4e8277f2 100644 --- a/ccan/rbuf/test/run-open.c +++ b/ccan/rbuf/test/run-open.c @@ -14,9 +14,9 @@ int main(void) /* This is how many tests you plan to run */ plan_tests(5); - ok1(!rbuf_open(&in, "nonexistent-file", NULL, 0)); + ok1(!rbuf_open(&in, "nonexistent-file", NULL, 0, NULL)); ok1(errno == ENOENT); - ok1(rbuf_open(&in, "test/run-open.c", NULL, 0)); + ok1(rbuf_open(&in, "test/run-open.c", NULL, 0, NULL)); ok1(close(in.fd) == 0); /* If this fails to stat, it should fall back */ ok1(rbuf_good_size(in.fd) == 4096); diff --git a/ccan/rbuf/test/run-partial-read.c b/ccan/rbuf/test/run-partial-read.c index 362e6b60..727fade8 100644 --- a/ccan/rbuf/test/run-partial-read.c +++ b/ccan/rbuf/test/run-partial-read.c @@ -27,7 +27,7 @@ int main(void) int i, fd = open("test/run.c", O_RDONLY); /* This is how many tests you plan to run */ - plan_tests(140); + plan_tests(160); /* Grab ourselves for comparison. */ buf[full_read(fd, buf, sizeof(buf))] = '\0'; @@ -41,26 +41,26 @@ int main(void) } lines[i] = NULL; - rbuf_init(&in, fd, malloc(31), 31); + rbuf_init(&in, fd, malloc(31), 31, membuf_realloc); ok1(in.fd == fd); - ok1(in.buf_end - in.buf == 31); - p = rbuf_read_str(&in, '\n', NULL); + ok1(membuf_num_space(&in.m) == 31); + p = rbuf_read_str(&in, '\n'); ok1(p); ok1(strcmp(p, lines[0]) == 0); - p = rbuf_read_str(&in, '\n', realloc); + p = rbuf_read_str(&in, '\n'); ok1(p); ok1(strcmp(p, lines[1]) == 0); for (i = 2; lines[i]; i++) { - ok1(p = rbuf_read_str(&in, '\n', realloc)); + ok1(p = rbuf_read_str(&in, '\n')); ok1(strcmp(p, lines[i]) == 0); } - p = rbuf_read_str(&in, '\n', realloc); + p = rbuf_read_str(&in, '\n'); ok1(errno == 0); ok1(p == NULL); - free(in.buf); + free(membuf_cleanup(&in.m)); /* This exits depending on whether all tests passed */ return exit_status(); diff --git a/ccan/rbuf/test/run-term-eof.c b/ccan/rbuf/test/run-term-eof.c index 097dcbbd..25b060f7 100644 --- a/ccan/rbuf/test/run-term-eof.c +++ b/ccan/rbuf/test/run-term-eof.c @@ -7,6 +7,14 @@ #include #include +static bool test_realloc_fail; +static void *test_realloc(struct membuf *mb, void *buf, size_t n) +{ + if (test_realloc_fail) + return NULL; + return realloc(buf, n); +} + int main(void) { struct rbuf in; @@ -14,7 +22,7 @@ int main(void) int fd = open("test/run-term-eof.c", O_RDONLY), len; /* This is how many tests you plan to run */ - plan_tests(6); + plan_tests(10); /* Grab ourselves for comparison. */ len = read(fd, buf, sizeof(buf)); @@ -22,23 +30,35 @@ int main(void) lseek(fd, SEEK_SET, 0); /* We have exact-size buffer, which causes problems adding term. */ - rbuf_init(&in, fd, malloc(len), len); - p = rbuf_read_str(&in, 64, NULL); /* At symbol does not appear. */ + rbuf_init(&in, fd, malloc(len), len, test_realloc); + test_realloc_fail = true; + p = rbuf_read_str(&in, 64); /* At symbol does not appear. */ ok1(errno == ENOMEM); ok1(!p); /* This should succeed... */ - p = rbuf_read_str(&in, 64, realloc); + test_realloc_fail = false; + p = rbuf_read_str(&in, 64); ok1(p); ok1(strcmp(p, buf) == 0); - free(in.buf); + ok1(rbuf_start(&in) == p + strlen(p)); + free(rbuf_cleanup(&in)); /* Try again. */ lseek(fd, SEEK_SET, 0); - rbuf_init(&in, fd, malloc(len), len); - p = rbuf_read_str(&in, 64, realloc); + rbuf_init(&in, fd, malloc(len), len, test_realloc); + p = rbuf_read_str(&in, 64); ok1(p); ok1(strcmp(p, buf) == 0); - free(in.buf); + ok1(rbuf_start(&in) == p + strlen(p)); + free(rbuf_cleanup(&in)); + /* Normal case, we get rbuf_start after nul */ + lseek(fd, SEEK_SET, 0); + rbuf_init(&in, fd, NULL, 0, test_realloc); + p = rbuf_read_str(&in, '^'); + ok1(p); + ok1(rbuf_start(&in) == p + strlen(p) + 1); + free(rbuf_cleanup(&in)); + return exit_status(); } diff --git a/ccan/rbuf/test/run.c b/ccan/rbuf/test/run.c index e5761881..593ef270 100644 --- a/ccan/rbuf/test/run.c +++ b/ccan/rbuf/test/run.c @@ -7,6 +7,14 @@ #include #include +static bool test_realloc_fail; +static void *test_realloc(struct membuf *mb, void *buf, size_t n) +{ + if (test_realloc_fail) + return NULL; + return realloc(buf, n); +} + int main(void) { struct rbuf in; @@ -15,7 +23,7 @@ int main(void) int i, fd = open("test/run.c", O_RDONLY), len; /* This is how many tests you plan to run */ - plan_tests(144); + plan_tests(164); /* Grab ourselves for comparison. */ len = read(fd, buf, sizeof(buf)); @@ -30,39 +38,41 @@ int main(void) } lines[i] = NULL; - rbuf_init(&in, fd, malloc(31), 31); + rbuf_init(&in, fd, malloc(31), 31, test_realloc); ok1(in.fd == fd); - ok1(in.buf_end - in.buf == 31); - p = rbuf_read_str(&in, '\n', NULL); + ok1(membuf_num_space(&in.m) == 31); + test_realloc_fail = true; + p = rbuf_read_str(&in, '\n'); ok1(p); ok1(strcmp(p, lines[0]) == 0); - p = rbuf_read_str(&in, '\n', realloc); + test_realloc_fail = false; + p = rbuf_read_str(&in, '\n'); ok1(p); ok1(strcmp(p, lines[1]) == 0); for (i = 2; lines[i]; i++) { - ok1(p = rbuf_read_str(&in, '\n', realloc)); + ok1(p = rbuf_read_str(&in, '\n')); ok1(strcmp(p, lines[i]) == 0); } - p = rbuf_read_str(&in, '\n', realloc); + p = rbuf_read_str(&in, '\n'); ok1(errno == 0); ok1(p == NULL); - free(in.buf); + free(rbuf_cleanup(&in)); /* Another way of reading the entire (text) file. */ lseek(fd, SEEK_SET, 0); - rbuf_init(&in, fd, NULL, 0); - p = rbuf_read_str(&in, 0, realloc); + rbuf_init(&in, fd, NULL, 0, test_realloc); + p = rbuf_read_str(&in, 0); ok1(p); ok1(strlen(p) == len); close(fd); - p = rbuf_read_str(&in, 0, realloc); + p = rbuf_read_str(&in, 0); ok1(errno == EBADF); ok1(!p); - free(in.buf); + free(rbuf_cleanup(&in)); return exit_status(); } -- 2.39.2