tdb2: more stats
[ccan] / ccan / tdb2 / tools / speed.c
1 /* Simple speed test for TDB */
2 #include <err.h>
3 #include <time.h>
4 #include <sys/types.h>
5 #include <sys/stat.h>
6 #include <unistd.h>
7 #include <sys/time.h>
8 #include <fcntl.h>
9 #include <stdio.h>
10 #include <stdlib.h>
11 #include <string.h>
12 #include <stdbool.h>
13 #include <ccan/tdb2/tdb2.h>
14
15 /* Nanoseconds per operation */
16 static size_t normalize(const struct timeval *start,
17                         const struct timeval *stop,
18                         unsigned int num)
19 {
20         struct timeval diff;
21
22         timersub(stop, start, &diff);
23
24         /* Floating point is more accurate here. */
25         return (double)(diff.tv_sec * 1000000 + diff.tv_usec)
26                 / num * 1000;
27 }
28
29 static size_t file_size(void)
30 {
31         struct stat st;
32
33         if (stat("/tmp/speed.tdb", &st) != 0)
34                 return -1;
35         return st.st_size;
36 }
37
38 static int count_record(struct tdb_context *tdb,
39                         TDB_DATA key, TDB_DATA data, void *p)
40 {
41         int *total = p;
42         *total += *(int *)data.dptr;
43         return 0;
44 }
45
46 static void dump_and_clear_stats(struct tdb_context **tdb,
47                                  int flags,
48                                  union tdb_attribute *attr)
49 {
50         union tdb_attribute stats;
51         enum TDB_ERROR ecode;
52
53         stats.base.attr = TDB_ATTRIBUTE_STATS;
54         stats.stats.size = sizeof(stats.stats);
55         ecode = tdb_get_attribute(*tdb, &stats);
56         if (ecode != TDB_SUCCESS)
57                 errx(1, "Getting stats: %s", tdb_errorstr(ecode));
58
59         printf("allocs = %llu\n",
60                (unsigned long long)stats.stats.allocs);
61         printf("  alloc_subhash = %llu\n",
62                (unsigned long long)stats.stats.alloc_subhash);
63         printf("  alloc_chain = %llu\n",
64                (unsigned long long)stats.stats.alloc_chain);
65         printf("  alloc_bucket_exact = %llu\n",
66                (unsigned long long)stats.stats.alloc_bucket_exact);
67         printf("  alloc_bucket_max = %llu\n",
68                (unsigned long long)stats.stats.alloc_bucket_max);
69         printf("  alloc_leftover = %llu\n",
70                (unsigned long long)stats.stats.alloc_leftover);
71         printf("  alloc_coalesce_tried = %llu\n",
72                (unsigned long long)stats.stats.alloc_coalesce_tried);
73         printf("    alloc_coalesce_iterate_clash = %llu\n",
74                (unsigned long long)stats.stats.alloc_coalesce_iterate_clash);
75         printf("    alloc_coalesce_lockfail = %llu\n",
76                (unsigned long long)stats.stats.alloc_coalesce_lockfail);
77         printf("    alloc_coalesce_race = %llu\n",
78                (unsigned long long)stats.stats.alloc_coalesce_race);
79         printf("    alloc_coalesce_succeeded = %llu\n",
80                (unsigned long long)stats.stats.alloc_coalesce_succeeded);
81         printf("      alloc_coalesce_num_merged = %llu\n",
82                (unsigned long long)stats.stats.alloc_coalesce_num_merged);
83         printf("compares = %llu\n",
84                (unsigned long long)stats.stats.compares);
85         printf("  compare_wrong_bucket = %llu\n",
86                (unsigned long long)stats.stats.compare_wrong_bucket);
87         printf("  compare_wrong_offsetbits = %llu\n",
88                (unsigned long long)stats.stats.compare_wrong_offsetbits);
89         printf("  compare_wrong_keylen = %llu\n",
90                (unsigned long long)stats.stats.compare_wrong_keylen);
91         printf("  compare_wrong_rechash = %llu\n",
92                (unsigned long long)stats.stats.compare_wrong_rechash);
93         printf("  compare_wrong_keycmp = %llu\n",
94                (unsigned long long)stats.stats.compare_wrong_keycmp);
95         printf("transactions = %llu\n",
96                (unsigned long long)stats.stats.transactions);
97         printf("  transaction_cancel = %llu\n",
98                (unsigned long long)stats.stats.transaction_cancel);
99         printf("  transaction_nest = %llu\n",
100                (unsigned long long)stats.stats.transaction_nest);
101         printf("  transaction_expand_file = %llu\n",
102                (unsigned long long)stats.stats.transaction_expand_file);
103         printf("  transaction_read_direct = %llu\n",
104                (unsigned long long)stats.stats.transaction_read_direct);
105         printf("    transaction_read_direct_fail = %llu\n",
106                (unsigned long long)stats.stats.transaction_read_direct_fail);
107         printf("  transaction_write_direct = %llu\n",
108                (unsigned long long)stats.stats.transaction_write_direct);
109         printf("    transaction_write_direct_fail = %llu\n",
110                (unsigned long long)stats.stats.transaction_write_direct_fail);
111         printf("expands = %llu\n",
112                (unsigned long long)stats.stats.expands);
113         printf("frees = %llu\n",
114                (unsigned long long)stats.stats.frees);
115         printf("locks = %llu\n",
116                (unsigned long long)stats.stats.locks);
117         printf("  lock_lowlevel = %llu\n",
118                (unsigned long long)stats.stats.lock_lowlevel);
119         printf("  lock_nonblock = %llu\n",
120                (unsigned long long)stats.stats.lock_nonblock);
121         printf("    lock_nonblock_fail = %llu\n",
122                (unsigned long long)stats.stats.lock_nonblock_fail);
123
124         /* Now clear. */
125         tdb_close(*tdb);
126         *tdb = tdb_open("/tmp/speed.tdb", flags, O_RDWR, 0, attr);
127 }
128
129 static void tdb_log(struct tdb_context *tdb, enum tdb_log_level level,
130                     const char *message, void *data)
131 {
132         fputs(message, stderr);
133         putc('\n', stderr);
134 }
135
136 int main(int argc, char *argv[])
137 {
138         unsigned int i, j, num = 1000, stage = 0, stopat = -1;
139         int flags = TDB_DEFAULT;
140         bool transaction = false, summary = false;
141         TDB_DATA key, data;
142         struct tdb_context *tdb;
143         struct timeval start, stop;
144         union tdb_attribute seed, log;
145         bool do_stats = false;
146         enum TDB_ERROR ecode;
147
148         /* Try to keep benchmarks even. */
149         seed.base.attr = TDB_ATTRIBUTE_SEED;
150         seed.base.next = NULL;
151         seed.seed.seed = 0;
152
153         log.base.attr = TDB_ATTRIBUTE_LOG;
154         log.base.next = &seed;
155         log.log.fn = tdb_log;
156
157         if (argv[1] && strcmp(argv[1], "--internal") == 0) {
158                 flags = TDB_INTERNAL;
159                 argc--;
160                 argv++;
161         }
162         if (argv[1] && strcmp(argv[1], "--transaction") == 0) {
163                 transaction = true;
164                 argc--;
165                 argv++;
166         }
167         if (argv[1] && strcmp(argv[1], "--no-sync") == 0) {
168                 flags |= TDB_NOSYNC;
169                 argc--;
170                 argv++;
171         }
172         if (argv[1] && strcmp(argv[1], "--summary") == 0) {
173                 summary = true;
174                 argc--;
175                 argv++;
176         }
177         if (argv[1] && strcmp(argv[1], "--stats") == 0) {
178                 do_stats = true;
179                 argc--;
180                 argv++;
181         }
182
183         tdb = tdb_open("/tmp/speed.tdb", flags, O_RDWR|O_CREAT|O_TRUNC,
184                        0600, &log);
185         if (!tdb)
186                 err(1, "Opening /tmp/speed.tdb");
187
188         key.dptr = (void *)&i;
189         key.dsize = sizeof(i);
190         data = key;
191
192         if (argv[1]) {
193                 num = atoi(argv[1]);
194                 argv++;
195                 argc--;
196         }
197
198         if (argv[1]) {
199                 stopat = atoi(argv[1]);
200                 argv++;
201                 argc--;
202         }
203
204         /* Add 1000 records. */
205         printf("Adding %u records: ", num); fflush(stdout);
206         if (transaction && (ecode = tdb_transaction_start(tdb)))
207                 errx(1, "starting transaction: %s", tdb_errorstr(ecode));
208         gettimeofday(&start, NULL);
209         for (i = 0; i < num; i++)
210                 if ((ecode = tdb_store(tdb, key, data, TDB_INSERT)) != 0)
211                         errx(1, "Inserting key %u in tdb: %s",
212                              i, tdb_errorstr(ecode));
213         gettimeofday(&stop, NULL);
214         if (transaction && (ecode = tdb_transaction_commit(tdb)))
215                 errx(1, "committing transaction: %s", tdb_errorstr(ecode));
216         printf(" %zu ns (%zu bytes)\n",
217                normalize(&start, &stop, num), file_size());
218
219         if (tdb_check(tdb, NULL, NULL))
220                 errx(1, "tdb_check failed!");
221         if (summary) {
222                 char *sumstr = NULL;
223                 tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
224                 printf("%s\n", sumstr);
225                 free(sumstr);
226         }
227         if (do_stats)
228                 dump_and_clear_stats(&tdb, flags, &log);
229
230         if (++stage == stopat)
231                 exit(0);
232
233         /* Finding 1000 records. */
234         printf("Finding %u records: ", num); fflush(stdout);
235         if (transaction && (ecode = tdb_transaction_start(tdb)))
236                 errx(1, "starting transaction: %s", tdb_errorstr(ecode));
237         gettimeofday(&start, NULL);
238         for (i = 0; i < num; i++) {
239                 struct tdb_data dbuf;
240                 if ((ecode = tdb_fetch(tdb, key, &dbuf)) != TDB_SUCCESS
241                     || *(int *)dbuf.dptr != i) {
242                         errx(1, "Fetching key %u in tdb gave %u",
243                              i, ecode ? ecode : *(int *)dbuf.dptr);
244                 }
245         }
246         gettimeofday(&stop, NULL);
247         if (transaction && (ecode = tdb_transaction_commit(tdb)))
248                 errx(1, "committing transaction: %s", tdb_errorstr(ecode));
249         printf(" %zu ns (%zu bytes)\n",
250                normalize(&start, &stop, num), file_size());
251         if (tdb_check(tdb, NULL, NULL))
252                 errx(1, "tdb_check failed!");
253         if (summary) {
254                 char *sumstr = NULL;
255                 tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
256                 printf("%s\n", sumstr);
257                 free(sumstr);
258         }
259         if (do_stats)
260                 dump_and_clear_stats(&tdb, flags, &log);
261         if (++stage == stopat)
262                 exit(0);
263
264         /* Missing 1000 records. */
265         printf("Missing %u records: ", num); fflush(stdout);
266         if (transaction && (ecode = tdb_transaction_start(tdb)))
267                 errx(1, "starting transaction: %s", tdb_errorstr(ecode));
268         gettimeofday(&start, NULL);
269         for (i = num; i < num*2; i++) {
270                 struct tdb_data dbuf;
271                 ecode = tdb_fetch(tdb, key, &dbuf);
272                 if (ecode != TDB_ERR_NOEXIST)
273                         errx(1, "Fetching key %u in tdb gave %s",
274                              i, tdb_errorstr(ecode));
275         }
276         gettimeofday(&stop, NULL);
277         if (transaction && (ecode = tdb_transaction_commit(tdb)))
278                 errx(1, "committing transaction: %s", tdb_errorstr(ecode));
279         printf(" %zu ns (%zu bytes)\n",
280                normalize(&start, &stop, num), file_size());
281         if (tdb_check(tdb, NULL, NULL))
282                 errx(1, "tdb_check failed!");
283         if (summary) {
284                 char *sumstr = NULL;
285                 tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
286                 printf("%s\n", sumstr);
287                 free(sumstr);
288         }
289         if (do_stats)
290                 dump_and_clear_stats(&tdb, flags, &log);
291         if (++stage == stopat)
292                 exit(0);
293
294         /* Traverse 1000 records. */
295         printf("Traversing %u records: ", num); fflush(stdout);
296         if (transaction && (ecode = tdb_transaction_start(tdb)))
297                 errx(1, "starting transaction: %s", tdb_errorstr(ecode));
298         i = 0;
299         gettimeofday(&start, NULL);
300         if (tdb_traverse(tdb, count_record, &i) != num)
301                 errx(1, "Traverse returned wrong number of records");
302         if (i != (num - 1) * (num / 2))
303                 errx(1, "Traverse tallied to %u", i);
304         gettimeofday(&stop, NULL);
305         if (transaction && (ecode = tdb_transaction_commit(tdb)))
306                 errx(1, "committing transaction: %s", tdb_errorstr(ecode));
307         printf(" %zu ns (%zu bytes)\n",
308                normalize(&start, &stop, num), file_size());
309         if (tdb_check(tdb, NULL, NULL))
310                 errx(1, "tdb_check failed!");
311         if (summary) {
312                 char *sumstr = NULL;
313                 tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
314                 printf("%s\n", sumstr);
315                 free(sumstr);
316         }
317         if (do_stats)
318                 dump_and_clear_stats(&tdb, flags, &log);
319         if (++stage == stopat)
320                 exit(0);
321
322         /* Delete 1000 records (not in order). */
323         printf("Deleting %u records: ", num); fflush(stdout);
324         if (transaction && (ecode = tdb_transaction_start(tdb)))
325                 errx(1, "starting transaction: %s", tdb_errorstr(ecode));
326         gettimeofday(&start, NULL);
327         for (j = 0; j < num; j++) {
328                 i = (j + 100003) % num;
329                 if ((ecode = tdb_delete(tdb, key)) != TDB_SUCCESS)
330                         errx(1, "Deleting key %u in tdb: %s",
331                              i, tdb_errorstr(ecode));
332         }
333         gettimeofday(&stop, NULL);
334         if (transaction && (ecode = tdb_transaction_commit(tdb)))
335                 errx(1, "committing transaction: %s", tdb_errorstr(ecode));
336         printf(" %zu ns (%zu bytes)\n",
337                normalize(&start, &stop, num), file_size());
338         if (tdb_check(tdb, NULL, NULL))
339                 errx(1, "tdb_check failed!");
340         if (summary) {
341                 char *sumstr = NULL;
342                 tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
343                 printf("%s\n", sumstr);
344                 free(sumstr);
345         }
346         if (do_stats)
347                 dump_and_clear_stats(&tdb, flags, &log);
348         if (++stage == stopat)
349                 exit(0);
350
351         /* Re-add 1000 records (not in order). */
352         printf("Re-adding %u records: ", num); fflush(stdout);
353         if (transaction && (ecode = tdb_transaction_start(tdb)))
354                 errx(1, "starting transaction: %s", tdb_errorstr(ecode));
355         gettimeofday(&start, NULL);
356         for (j = 0; j < num; j++) {
357                 i = (j + 100003) % num;
358                 if ((ecode = tdb_store(tdb, key, data, TDB_INSERT)) != 0)
359                         errx(1, "Inserting key %u in tdb: %s",
360                              i, tdb_errorstr(ecode));
361         }
362         gettimeofday(&stop, NULL);
363         if (transaction && (ecode = tdb_transaction_commit(tdb)))
364                 errx(1, "committing transaction: %s", tdb_errorstr(ecode));
365         printf(" %zu ns (%zu bytes)\n",
366                normalize(&start, &stop, num), file_size());
367         if (tdb_check(tdb, NULL, NULL))
368                 errx(1, "tdb_check failed!");
369         if (summary) {
370                 char *sumstr = NULL;
371                 tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
372                 printf("%s\n", sumstr);
373                 free(sumstr);
374         }
375         if (do_stats)
376                 dump_and_clear_stats(&tdb, flags, &log);
377         if (++stage == stopat)
378                 exit(0);
379
380         /* Append 1000 records. */
381         if (transaction && (ecode = tdb_transaction_start(tdb)))
382                 errx(1, "starting transaction: %s", tdb_errorstr(ecode));
383         printf("Appending %u records: ", num); fflush(stdout);
384         gettimeofday(&start, NULL);
385         for (i = 0; i < num; i++)
386                 if ((ecode = tdb_append(tdb, key, data)) != TDB_SUCCESS)
387                         errx(1, "Appending key %u in tdb: %s",
388                              i, tdb_errorstr(ecode));
389         gettimeofday(&stop, NULL);
390         if (transaction && (ecode = tdb_transaction_commit(tdb)))
391                 errx(1, "committing transaction: %s", tdb_errorstr(ecode));
392         printf(" %zu ns (%zu bytes)\n",
393                normalize(&start, &stop, num), file_size());
394         if (tdb_check(tdb, NULL, NULL))
395                 errx(1, "tdb_check failed!");
396         if (summary) {
397                 char *sumstr = NULL;
398                 tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
399                 printf("%s\n", sumstr);
400                 free(sumstr);
401         }
402         if (++stage == stopat)
403                 exit(0);
404
405         /* Churn 1000 records: not in order! */
406         if (transaction && (ecode = tdb_transaction_start(tdb)))
407                 errx(1, "starting transaction: %s", tdb_errorstr(ecode));
408         printf("Churning %u records: ", num); fflush(stdout);
409         gettimeofday(&start, NULL);
410         for (j = 0; j < num; j++) {
411                 i = (j + 1000019) % num;
412                 if ((ecode = tdb_delete(tdb, key)) != TDB_SUCCESS)
413                         errx(1, "Deleting key %u in tdb: %s",
414                              i, tdb_errorstr(ecode));
415                 i += num;
416                 if ((ecode = tdb_store(tdb, key, data, TDB_INSERT)) != 0)
417                         errx(1, "Inserting key %u in tdb: %s",
418                              i, tdb_errorstr(ecode));
419         }
420         gettimeofday(&stop, NULL);
421         if (transaction && (ecode = tdb_transaction_commit(tdb)))
422                 errx(1, "committing transaction: %s", tdb_errorstr(ecode));
423         printf(" %zu ns (%zu bytes)\n",
424                normalize(&start, &stop, num), file_size());
425
426         if (tdb_check(tdb, NULL, NULL))
427                 errx(1, "tdb_check failed!");
428         if (summary) {
429                 char *sumstr = NULL;
430                 tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS, &sumstr);
431                 printf("%s\n", sumstr);
432                 free(sumstr);
433         }
434         if (do_stats)
435                 dump_and_clear_stats(&tdb, flags, &log);
436         if (++stage == stopat)
437                 exit(0);
438
439         return 0;
440 }