git.ozlabs.org Git - ccan/blob - ccan/ntdb/tools/speed.c

   1 /* Simple speed test for NTDB */
   2 #include <ccan/err/err.h>
   3 #include <time.h>
   4 #include <unistd.h>
   5 #include <sys/time.h>
   6 #include <fcntl.h>
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <stdbool.h>
  11 #include "ntdb.h"
  12
  13 /* Nanoseconds per operation */
  14 static size_t normalize(const struct timeval *start,
  15                         const struct timeval *stop,
  16                         unsigned int num)
  17 {
  18         struct timeval diff;
  19
  20         timersub(stop, start, &diff);
  21
  22         /* Floating point is more accurate here. */
  23         return (double)(diff.tv_sec * 1000000 + diff.tv_usec)
  24                 / num * 1000;
  25 }
  26
  27 static size_t file_size(void)
  28 {
  29         struct stat st;
  30
  31         if (stat("/tmp/speed.ntdb", &st) != 0)
  32                 return -1;
  33         return st.st_size;
  34 }
  35
  36 static int count_record(struct ntdb_context *ntdb,
  37                         NTDB_DATA key, NTDB_DATA data, void *p)
  38 {
  39         int *total = p;
  40         *total += *(int *)data.dptr;
  41         return 0;
  42 }
  43
  44 static void dump_and_clear_stats(struct ntdb_context **ntdb,
  45                                  int flags,
  46                                  union ntdb_attribute *attr)
  47 {
  48         union ntdb_attribute stats;
  49         enum NTDB_ERROR ecode;
  50
  51         stats.base.attr = NTDB_ATTRIBUTE_STATS;
  52         stats.stats.size = sizeof(stats.stats);
  53         ecode = ntdb_get_attribute(*ntdb, &stats);
  54         if (ecode != NTDB_SUCCESS)
  55                 errx(1, "Getting stats: %s", ntdb_errorstr(ecode));
  56
  57         printf("allocs = %llu\n",
  58                (unsigned long long)stats.stats.allocs);
  59         printf("  alloc_subhash = %llu\n",
  60                (unsigned long long)stats.stats.alloc_subhash);
  61         printf("  alloc_chain = %llu\n",
  62                (unsigned long long)stats.stats.alloc_chain);
  63         printf("  alloc_bucket_exact = %llu\n",
  64                (unsigned long long)stats.stats.alloc_bucket_exact);
  65         printf("  alloc_bucket_max = %llu\n",
  66                (unsigned long long)stats.stats.alloc_bucket_max);
  67         printf("  alloc_leftover = %llu\n",
  68                (unsigned long long)stats.stats.alloc_leftover);
  69         printf("  alloc_coalesce_tried = %llu\n",
  70                (unsigned long long)stats.stats.alloc_coalesce_tried);
  71         printf("    alloc_coalesce_iterate_clash = %llu\n",
  72                (unsigned long long)stats.stats.alloc_coalesce_iterate_clash);
  73         printf("    alloc_coalesce_lockfail = %llu\n",
  74                (unsigned long long)stats.stats.alloc_coalesce_lockfail);
  75         printf("    alloc_coalesce_race = %llu\n",
  76                (unsigned long long)stats.stats.alloc_coalesce_race);
  77         printf("    alloc_coalesce_succeeded = %llu\n",
  78                (unsigned long long)stats.stats.alloc_coalesce_succeeded);
  79         printf("      alloc_coalesce_num_merged = %llu\n",
  80                (unsigned long long)stats.stats.alloc_coalesce_num_merged);
  81         printf("compares = %llu\n",
  82                (unsigned long long)stats.stats.compares);
  83         printf("  compare_wrong_offsetbits = %llu\n",
  84                (unsigned long long)stats.stats.compare_wrong_offsetbits);
  85         printf("  compare_wrong_keylen = %llu\n",
  86                (unsigned long long)stats.stats.compare_wrong_keylen);
  87         printf("  compare_wrong_rechash = %llu\n",
  88                (unsigned long long)stats.stats.compare_wrong_rechash);
  89         printf("  compare_wrong_keycmp = %llu\n",
  90                (unsigned long long)stats.stats.compare_wrong_keycmp);
  91         printf("transactions = %llu\n",
  92                (unsigned long long)stats.stats.transactions);
  93         printf("  transaction_cancel = %llu\n",
  94                (unsigned long long)stats.stats.transaction_cancel);
  95         printf("  transaction_nest = %llu\n",
  96                (unsigned long long)stats.stats.transaction_nest);
  97         printf("  transaction_expand_file = %llu\n",
  98                (unsigned long long)stats.stats.transaction_expand_file);
  99         printf("  transaction_read_direct = %llu\n",
 100                (unsigned long long)stats.stats.transaction_read_direct);
 101         printf("    transaction_read_direct_fail = %llu\n",
 102                (unsigned long long)stats.stats.transaction_read_direct_fail);
 103         printf("  transaction_write_direct = %llu\n",
 104                (unsigned long long)stats.stats.transaction_write_direct);
 105         printf("    transaction_write_direct_fail = %llu\n",
 106                (unsigned long long)stats.stats.transaction_write_direct_fail);
 107         printf("expands = %llu\n",
 108                (unsigned long long)stats.stats.expands);
 109         printf("frees = %llu\n",
 110                (unsigned long long)stats.stats.frees);
 111         printf("locks = %llu\n",
 112                (unsigned long long)stats.stats.locks);
 113         printf("  lock_lowlevel = %llu\n",
 114                (unsigned long long)stats.stats.lock_lowlevel);
 115         printf("  lock_nonblock = %llu\n",
 116                (unsigned long long)stats.stats.lock_nonblock);
 117         printf("    lock_nonblock_fail = %llu\n",
 118                (unsigned long long)stats.stats.lock_nonblock_fail);
 119
 120         /* Now clear. */
 121         ntdb_close(*ntdb);
 122         *ntdb = ntdb_open("/tmp/speed.ntdb", flags, O_RDWR, 0, attr);
 123 }
 124
 125 static void ntdb_log(struct ntdb_context *ntdb,
 126                     enum ntdb_log_level level,
 127                     enum NTDB_ERROR ecode,
 128                     const char *message,
 129                     void *data)
 130 {
 131         fprintf(stderr, "ntdb:%s:%s:%s\n",
 132                 ntdb_name(ntdb), ntdb_errorstr(ecode), message);
 133 }
 134
 135 int main(int argc, char *argv[])
 136 {
 137         unsigned int i, j, num = 1000, stage = 0, stopat = -1;
 138         int flags = NTDB_DEFAULT;
 139         bool transaction = false, summary = false;
 140         NTDB_DATA key, data;
 141         struct ntdb_context *ntdb;
 142         struct timeval start, stop;
 143         union ntdb_attribute seed, log;
 144         bool do_stats = false;
 145         enum NTDB_ERROR ecode;
 146
 147         /* Try to keep benchmarks even. */
 148         seed.base.attr = NTDB_ATTRIBUTE_SEED;
 149         seed.base.next = NULL;
 150         seed.seed.seed = 0;
 151
 152         log.base.attr = NTDB_ATTRIBUTE_LOG;
 153         log.base.next = &seed;
 154         log.log.fn = ntdb_log;
 155
 156         if (argv[1] && strcmp(argv[1], "--internal") == 0) {
 157                 flags = NTDB_INTERNAL;
 158                 argc--;
 159                 argv++;
 160         }
 161         if (argv[1] && strcmp(argv[1], "--transaction") == 0) {
 162                 transaction = true;
 163                 argc--;
 164                 argv++;
 165         }
 166         if (argv[1] && strcmp(argv[1], "--no-sync") == 0) {
 167                 flags |= NTDB_NOSYNC;
 168                 argc--;
 169                 argv++;
 170         }
 171         if (argv[1] && strcmp(argv[1], "--summary") == 0) {
 172                 summary = true;
 173                 argc--;
 174                 argv++;
 175         }
 176         if (argv[1] && strcmp(argv[1], "--stats") == 0) {
 177                 do_stats = true;
 178                 argc--;
 179                 argv++;
 180         }
 181
 182         ntdb = ntdb_open("/tmp/speed.ntdb", flags, O_RDWR|O_CREAT|O_TRUNC,
 183                        0600, &log);
 184         if (!ntdb)
 185                 err(1, "Opening /tmp/speed.ntdb");
 186
 187         key.dptr = (void *)&i;
 188         key.dsize = sizeof(i);
 189         data = key;
 190
 191         if (argv[1]) {
 192                 num = atoi(argv[1]);
 193                 argv++;
 194                 argc--;
 195         }
 196
 197         if (argv[1]) {
 198                 stopat = atoi(argv[1]);
 199                 argv++;
 200                 argc--;
 201         }
 202
 203         /* Add 1000 records. */
 204         printf("Adding %u records: ", num); fflush(stdout);
 205         if (transaction && (ecode = ntdb_transaction_start(ntdb)))
 206                 errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
 207         gettimeofday(&start, NULL);
 208         for (i = 0; i < num; i++)
 209                 if ((ecode = ntdb_store(ntdb, key, data, NTDB_INSERT)) != 0)
 210                         errx(1, "Inserting key %u in ntdb: %s",
 211                              i, ntdb_errorstr(ecode));
 212         gettimeofday(&stop, NULL);
 213         if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
 214                 errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
 215         printf(" %zu ns (%zu bytes)\n",
 216                normalize(&start, &stop, num), file_size());
 217
 218         if (ntdb_check(ntdb, NULL, NULL))
 219                 errx(1, "ntdb_check failed!");
 220         if (summary) {
 221                 char *sumstr = NULL;
 222                 ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
 223                 printf("%s\n", sumstr);
 224                 free(sumstr);
 225         }
 226         if (do_stats)
 227                 dump_and_clear_stats(&ntdb, flags, &log);
 228
 229         if (++stage == stopat)
 230                 exit(0);
 231
 232         /* Finding 1000 records. */
 233         printf("Finding %u records: ", num); fflush(stdout);
 234         if (transaction && (ecode = ntdb_transaction_start(ntdb)))
 235                 errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
 236         gettimeofday(&start, NULL);
 237         for (i = 0; i < num; i++) {
 238                 NTDB_DATA dbuf;
 239                 if ((ecode = ntdb_fetch(ntdb, key, &dbuf)) != NTDB_SUCCESS
 240                     || *(int *)dbuf.dptr != i) {
 241                         errx(1, "Fetching key %u in ntdb gave %u",
 242                              i, ecode ? ecode : *(int *)dbuf.dptr);
 243                 }
 244         }
 245         gettimeofday(&stop, NULL);
 246         if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
 247                 errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
 248         printf(" %zu ns (%zu bytes)\n",
 249                normalize(&start, &stop, num), file_size());
 250         if (ntdb_check(ntdb, NULL, NULL))
 251                 errx(1, "ntdb_check failed!");
 252         if (summary) {
 253                 char *sumstr = NULL;
 254                 ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
 255                 printf("%s\n", sumstr);
 256                 free(sumstr);
 257         }
 258         if (do_stats)
 259                 dump_and_clear_stats(&ntdb, flags, &log);
 260         if (++stage == stopat)
 261                 exit(0);
 262
 263         /* Missing 1000 records. */
 264         printf("Missing %u records: ", num); fflush(stdout);
 265         if (transaction && (ecode = ntdb_transaction_start(ntdb)))
 266                 errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
 267         gettimeofday(&start, NULL);
 268         for (i = num; i < num*2; i++) {
 269                 NTDB_DATA dbuf;
 270                 ecode = ntdb_fetch(ntdb, key, &dbuf);
 271                 if (ecode != NTDB_ERR_NOEXIST)
 272                         errx(1, "Fetching key %u in ntdb gave %s",
 273                              i, ntdb_errorstr(ecode));
 274         }
 275         gettimeofday(&stop, NULL);
 276         if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
 277                 errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
 278         printf(" %zu ns (%zu bytes)\n",
 279                normalize(&start, &stop, num), file_size());
 280         if (ntdb_check(ntdb, NULL, NULL))
 281                 errx(1, "ntdb_check failed!");
 282         if (summary) {
 283                 char *sumstr = NULL;
 284                 ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
 285                 printf("%s\n", sumstr);
 286                 free(sumstr);
 287         }
 288         if (do_stats)
 289                 dump_and_clear_stats(&ntdb, flags, &log);
 290         if (++stage == stopat)
 291                 exit(0);
 292
 293         /* Traverse 1000 records. */
 294         printf("Traversing %u records: ", num); fflush(stdout);
 295         if (transaction && (ecode = ntdb_transaction_start(ntdb)))
 296                 errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
 297         i = 0;
 298         gettimeofday(&start, NULL);
 299         if (ntdb_traverse(ntdb, count_record, &i) != num)
 300                 errx(1, "Traverse returned wrong number of records");
 301         if (i != (num - 1) * (num / 2))
 302                 errx(1, "Traverse tallied to %u", i);
 303         gettimeofday(&stop, NULL);
 304         if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
 305                 errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
 306         printf(" %zu ns (%zu bytes)\n",
 307                normalize(&start, &stop, num), file_size());
 308         if (ntdb_check(ntdb, NULL, NULL))
 309                 errx(1, "ntdb_check failed!");
 310         if (summary) {
 311                 char *sumstr = NULL;
 312                 ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
 313                 printf("%s\n", sumstr);
 314                 free(sumstr);
 315         }
 316         if (do_stats)
 317                 dump_and_clear_stats(&ntdb, flags, &log);
 318         if (++stage == stopat)
 319                 exit(0);
 320
 321         /* Delete 1000 records (not in order). */
 322         printf("Deleting %u records: ", num); fflush(stdout);
 323         if (transaction && (ecode = ntdb_transaction_start(ntdb)))
 324                 errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
 325         gettimeofday(&start, NULL);
 326         for (j = 0; j < num; j++) {
 327                 i = (j + 100003) % num;
 328                 if ((ecode = ntdb_delete(ntdb, key)) != NTDB_SUCCESS)
 329                         errx(1, "Deleting key %u in ntdb: %s",
 330                              i, ntdb_errorstr(ecode));
 331         }
 332         gettimeofday(&stop, NULL);
 333         if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
 334                 errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
 335         printf(" %zu ns (%zu bytes)\n",
 336                normalize(&start, &stop, num), file_size());
 337         if (ntdb_check(ntdb, NULL, NULL))
 338                 errx(1, "ntdb_check failed!");
 339         if (summary) {
 340                 char *sumstr = NULL;
 341                 ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
 342                 printf("%s\n", sumstr);
 343                 free(sumstr);
 344         }
 345         if (do_stats)
 346                 dump_and_clear_stats(&ntdb, flags, &log);
 347         if (++stage == stopat)
 348                 exit(0);
 349
 350         /* Re-add 1000 records (not in order). */
 351         printf("Re-adding %u records: ", num); fflush(stdout);
 352         if (transaction && (ecode = ntdb_transaction_start(ntdb)))
 353                 errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
 354         gettimeofday(&start, NULL);
 355         for (j = 0; j < num; j++) {
 356                 i = (j + 100003) % num;
 357                 if ((ecode = ntdb_store(ntdb, key, data, NTDB_INSERT)) != 0)
 358                         errx(1, "Inserting key %u in ntdb: %s",
 359                              i, ntdb_errorstr(ecode));
 360         }
 361         gettimeofday(&stop, NULL);
 362         if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
 363                 errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
 364         printf(" %zu ns (%zu bytes)\n",
 365                normalize(&start, &stop, num), file_size());
 366         if (ntdb_check(ntdb, NULL, NULL))
 367                 errx(1, "ntdb_check failed!");
 368         if (summary) {
 369                 char *sumstr = NULL;
 370                 ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
 371                 printf("%s\n", sumstr);
 372                 free(sumstr);
 373         }
 374         if (do_stats)
 375                 dump_and_clear_stats(&ntdb, flags, &log);
 376         if (++stage == stopat)
 377                 exit(0);
 378
 379         /* Append 1000 records. */
 380         if (transaction && (ecode = ntdb_transaction_start(ntdb)))
 381                 errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
 382         printf("Appending %u records: ", num); fflush(stdout);
 383         gettimeofday(&start, NULL);
 384         for (i = 0; i < num; i++)
 385                 if ((ecode = ntdb_append(ntdb, key, data)) != NTDB_SUCCESS)
 386                         errx(1, "Appending key %u in ntdb: %s",
 387                              i, ntdb_errorstr(ecode));
 388         gettimeofday(&stop, NULL);
 389         if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
 390                 errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
 391         printf(" %zu ns (%zu bytes)\n",
 392                normalize(&start, &stop, num), file_size());
 393         if (ntdb_check(ntdb, NULL, NULL))
 394                 errx(1, "ntdb_check failed!");
 395         if (summary) {
 396                 char *sumstr = NULL;
 397                 ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
 398                 printf("%s\n", sumstr);
 399                 free(sumstr);
 400         }
 401         if (++stage == stopat)
 402                 exit(0);
 403
 404         /* Churn 1000 records: not in order! */
 405         if (transaction && (ecode = ntdb_transaction_start(ntdb)))
 406                 errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
 407         printf("Churning %u records: ", num); fflush(stdout);
 408         gettimeofday(&start, NULL);
 409         for (j = 0; j < num; j++) {
 410                 i = (j + 1000019) % num;
 411                 if ((ecode = ntdb_delete(ntdb, key)) != NTDB_SUCCESS)
 412                         errx(1, "Deleting key %u in ntdb: %s",
 413                              i, ntdb_errorstr(ecode));
 414                 i += num;
 415                 if ((ecode = ntdb_store(ntdb, key, data, NTDB_INSERT)) != 0)
 416                         errx(1, "Inserting key %u in ntdb: %s",
 417                              i, ntdb_errorstr(ecode));
 418         }
 419         gettimeofday(&stop, NULL);
 420         if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
 421                 errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
 422         printf(" %zu ns (%zu bytes)\n",
 423                normalize(&start, &stop, num), file_size());
 424
 425         if (ntdb_check(ntdb, NULL, NULL))
 426                 errx(1, "ntdb_check failed!");
 427         if (summary) {
 428                 char *sumstr = NULL;
 429                 ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
 430                 printf("%s\n", sumstr);
 431                 free(sumstr);
 432         }
 433         if (do_stats)
 434                 dump_and_clear_stats(&ntdb, flags, &log);
 435         if (++stage == stopat)
 436                 exit(0);
 437
 438         return 0;
 439 }