git.ozlabs.org Git - ccan/blob - ccan/tally/tally.c

   1 /* Licensed under LGPLv3+ - see LICENSE file for details */
   2 #include <ccan/tally/tally.h>
   3 #include <ccan/build_assert/build_assert.h>
   4 #include <ccan/likely/likely.h>
   5 #include <stdint.h>
   6 #include <limits.h>
   7 #include <string.h>
   8 #include <stdio.h>
   9 #include <assert.h>
  10 #include <stdlib.h>
  11
  12 #define SIZET_BITS (sizeof(size_t)*CHAR_BIT)
  13
  14 /* We use power of 2 steps.  I tried being tricky, but it got buggy. */
  15 struct tally {
  16         ssize_t min, max;
  17         size_t total[2];
  18         /* This allows limited frequency analysis. */
  19         unsigned buckets, step_bits;
  20         size_t counts[1 /* Actually: [buckets] */ ];
  21 };
  22
  23 struct tally *tally_new(unsigned buckets)
  24 {
  25         struct tally *tally;
  26
  27         /* There is always 1 bucket. */
  28         if (buckets == 0) {
  29                 buckets = 1;
  30         }
  31
  32         /* Overly cautious check for overflow. */
  33         if (sizeof(*tally) * buckets / sizeof(*tally) != buckets) {
  34                 return NULL;
  35         }
  36
  37         tally = (struct tally *)malloc(
  38                 sizeof(*tally) + sizeof(tally->counts[0])*(buckets-1));
  39         if (tally == NULL) {
  40                 return NULL;
  41         }
  42
  43         tally->max = ((size_t)1 << (SIZET_BITS - 1));
  44         tally->min = ~tally->max;
  45         tally->total[0] = tally->total[1] = 0;
  46         tally->buckets = buckets;
  47         tally->step_bits = 0;
  48         memset(tally->counts, 0, sizeof(tally->counts[0])*buckets);
  49         return tally;
  50 }
  51
  52 static unsigned bucket_of(ssize_t min, unsigned step_bits, ssize_t val)
  53 {
  54         /* Don't over-shift. */
  55         if (step_bits == SIZET_BITS) {
  56                 return 0;
  57         }
  58         assert(step_bits < SIZET_BITS);
  59         return ((size_t)val - (size_t)min) >> step_bits;
  60 }
  61
  62 /* Return the min value in bucket b. */
  63 static ssize_t bucket_min(ssize_t min, unsigned step_bits, unsigned b)
  64 {
  65         /* Don't over-shift. */
  66         if (step_bits == SIZET_BITS) {
  67                 return min;
  68         }
  69         assert(step_bits < SIZET_BITS);
  70         return min + ((size_t)b << step_bits);
  71 }
  72
  73 /* Does shifting by this many bits truncate the number? */
  74 static bool shift_overflows(size_t num, unsigned bits)
  75 {
  76         if (bits == 0) {
  77                 return false;
  78         }
  79         if (bits >= SIZET_BITS) {
  80                 return true;
  81         }
  82
  83         return ((num << bits) >> 1) != (num << (bits - 1));
  84 }
  85
  86 /* When min or max change, we may need to shuffle the frequency counts. */
  87 static void renormalize(struct tally *tally,
  88                         ssize_t new_min, ssize_t new_max)
  89 {
  90         size_t range, spill;
  91         unsigned int i, old_min;
  92
  93         /* Uninitialized?  Don't do anything... */
  94         if (tally->max < tally->min) {
  95                 goto update;
  96         }
  97
  98         /* If we don't have sufficient range, increase step bits until
  99          * buckets cover entire range of ssize_t anyway. */
 100         range = ((size_t)new_max - (size_t)new_min) + 1;
 101         while (!shift_overflows(tally->buckets, tally->step_bits)
 102                && range > ((size_t)tally->buckets << tally->step_bits)) {
 103                 /* Collapse down. */
 104                 for (i = 1; i < tally->buckets; i++) {
 105                         tally->counts[i/2] += tally->counts[i];
 106                         tally->counts[i] = 0;
 107                 }
 108                 tally->step_bits++;
 109         }
 110
 111         /* Now if minimum has dropped, move buckets up. */
 112         old_min = bucket_of(new_min, tally->step_bits, tally->min);
 113         memmove(tally->counts + old_min,
 114                 tally->counts,
 115                 sizeof(tally->counts[0]) * (tally->buckets - old_min));
 116         memset(tally->counts, 0, sizeof(tally->counts[0]) * old_min);
 117
 118         /* If we moved boundaries, adjust buckets to that ratio. */
 119         if (tally->step_bits < SIZET_BITS) {
 120                 spill = (tally->min - new_min) % ((size_t)1 << tally->step_bits);
 121                 for (i = 0; i < tally->buckets-1; i++) {
 122                         size_t adjust = (tally->counts[i] >> tally->step_bits) * spill;
 123                         tally->counts[i] -= adjust;
 124                         tally->counts[i+1] += adjust;
 125                 }
 126         }
 127
 128 update:
 129         tally->min = new_min;
 130         tally->max = new_max;
 131 }
 132
 133 void tally_add(struct tally *tally, ssize_t val)
 134 {
 135         ssize_t new_min = tally->min, new_max = tally->max;
 136         bool need_renormalize = false;
 137
 138         if (val < tally->min) {
 139                 new_min = val;
 140                 need_renormalize = true;
 141         }
 142         if (val > tally->max) {
 143                 new_max = val;
 144                 need_renormalize = true;
 145         }
 146         if (need_renormalize) {
 147                 renormalize(tally, new_min, new_max);
 148         }
 149
 150         /* 128-bit arithmetic!  If we didn't want exact mean, we could just
 151          * pull it out of counts. */
 152         if (val > 0 && tally->total[0] + val < tally->total[0]) {
 153                 tally->total[1]++;
 154         } else if (val < 0 && tally->total[0] + val > tally->total[0]) {
 155                 tally->total[1]--;
 156         }
 157         tally->total[0] += val;
 158         tally->counts[bucket_of(tally->min, tally->step_bits, val)]++;
 159 }
 160
 161 size_t tally_num(const struct tally *tally)
 162 {
 163         size_t i, num = 0;
 164         for (i = 0; i < tally->buckets; i++) {
 165                 num += tally->counts[i];
 166         }
 167         return num;
 168 }
 169
 170 ssize_t tally_min(const struct tally *tally)
 171 {
 172         return tally->min;
 173 }
 174
 175 ssize_t tally_max(const struct tally *tally)
 176 {
 177         return tally->max;
 178 }
 179
 180 /* FIXME: Own ccan module please! */
 181 static unsigned fls64(uint64_t val)
 182 {
 183 #if HAVE_BUILTIN_CLZL
 184         if (val <= ULONG_MAX) {
 185                 /* This is significantly faster! */
 186                 return val ? sizeof(long) * CHAR_BIT - __builtin_clzl(val) : 0;
 187         } else {
 188 #endif
 189         uint64_t r = 64;
 190
 191         if (!val) {
 192                 return 0;
 193         }
 194         if (!(val & 0xffffffff00000000ull)) {
 195                 val <<= 32;
 196                 r -= 32;
 197         }
 198         if (!(val & 0xffff000000000000ull)) {
 199                 val <<= 16;
 200                 r -= 16;
 201         }
 202         if (!(val & 0xff00000000000000ull)) {
 203                 val <<= 8;
 204                 r -= 8;
 205         }
 206         if (!(val & 0xf000000000000000ull)) {
 207                 val <<= 4;
 208                 r -= 4;
 209         }
 210         if (!(val & 0xc000000000000000ull)) {
 211                 val <<= 2;
 212                 r -= 2;
 213         }
 214         if (!(val & 0x8000000000000000ull)) {
 215                 val <<= 1;
 216                 r -= 1;
 217         }
 218         return r;
 219 #if HAVE_BUILTIN_CLZL
 220         }
 221 #endif
 222 }
 223
 224 /* This is stolen straight from Hacker's Delight. */
 225 static uint64_t divlu64(uint64_t u1, uint64_t u0, uint64_t v)
 226 {
 227         const uint64_t b = 4294967296ULL; /* Number base (32 bits). */
 228         uint32_t un[4],           /* Dividend and divisor */
 229                 vn[2];            /* normalized and broken */
 230                                   /* up into halfwords. */
 231         uint32_t q[2];            /* Quotient as halfwords. */
 232         uint64_t un1, un0,        /* Dividend and divisor */
 233                 vn0;              /* as fullwords. */
 234         uint64_t qhat;            /* Estimated quotient digit. */
 235         uint64_t rhat;            /* A remainder. */
 236         uint64_t p;               /* Product of two digits. */
 237         int64_t s, i, j, t, k;
 238
 239         if (u1 >= v) {            /* If overflow, return the largest */
 240                 return (uint64_t)-1; /* possible quotient. */
 241         }
 242
 243         s = 64 - fls64(v);                /* 0 <= s <= 63. */
 244         vn0 = v << s;             /* Normalize divisor. */
 245         vn[1] = vn0 >> 32;        /* Break divisor up into */
 246         vn[0] = vn0 & 0xFFFFFFFF; /* two 32-bit halves. */
 247
 248         // Shift dividend left.
 249         un1 = ((u1 << s) | (u0 >> (64 - s))) & (-s >> 63);
 250         un0 = u0 << s;
 251         un[3] = un1 >> 32;        /* Break dividend up into */
 252         un[2] = un1;              /* four 32-bit halfwords */
 253         un[1] = un0 >> 32;        /* Note: storing into */
 254         un[0] = un0;              /* halfwords truncates. */
 255
 256         for (j = 1; j >= 0; j--) {
 257                 /* Compute estimate qhat of q[j]. */
 258                 qhat = (un[j+2]*b + un[j+1])/vn[1];
 259                 rhat = (un[j+2]*b + un[j+1]) - qhat*vn[1];
 260         again:
 261                 if (qhat >= b || qhat*vn[0] > b*rhat + un[j]) {
 262                         qhat = qhat - 1;
 263                         rhat = rhat + vn[1];
 264                         if (rhat < b) {
 265                                 goto again;
 266                         }
 267                 }
 268
 269                 /* Multiply and subtract. */
 270                 k = 0;
 271                 for (i = 0; i < 2; i++) {
 272                         p = qhat*vn[i];
 273                         t = un[i+j] - k - (p & 0xFFFFFFFF);
 274                         un[i+j] = t;
 275                         k = (p >> 32) - (t >> 32);
 276                 }
 277                 t = un[j+2] - k;
 278                 un[j+2] = t;
 279
 280                 q[j] = qhat;              /* Store quotient digit. */
 281                 if (t < 0) {              /* If we subtracted too */
 282                         q[j] = q[j] - 1;  /* much, add back. */
 283                         k = 0;
 284                         for (i = 0; i < 2; i++) {
 285                                 t = un[i+j] + vn[i] + k;
 286                                 un[i+j] = t;
 287                                 k = t >> 32;
 288                         }
 289                         un[j+2] = un[j+2] + k;
 290                 }
 291         } /* End j. */
 292
 293         return q[1]*b + q[0];
 294 }
 295
 296 static int64_t divls64(int64_t u1, uint64_t u0, int64_t v)
 297 {
 298         int64_t q, uneg, vneg, diff, borrow;
 299
 300         uneg = u1 >> 63;          /* -1 if u < 0. */
 301         if (uneg) {               /* Compute the absolute */
 302                 u0 = -u0;         /* value of the dividend u. */
 303                 borrow = (u0 != 0);
 304                 u1 = -u1 - borrow;
 305         }
 306
 307         vneg = v >> 63;           /* -1 if v < 0. */
 308         v = (v ^ vneg) - vneg;    /* Absolute value of v. */
 309
 310         if ((uint64_t)u1 >= (uint64_t)v) {
 311                 goto overflow;
 312         }
 313
 314         q = divlu64(u1, u0, v);
 315
 316         diff = uneg ^ vneg;       /* Negate q if signs of */
 317         q = (q ^ diff) - diff;    /* u and v differed. */
 318
 319         if ((diff ^ q) < 0 && q != 0) {    /* If overflow, return the
 320                                               largest */
 321         overflow:                          /* possible neg. quotient. */
 322                 q = 0x8000000000000000ULL;
 323         }
 324         return q;
 325 }
 326
 327 ssize_t tally_mean(const struct tally *tally)
 328 {
 329         size_t count = tally_num(tally);
 330         if (!count) {
 331                 return 0;
 332         }
 333
 334         if (sizeof(tally->total[0]) == sizeof(uint32_t)) {
 335                 /* Use standard 64-bit arithmetic. */
 336                 int64_t total = tally->total[0]
 337                         | (((uint64_t)tally->total[1]) << 32);
 338                 return total / count;
 339         }
 340         return divls64(tally->total[1], tally->total[0], count);
 341 }
 342
 343 ssize_t tally_total(const struct tally *tally, ssize_t *overflow)
 344 {
 345         if (overflow) {
 346                 *overflow = tally->total[1];
 347                 return tally->total[0];
 348         }
 349
 350         /* If result is negative, make sure we can represent it. */
 351         if (tally->total[1] & ((size_t)1 << (SIZET_BITS-1))) {
 352                 /* Must have only underflowed once, and must be able to
 353                  * represent result at ssize_t. */
 354                 if ((~tally->total[1])+1 != 0
 355                     || (ssize_t)tally->total[0] >= 0) {
 356                         /* Underflow, return minimum. */
 357                         return (ssize_t)((size_t)1 << (SIZET_BITS - 1));
 358                 }
 359         } else {
 360                 /* Result is positive, must not have overflowed, and must be
 361                  * able to represent as ssize_t. */
 362                 if (tally->total[1] || (ssize_t)tally->total[0] < 0) {
 363                         /* Overflow.  Return maximum. */
 364                         return (ssize_t)~((size_t)1 << (SIZET_BITS - 1));
 365                 }
 366         }
 367         return tally->total[0];
 368 }
 369
 370 static ssize_t bucket_range(const struct tally *tally, unsigned b, size_t *err)
 371 {
 372         ssize_t min, max;
 373
 374         min = bucket_min(tally->min, tally->step_bits, b);
 375         if (b == tally->buckets - 1) {
 376                 max = tally->max;
 377         } else {
 378                 max = bucket_min(tally->min, tally->step_bits, b+1) - 1;
 379         }
 380
 381         /* FIXME: Think harder about cumulative error; is this enough?. */
 382         *err = (max - min + 1) / 2;
 383         /* Avoid overflow. */
 384         return min + (max - min) / 2;
 385 }
 386
 387 ssize_t tally_approx_median(const struct tally *tally, size_t *err)
 388 {
 389         size_t count = tally_num(tally), total = 0;
 390         unsigned int i;
 391
 392         for (i = 0; i < tally->buckets; i++) {
 393                 total += tally->counts[i];
 394                 if (total * 2 >= count) {
 395                         break;
 396                 }
 397         }
 398         return bucket_range(tally, i, err);
 399 }
 400
 401 ssize_t tally_approx_mode(const struct tally *tally, size_t *err)
 402 {
 403         unsigned int i, min_best = 0, max_best = 0;
 404
 405         for (i = 0; i < tally->buckets; i++) {
 406                 if (tally->counts[i] > tally->counts[min_best]) {
 407                         min_best = max_best = i;
 408                 } else if (tally->counts[i] == tally->counts[min_best]) {
 409                         max_best = i;
 410                 }
 411         }
 412
 413         /* We can have more than one best, making our error huge. */
 414         if (min_best != max_best) {
 415                 ssize_t min, max;
 416                 min = bucket_range(tally, min_best, err);
 417                 max = bucket_range(tally, max_best, err);
 418                 max += *err;
 419                 *err += (size_t)(max - min);
 420                 return min + (max - min) / 2;
 421         }
 422
 423         return bucket_range(tally, min_best, err);
 424 }
 425
 426 static unsigned get_max_bucket(const struct tally *tally)
 427 {
 428         unsigned int i;
 429
 430         for (i = tally->buckets; i > 0; i--) {
 431                 if (tally->counts[i-1]) {
 432                         break;
 433                 }
 434         }
 435         return i;
 436 }
 437
 438 char *tally_histogram(const struct tally *tally,
 439                       unsigned width, unsigned height)
 440 {
 441         unsigned int i, count, max_bucket, largest_bucket;
 442         struct tally *tmp;
 443         char *graph, *p;
 444
 445         assert(width >= TALLY_MIN_HISTO_WIDTH);
 446         assert(height >= TALLY_MIN_HISTO_HEIGHT);
 447
 448         /* Ignore unused buckets. */
 449         max_bucket = get_max_bucket(tally);
 450
 451         /* FIXME: It'd be nice to smooth here... */
 452         if (height >= max_bucket) {
 453                 height = max_bucket;
 454                 tmp = NULL;
 455         } else {
 456                 /* We create a temporary then renormalize so < height. */
 457                 /* FIXME: Antialias properly! */
 458                 tmp = tally_new(tally->buckets);
 459                 if (!tmp) {
 460                         return NULL;
 461                 }
 462                 tmp->min = tally->min;
 463                 tmp->max = tally->max;
 464                 tmp->step_bits = tally->step_bits;
 465                 memcpy(tmp->counts, tally->counts,
 466                        sizeof(tally->counts[0]) * tmp->buckets);
 467                 while ((max_bucket = get_max_bucket(tmp)) >= height) {
 468                         renormalize(tmp, tmp->min, tmp->max * 2);
 469                 }
 470                 /* Restore max */
 471                 tmp->max = tally->max;
 472                 tally = tmp;
 473                 height = max_bucket;
 474         }
 475
 476         /* Figure out longest line, for scale. */
 477         largest_bucket = 0;
 478         for (i = 0; i < tally->buckets; i++) {
 479                 if (tally->counts[i] > largest_bucket) {
 480                         largest_bucket = tally->counts[i];
 481                 }
 482         }
 483
 484         p = graph = (char *)malloc(height * (width + 1) + 1);
 485         if (!graph) {
 486                 free(tmp);
 487                 return NULL;
 488         }
 489
 490         for (i = 0; i < height; i++) {
 491                 unsigned covered = 1, row;
 492
 493                 /* People expect minimum at the bottom. */
 494                 row = height - i - 1;
 495                 count = (double)tally->counts[row] / largest_bucket * (width-1)+1;
 496
 497                 if (row == 0) {
 498                         covered = snprintf(p, width, "%zi", tally->min);
 499                 } else if (row == height - 1) {
 500                         covered = snprintf(p, width, "%zi", tally->max);
 501                 } else if (row == bucket_of(tally->min, tally->step_bits, 0)) {
 502                         *p = '+';
 503                 } else {
 504                         *p = '|';
 505                 }
 506
 507                 if (covered > width) {
 508                         covered = width;
 509                 }
 510                 p += covered;
 511
 512                 if (count > covered) {
 513                         count -= covered;
 514                         memset(p, '*', count);
 515                 } else {
 516                         count = 0;
 517                 }
 518
 519                 p += count;
 520                 *p = '\n';
 521                 p++;
 522         }
 523         *p = '\0';
 524         free(tmp);
 525         return graph;
 526 }