git.ozlabs.org Git - ccan/blob - ccan/tally/tally.c

   1 /* Licensed under LGPLv3+ - see LICENSE file for details */
   2 #include <ccan/tally/tally.h>
   3 #include <ccan/build_assert/build_assert.h>
   4 #include <ccan/likely/likely.h>
   5 #include <stdint.h>
   6 #include <limits.h>
   7 #include <string.h>
   8 #include <stdio.h>
   9 #include <assert.h>
  10 #include <stdlib.h>
  11
  12 #define SIZET_BITS (sizeof(size_t)*CHAR_BIT)
  13
  14 /* We use power of 2 steps.  I tried being tricky, but it got buggy. */
  15 struct tally {
  16         ssize_t min, max;
  17         size_t total[2];
  18         /* This allows limited frequency analysis. */
  19         unsigned buckets, step_bits;
  20         size_t counts[1 /* Actually: [buckets] */ ];
  21 };
  22
  23 struct tally *tally_new(unsigned buckets)
  24 {
  25         struct tally *tally;
  26
  27         /* There is always 1 bucket. */
  28         if (buckets == 0) {
  29                 buckets = 1;
  30         }
  31
  32         /* Overly cautious check for overflow. */
  33         if (sizeof(*tally) * buckets / sizeof(*tally) != buckets) {
  34                 return NULL;
  35         }
  36
  37         tally = (struct tally *)malloc(
  38                 sizeof(*tally) + sizeof(tally->counts[0])*(buckets-1));
  39         if (tally == NULL) {
  40                 return NULL;
  41         }
  42
  43         tally->max = ((size_t)1 << (SIZET_BITS - 1));
  44         tally->min = ~tally->max;
  45         tally->total[0] = tally->total[1] = 0;
  46         tally->buckets = buckets;
  47         tally->step_bits = 0;
  48         memset(tally->counts, 0, sizeof(tally->counts[0])*buckets);
  49         return tally;
  50 }
  51
  52 static unsigned bucket_of(ssize_t min, unsigned step_bits, ssize_t val)
  53 {
  54         /* Don't over-shift. */
  55         if (step_bits == SIZET_BITS) {
  56                 return 0;
  57         }
  58         assert(step_bits < SIZET_BITS);
  59         return (size_t)(val - min) >> step_bits;
  60 }
  61
  62 /* Return the min value in bucket b. */
  63 static ssize_t bucket_min(ssize_t min, unsigned step_bits, unsigned b)
  64 {
  65         /* Don't over-shift. */
  66         if (step_bits == SIZET_BITS)
  67                 return min;
  68         assert(step_bits < SIZET_BITS);
  69         return min + ((ssize_t)b << step_bits);
  70 }
  71
  72 /* Does shifting by this many bits truncate the number? */
  73 static bool shift_overflows(size_t num, unsigned bits)
  74 {
  75         if (bits == 0)
  76                 return false;
  77
  78         return ((num << bits) >> 1) != (num << (bits - 1));
  79 }
  80
  81 /* When min or max change, we may need to shuffle the frequency counts. */
  82 static void renormalize(struct tally *tally,
  83                         ssize_t new_min, ssize_t new_max)
  84 {
  85         size_t range, spill;
  86         unsigned int i, old_min;
  87
  88         /* Uninitialized?  Don't do anything... */
  89         if (tally->max < tally->min)
  90                 goto update;
  91
  92         /* If we don't have sufficient range, increase step bits until
  93          * buckets cover entire range of ssize_t anyway. */
  94         range = (new_max - new_min) + 1;
  95         while (!shift_overflows(tally->buckets, tally->step_bits)
  96                && range > ((size_t)tally->buckets << tally->step_bits)) {
  97                 /* Collapse down. */
  98                 for (i = 1; i < tally->buckets; i++) {
  99                         tally->counts[i/2] += tally->counts[i];
 100                         tally->counts[i] = 0;
 101                 }
 102                 tally->step_bits++;
 103         }
 104
 105         /* Now if minimum has dropped, move buckets up. */
 106         old_min = bucket_of(new_min, tally->step_bits, tally->min);
 107         memmove(tally->counts + old_min,
 108                 tally->counts,
 109                 sizeof(tally->counts[0]) * (tally->buckets - old_min));
 110         memset(tally->counts, 0, sizeof(tally->counts[0]) * old_min);
 111
 112         /* If we moved boundaries, adjust buckets to that ratio. */
 113         spill = (tally->min - new_min) % (1 << tally->step_bits);
 114         for (i = 0; i < tally->buckets-1; i++) {
 115                 size_t adjust = (tally->counts[i] >> tally->step_bits) * spill;
 116                 tally->counts[i] -= adjust;
 117                 tally->counts[i+1] += adjust;
 118         }
 119
 120 update:
 121         tally->min = new_min;
 122         tally->max = new_max;
 123 }
 124
 125 void tally_add(struct tally *tally, ssize_t val)
 126 {
 127         ssize_t new_min = tally->min, new_max = tally->max;
 128         bool need_renormalize = false;
 129
 130         if (val < tally->min) {
 131                 new_min = val;
 132                 need_renormalize = true;
 133         }
 134         if (val > tally->max) {
 135                 new_max = val;
 136                 need_renormalize = true;
 137         }
 138         if (need_renormalize)
 139                 renormalize(tally, new_min, new_max);
 140
 141         /* 128-bit arithmetic!  If we didn't want exact mean, we could just
 142          * pull it out of counts. */
 143         if (val > 0 && tally->total[0] + val < tally->total[0])
 144                 tally->total[1]++;
 145         else if (val < 0 && tally->total[0] + val > tally->total[0])
 146                 tally->total[1]--;
 147         tally->total[0] += val;
 148         tally->counts[bucket_of(tally->min, tally->step_bits, val)]++;
 149 }
 150
 151 size_t tally_num(const struct tally *tally)
 152 {
 153         size_t i, num = 0;
 154         for (i = 0; i < tally->buckets; i++)
 155                 num += tally->counts[i];
 156         return num;
 157 }
 158
 159 ssize_t tally_min(const struct tally *tally)
 160 {
 161         return tally->min;
 162 }
 163
 164 ssize_t tally_max(const struct tally *tally)
 165 {
 166         return tally->max;
 167 }
 168
 169 /* FIXME: Own ccan module please! */
 170 static unsigned fls64(uint64_t val)
 171 {
 172 #if HAVE_BUILTIN_CLZL
 173         if (val <= ULONG_MAX) {
 174                 /* This is significantly faster! */
 175                 return val ? sizeof(long) * CHAR_BIT - __builtin_clzl(val) : 0;
 176         } else {
 177 #endif
 178         uint64_t r = 64;
 179
 180         if (!val)
 181                 return 0;
 182         if (!(val & 0xffffffff00000000ull)) {
 183                 val <<= 32;
 184                 r -= 32;
 185         }
 186         if (!(val & 0xffff000000000000ull)) {
 187                 val <<= 16;
 188                 r -= 16;
 189         }
 190         if (!(val & 0xff00000000000000ull)) {
 191                 val <<= 8;
 192                 r -= 8;
 193         }
 194         if (!(val & 0xf000000000000000ull)) {
 195                 val <<= 4;
 196                 r -= 4;
 197         }
 198         if (!(val & 0xc000000000000000ull)) {
 199                 val <<= 2;
 200                 r -= 2;
 201         }
 202         if (!(val & 0x8000000000000000ull)) {
 203                 val <<= 1;
 204                 r -= 1;
 205         }
 206         return r;
 207 #if HAVE_BUILTIN_CLZL
 208         }
 209 #endif
 210 }
 211
 212 /* This is stolen straight from Hacker's Delight. */
 213 static uint64_t divlu64(uint64_t u1, uint64_t u0, uint64_t v)
 214 {
 215         const uint64_t b = 4294967296ULL; // Number base (32 bits).
 216         uint32_t un[4],           // Dividend and divisor
 217                 vn[2];            // normalized and broken
 218                                   // up into halfwords.
 219         uint32_t q[2];            // Quotient as halfwords.
 220         uint64_t un1, un0,        // Dividend and divisor
 221                 vn0;              // as fullwords.
 222         uint64_t qhat;            // Estimated quotient digit.
 223         uint64_t rhat;            // A remainder.
 224         uint64_t p;               // Product of two digits.
 225         int64_t s, i, j, t, k;
 226
 227         if (u1 >= v)              // If overflow, return the largest
 228                 return (uint64_t)-1; // possible quotient.
 229
 230         s = 64 - fls64(v);                // 0 <= s <= 63.
 231         vn0 = v << s;             // Normalize divisor.
 232         vn[1] = vn0 >> 32;        // Break divisor up into
 233         vn[0] = vn0 & 0xFFFFFFFF; // two 32-bit halves.
 234
 235         // Shift dividend left.
 236         un1 = ((u1 << s) | (u0 >> (64 - s))) & (-s >> 63);
 237         un0 = u0 << s;
 238         un[3] = un1 >> 32;        // Break dividend up into
 239         un[2] = un1;              // four 32-bit halfwords
 240         un[1] = un0 >> 32;        // Note: storing into
 241         un[0] = un0;              // halfwords truncates.
 242
 243         for (j = 1; j >= 0; j--) {
 244                 // Compute estimate qhat of q[j].
 245                 qhat = (un[j+2]*b + un[j+1])/vn[1];
 246                 rhat = (un[j+2]*b + un[j+1]) - qhat*vn[1];
 247         again:
 248                 if (qhat >= b || qhat*vn[0] > b*rhat + un[j]) {
 249                         qhat = qhat - 1;
 250                         rhat = rhat + vn[1];
 251                         if (rhat < b) goto again;
 252                 }
 253
 254                 // Multiply and subtract.
 255                 k = 0;
 256                 for (i = 0; i < 2; i++) {
 257                         p = qhat*vn[i];
 258                         t = un[i+j] - k - (p & 0xFFFFFFFF);
 259                         un[i+j] = t;
 260                         k = (p >> 32) - (t >> 32);
 261                 }
 262                 t = un[j+2] - k;
 263                 un[j+2] = t;
 264
 265                 q[j] = qhat;              // Store quotient digit.
 266                 if (t < 0) {              // If we subtracted too
 267                         q[j] = q[j] - 1;  // much, add back.
 268                         k = 0;
 269                         for (i = 0; i < 2; i++) {
 270                                 t = un[i+j] + vn[i] + k;
 271                                 un[i+j] = t;
 272                                 k = t >> 32;
 273                         }
 274                         un[j+2] = un[j+2] + k;
 275                 }
 276         } // End j.
 277
 278         return q[1]*b + q[0];
 279 }
 280
 281 static int64_t divls64(int64_t u1, uint64_t u0, int64_t v)
 282 {
 283         int64_t q, uneg, vneg, diff, borrow;
 284
 285         uneg = u1 >> 63;          // -1 if u < 0.
 286         if (uneg) {               // Compute the absolute
 287                 u0 = -u0;         // value of the dividend u.
 288                 borrow = (u0 != 0);
 289                 u1 = -u1 - borrow;
 290         }
 291
 292         vneg = v >> 63;           // -1 if v < 0.
 293         v = (v ^ vneg) - vneg;    // Absolute value of v.
 294
 295         if ((uint64_t)u1 >= (uint64_t)v)
 296                 goto overflow;
 297
 298         q = divlu64(u1, u0, v);
 299
 300         diff = uneg ^ vneg;       // Negate q if signs of
 301         q = (q ^ diff) - diff;    // u and v differed.
 302
 303         if ((diff ^ q) < 0 && q != 0) {    // If overflow, return the largest
 304         overflow:                          // possible neg. quotient.
 305                 q = 0x8000000000000000ULL;
 306         }
 307         return q;
 308 }
 309
 310 ssize_t tally_mean(const struct tally *tally)
 311 {
 312         size_t count = tally_num(tally);
 313         if (!count)
 314                 return 0;
 315
 316         if (sizeof(tally->total[0]) == sizeof(uint32_t)) {
 317                 /* Use standard 64-bit arithmetic. */
 318                 int64_t total = tally->total[0]
 319                         | (((uint64_t)tally->total[1]) << 32);
 320                 return total / count;
 321         }
 322         return divls64(tally->total[1], tally->total[0], count);
 323 }
 324
 325 ssize_t tally_total(const struct tally *tally, ssize_t *overflow)
 326 {
 327         if (overflow) {
 328                 *overflow = tally->total[1];
 329                 return tally->total[0];
 330         }
 331
 332         /* If result is negative, make sure we can represent it. */
 333         if (tally->total[1] & ((size_t)1 << (SIZET_BITS-1))) {
 334                 /* Must have only underflowed once, and must be able to
 335                  * represent result at ssize_t. */
 336                 if ((~tally->total[1])+1 != 0
 337                     || (ssize_t)tally->total[0] >= 0) {
 338                         /* Underflow, return minimum. */
 339                         return (ssize_t)((size_t)1 << (SIZET_BITS - 1));
 340                 }
 341         } else {
 342                 /* Result is positive, must not have overflowed, and must be
 343                  * able to represent as ssize_t. */
 344                 if (tally->total[1] || (ssize_t)tally->total[0] < 0) {
 345                         /* Overflow.  Return maximum. */
 346                         return (ssize_t)~((size_t)1 << (SIZET_BITS - 1));
 347                 }
 348         }
 349         return tally->total[0];
 350 }
 351
 352 static ssize_t bucket_range(const struct tally *tally, unsigned b, size_t *err)
 353 {
 354         ssize_t min, max;
 355
 356         min = bucket_min(tally->min, tally->step_bits, b);
 357         if (b == tally->buckets - 1)
 358                 max = tally->max;
 359         else
 360                 max = bucket_min(tally->min, tally->step_bits, b+1) - 1;
 361
 362         /* FIXME: Think harder about cumulative error; is this enough?. */
 363         *err = (max - min + 1) / 2;
 364         /* Avoid overflow. */
 365         return min + (max - min) / 2;
 366 }
 367
 368 ssize_t tally_approx_median(const struct tally *tally, size_t *err)
 369 {
 370         size_t count = tally_num(tally), total = 0;
 371         unsigned int i;
 372
 373         for (i = 0; i < tally->buckets; i++) {
 374                 total += tally->counts[i];
 375                 if (total * 2 >= count)
 376                         break;
 377         }
 378         return bucket_range(tally, i, err);
 379 }
 380
 381 ssize_t tally_approx_mode(const struct tally *tally, size_t *err)
 382 {
 383         unsigned int i, min_best = 0, max_best = 0;
 384
 385         for (i = 0; i < tally->buckets; i++) {
 386                 if (tally->counts[i] > tally->counts[min_best]) {
 387                         min_best = max_best = i;
 388                 } else if (tally->counts[i] == tally->counts[min_best]) {
 389                         max_best = i;
 390                 }
 391         }
 392
 393         /* We can have more than one best, making our error huge. */
 394         if (min_best != max_best) {
 395                 ssize_t min, max;
 396                 min = bucket_range(tally, min_best, err);
 397                 max = bucket_range(tally, max_best, err);
 398                 max += *err;
 399                 *err += (size_t)(max - min);
 400                 return min + (max - min) / 2;
 401         }
 402
 403         return bucket_range(tally, min_best, err);
 404 }
 405
 406 static unsigned get_max_bucket(const struct tally *tally)
 407 {
 408         unsigned int i;
 409
 410         for (i = tally->buckets; i > 0; i--)
 411                 if (tally->counts[i-1])
 412                         break;
 413         return i;
 414 }
 415
 416 char *tally_histogram(const struct tally *tally,
 417                       unsigned width, unsigned height)
 418 {
 419         unsigned int i, count, max_bucket, largest_bucket;
 420         struct tally *tmp;
 421         char *graph, *p;
 422
 423         assert(width >= TALLY_MIN_HISTO_WIDTH);
 424         assert(height >= TALLY_MIN_HISTO_HEIGHT);
 425
 426         /* Ignore unused buckets. */
 427         max_bucket = get_max_bucket(tally);
 428
 429         /* FIXME: It'd be nice to smooth here... */
 430         if (height >= max_bucket) {
 431                 height = max_bucket;
 432                 tmp = NULL;
 433         } else {
 434                 /* We create a temporary then renormalize so < height. */
 435                 /* FIXME: Antialias properly! */
 436                 tmp = tally_new(tally->buckets);
 437                 if (!tmp)
 438                         return NULL;
 439                 tmp->min = tally->min;
 440                 tmp->max = tally->max;
 441                 tmp->step_bits = tally->step_bits;
 442                 memcpy(tmp->counts, tally->counts,
 443                        sizeof(tally->counts[0]) * tmp->buckets);
 444                 while ((max_bucket = get_max_bucket(tmp)) >= height)
 445                         renormalize(tmp, tmp->min, tmp->max * 2);
 446                 /* Restore max */
 447                 tmp->max = tally->max;
 448                 tally = tmp;
 449                 height = max_bucket;
 450         }
 451
 452         /* Figure out longest line, for scale. */
 453         largest_bucket = 0;
 454         for (i = 0; i < tally->buckets; i++) {
 455                 if (tally->counts[i] > largest_bucket)
 456                         largest_bucket = tally->counts[i];
 457         }
 458
 459         p = graph = (char *)malloc(height * (width + 1) + 1);
 460         if (!graph) {
 461                 free(tmp);
 462                 return NULL;
 463         }
 464
 465         for (i = 0; i < height; i++) {
 466                 unsigned covered = 1, row;
 467
 468                 /* People expect minimum at the bottom. */
 469                 row = height - i - 1;
 470                 count = (double)tally->counts[row] / largest_bucket * (width-1)+1;
 471
 472                 if (row == 0)
 473                         covered = snprintf(p, width, "%zi", tally->min);
 474                 else if (row == height - 1)
 475                         covered = snprintf(p, width, "%zi", tally->max);
 476                 else if (row == bucket_of(tally->min, tally->step_bits, 0))
 477                         *p = '+';
 478                 else
 479                         *p = '|';
 480
 481                 if (covered > width)
 482                         covered = width;
 483                 p += covered;
 484
 485                 if (count > covered)
 486                         count -= covered;
 487                 else
 488                         count = 0;
 489
 490                 memset(p, '*', count);
 491                 p += count;
 492                 *p = '\n';
 493                 p++;
 494         }
 495         *p = '\0';
 496         free(tmp);
 497         return graph;
 498 }