git.ozlabs.org Git - ccan/blob - ccan/tally/tally.c

   1 #include "config.h"
   2 #include <ccan/tally/tally.h>
   3 #include <ccan/build_assert/build_assert.h>
   4 #include <ccan/likely/likely.h>
   5 #include <stdint.h>
   6 #include <limits.h>
   7 #include <string.h>
   8 #include <stdio.h>
   9 #include <assert.h>
  10
  11 #define MAX_STEP_BITS (sizeof(size_t)*CHAR_BIT)
  12
  13 /* We use power of 2 steps.  I tried being tricky, but it got buggy. */
  14 struct tally {
  15         ssize_t min, max;
  16         size_t total[2];
  17         /* This allows limited frequency analysis. */
  18         size_t buckets;
  19         size_t step_bits;
  20         size_t counts[1 /* [buckets] */ ];
  21 };
  22
  23 struct tally *tally_new(size_t buckets)
  24 {
  25         struct tally *tally;
  26
  27         /* Check for overflow. */
  28         if (buckets && SIZE_MAX / buckets < sizeof(tally->counts[0]))
  29                 return NULL;
  30         tally = malloc(sizeof(*tally) + sizeof(tally->counts[0])*buckets);
  31         if (tally) {
  32                 /* SSIZE_MAX isn't portable, so make it one of these types. */
  33                 BUILD_ASSERT(sizeof(tally->min) == sizeof(int)
  34                              || sizeof(tally->min) == sizeof(long)
  35                              || sizeof(tally->min) == sizeof(long long));
  36                 if (sizeof(tally->min) == sizeof(int)) {
  37                         tally->min = INT_MAX;
  38                         tally->max = INT_MIN;
  39                 } else if (sizeof(tally->min) == sizeof(long)) {
  40                         tally->min = LONG_MAX;
  41                         tally->max = LONG_MIN;
  42                 } else if (sizeof(tally->min) == sizeof(long long)) {
  43                         tally->min = (ssize_t)LLONG_MAX;
  44                         tally->max = (ssize_t)LLONG_MIN;
  45                 }
  46                 tally->total[0] = tally->total[1] = 0;
  47                 /* There is always 1 bucket. */
  48                 tally->buckets = buckets+1;
  49                 tally->step_bits = 0;
  50                 memset(tally->counts, 0, sizeof(tally->counts[0])*(buckets+1));
  51         }
  52         return tally;
  53 }
  54
  55 static unsigned bucket_of(ssize_t min, unsigned step_bits, ssize_t val)
  56 {
  57         /* Don't over-shift. */
  58         if (step_bits == MAX_STEP_BITS)
  59                 return 0;
  60         assert(step_bits < MAX_STEP_BITS);
  61         return (size_t)(val - min) >> step_bits;
  62 }
  63
  64 /* Return the min value in bucket b. */
  65 static ssize_t bucket_min(ssize_t min, unsigned step_bits, unsigned b)
  66 {
  67         /* Don't over-shift. */
  68         if (step_bits == MAX_STEP_BITS)
  69                 return min;
  70         assert(step_bits < MAX_STEP_BITS);
  71         return min + ((ssize_t)b << step_bits);
  72 }
  73
  74 /* Does shifting by this many bits truncate the number? */
  75 static bool shift_overflows(size_t num, unsigned bits)
  76 {
  77         if (bits == 0)
  78                 return false;
  79
  80         return ((num << bits) >> 1) != (num << (bits - 1));
  81 }
  82
  83 /* When min or max change, we may need to shuffle the frequency counts. */
  84 static void renormalize(struct tally *tally,
  85                         ssize_t new_min, ssize_t new_max)
  86 {
  87         size_t range, spill;
  88         unsigned int i, old_min;
  89
  90         /* Uninitialized?  Don't do anything... */
  91         if (tally->max < tally->min)
  92                 goto update;
  93
  94         /* If we don't have sufficient range, increase step bits until
  95          * buckets cover entire range of ssize_t anyway. */
  96         range = (new_max - new_min) + 1;
  97         while (!shift_overflows(tally->buckets, tally->step_bits)
  98                && range > ((size_t)tally->buckets << tally->step_bits)) {
  99                 /* Collapse down. */
 100                 for (i = 1; i < tally->buckets; i++) {
 101                         tally->counts[i/2] += tally->counts[i];
 102                         tally->counts[i] = 0;
 103                 }
 104                 tally->step_bits++;
 105         }
 106
 107         /* Now if minimum has dropped, move buckets up. */
 108         old_min = bucket_of(new_min, tally->step_bits, tally->min);
 109         memmove(tally->counts + old_min,
 110                 tally->counts,
 111                 sizeof(tally->counts[0]) * (tally->buckets - old_min));
 112         memset(tally->counts, 0, sizeof(tally->counts[0]) * old_min);
 113
 114         /* If we moved boundaries, adjust buckets to that ratio. */
 115         spill = (tally->min - new_min) % (1 << tally->step_bits);
 116         for (i = 0; i < tally->buckets-1; i++) {
 117                 size_t adjust = (tally->counts[i] >> tally->step_bits) * spill;
 118                 tally->counts[i] -= adjust;
 119                 tally->counts[i+1] += adjust;
 120         }
 121
 122 update:
 123         tally->min = new_min;
 124         tally->max = new_max;
 125 }
 126
 127 void tally_add(struct tally *tally, ssize_t val)
 128 {
 129         ssize_t new_min = tally->min, new_max = tally->max;
 130         bool need_renormalize = false;
 131
 132         if (val < tally->min) {
 133                 new_min = val;
 134                 need_renormalize = true;
 135         }
 136         if (val > tally->max) {
 137                 new_max = val;
 138                 need_renormalize = true;
 139         }
 140         if (need_renormalize)
 141                 renormalize(tally, new_min, new_max);
 142
 143         /* 128-bit arithmetic!  If we didn't want exact mean, we could just
 144          * pull it out of counts. */
 145         if (val > 0 && tally->total[0] + val < tally->total[0])
 146                 tally->total[1]++;
 147         else if (val < 0 && tally->total[0] + val > tally->total[0])
 148                 tally->total[1]--;
 149         tally->total[0] += val;
 150         tally->counts[bucket_of(tally->min, tally->step_bits, val)]++;
 151 }
 152
 153 size_t tally_num(const struct tally *tally)
 154 {
 155         size_t i, num = 0;
 156         for (i = 0; i < tally->buckets; i++)
 157                 num += tally->counts[i];
 158         return num;
 159 }
 160
 161 ssize_t tally_min(const struct tally *tally)
 162 {
 163         return tally->min;
 164 }
 165
 166 ssize_t tally_max(const struct tally *tally)
 167 {
 168         return tally->max;
 169 }
 170
 171 /* FIXME: Own ccan module please! */
 172 static unsigned fls64(uint64_t val)
 173 {
 174 #if HAVE_BUILTIN_CLZL
 175         if (val <= ULONG_MAX) {
 176                 /* This is significantly faster! */
 177                 return val ? sizeof(long) * CHAR_BIT - __builtin_clzl(val) : 0;
 178         } else {
 179 #endif
 180         uint64_t r = 64;
 181
 182         if (!val)
 183                 return 0;
 184         if (!(val & 0xffffffff00000000ull)) {
 185                 val <<= 32;
 186                 r -= 32;
 187         }
 188         if (!(val & 0xffff000000000000ull)) {
 189                 val <<= 16;
 190                 r -= 16;
 191         }
 192         if (!(val & 0xff00000000000000ull)) {
 193                 val <<= 8;
 194                 r -= 8;
 195         }
 196         if (!(val & 0xf000000000000000ull)) {
 197                 val <<= 4;
 198                 r -= 4;
 199         }
 200         if (!(val & 0xc000000000000000ull)) {
 201                 val <<= 2;
 202                 r -= 2;
 203         }
 204         if (!(val & 0x8000000000000000ull)) {
 205                 val <<= 1;
 206                 r -= 1;
 207         }
 208         return r;
 209 #if HAVE_BUILTIN_CLZL
 210         }
 211 #endif
 212 }
 213
 214 /* This is stolen straight from Hacker's Delight. */
 215 static uint64_t divlu64(uint64_t u1, uint64_t u0, uint64_t v)
 216 {
 217         const uint64_t b = 4294967296ULL; // Number base (32 bits).
 218         uint32_t un[4],           // Dividend and divisor
 219                 vn[2];            // normalized and broken
 220                                   // up into halfwords.
 221         uint32_t q[2];            // Quotient as halfwords.
 222         uint64_t un1, un0,        // Dividend and divisor
 223                 vn0;              // as fullwords.
 224         uint64_t qhat;            // Estimated quotient digit.
 225         uint64_t rhat;            // A remainder.
 226         uint64_t p;               // Product of two digits.
 227         int64_t s, i, j, t, k;
 228
 229         if (u1 >= v)              // If overflow, return the largest
 230                 return (uint64_t)-1; // possible quotient.
 231
 232         s = 64 - fls64(v);                // 0 <= s <= 63.
 233         vn0 = v << s;             // Normalize divisor.
 234         vn[1] = vn0 >> 32;        // Break divisor up into
 235         vn[0] = vn0 & 0xFFFFFFFF; // two 32-bit halves.
 236
 237         // Shift dividend left.
 238         un1 = ((u1 << s) | (u0 >> (64 - s))) & (-s >> 63);
 239         un0 = u0 << s;
 240         un[3] = un1 >> 32;        // Break dividend up into
 241         un[2] = un1;              // four 32-bit halfwords
 242         un[1] = un0 >> 32;        // Note: storing into
 243         un[0] = un0;              // halfwords truncates.
 244
 245         for (j = 1; j >= 0; j--) {
 246                 // Compute estimate qhat of q[j].
 247                 qhat = (un[j+2]*b + un[j+1])/vn[1];
 248                 rhat = (un[j+2]*b + un[j+1]) - qhat*vn[1];
 249         again:
 250                 if (qhat >= b || qhat*vn[0] > b*rhat + un[j]) {
 251                         qhat = qhat - 1;
 252                         rhat = rhat + vn[1];
 253                         if (rhat < b) goto again;
 254                 }
 255
 256                 // Multiply and subtract.
 257                 k = 0;
 258                 for (i = 0; i < 2; i++) {
 259                         p = qhat*vn[i];
 260                         t = un[i+j] - k - (p & 0xFFFFFFFF);
 261                         un[i+j] = t;
 262                         k = (p >> 32) - (t >> 32);
 263                 }
 264                 t = un[j+2] - k;
 265                 un[j+2] = t;
 266
 267                 q[j] = qhat;              // Store quotient digit.
 268                 if (t < 0) {              // If we subtracted too
 269                         q[j] = q[j] - 1;  // much, add back.
 270                         k = 0;
 271                         for (i = 0; i < 2; i++) {
 272                                 t = un[i+j] + vn[i] + k;
 273                                 un[i+j] = t;
 274                                 k = t >> 32;
 275                         }
 276                         un[j+2] = un[j+2] + k;
 277                 }
 278         } // End j.
 279
 280         return q[1]*b + q[0];
 281 }
 282
 283 static int64_t divls64(int64_t u1, uint64_t u0, int64_t v)
 284 {
 285         int64_t q, uneg, vneg, diff, borrow;
 286
 287         uneg = u1 >> 63;          // -1 if u < 0.
 288         if (uneg) {               // Compute the absolute
 289                 u0 = -u0;         // value of the dividend u.
 290                 borrow = (u0 != 0);
 291                 u1 = -u1 - borrow;
 292         }
 293
 294         vneg = v >> 63;           // -1 if v < 0.
 295         v = (v ^ vneg) - vneg;    // Absolute value of v.
 296
 297         if ((uint64_t)u1 >= (uint64_t)v)
 298                 goto overflow;
 299
 300         q = divlu64(u1, u0, v);
 301
 302         diff = uneg ^ vneg;       // Negate q if signs of
 303         q = (q ^ diff) - diff;    // u and v differed.
 304
 305         if ((diff ^ q) < 0 && q != 0) {    // If overflow, return the largest
 306         overflow:                          // possible neg. quotient.
 307                 q = 0x8000000000000000ULL;
 308         }
 309         return q;
 310 }
 311
 312 ssize_t tally_mean(const struct tally *tally)
 313 {
 314         size_t count = tally_num(tally);
 315         if (!count)
 316                 return 0;
 317
 318         if (sizeof(tally->total[0]) == sizeof(uint32_t)) {
 319                 /* Use standard 64-bit arithmetic. */
 320                 int64_t total = tally->total[0]
 321                         | (((uint64_t)tally->total[1]) << 32);
 322                 return total / count;
 323         }
 324         return divls64(tally->total[1], tally->total[0], count);
 325 }
 326
 327 static ssize_t bucket_range(const struct tally *tally, unsigned b, size_t *err)
 328 {
 329         ssize_t min, max;
 330
 331         min = bucket_min(tally->min, tally->step_bits, b);
 332         if (b == tally->buckets - 1)
 333                 max = tally->max;
 334         else
 335                 max = bucket_min(tally->min, tally->step_bits, b+1) - 1;
 336
 337         /* FIXME: Think harder about cumulative error; is this enough?. */
 338         *err = (max - min + 1) / 2;
 339         /* Avoid overflow. */
 340         return min + (max - min) / 2;
 341 }
 342
 343 ssize_t tally_approx_median(const struct tally *tally, size_t *err)
 344 {
 345         size_t count = tally_num(tally), total = 0;
 346         unsigned int i;
 347
 348         for (i = 0; i < tally->buckets; i++) {
 349                 total += tally->counts[i];
 350                 if (total * 2 >= count)
 351                         break;
 352         }
 353         return bucket_range(tally, i, err);
 354 }
 355
 356 ssize_t tally_approx_mode(const struct tally *tally, size_t *err)
 357 {
 358         unsigned int i, min_best = 0, max_best = 0;
 359
 360         for (i = 0; i < tally->buckets; i++) {
 361                 if (tally->counts[i] > tally->counts[min_best]) {
 362                         min_best = max_best = i;
 363                 } else if (tally->counts[i] == tally->counts[min_best]) {
 364                         max_best = i;
 365                 }
 366         }
 367
 368         /* We can have more than one best, making our error huge. */
 369         if (min_best != max_best) {
 370                 ssize_t min, max;
 371                 min = bucket_range(tally, min_best, err);
 372                 max = bucket_range(tally, max_best, err);
 373                 max += *err;
 374                 *err += (size_t)(max - min);
 375                 return min + (max - min) / 2;
 376         }
 377
 378         return bucket_range(tally, min_best, err);
 379 }
 380
 381 static unsigned get_max_bucket(const struct tally *tally)
 382 {
 383         unsigned int i;
 384
 385         for (i = tally->buckets; i > 0; i--)
 386                 if (tally->counts[i-1])
 387                         break;
 388         return i;
 389 }
 390
 391 char *tally_histogram(const struct tally *tally,
 392                       unsigned width, unsigned height)
 393 {
 394         unsigned int i, count, max_bucket, largest_bucket;
 395         struct tally *tmp;
 396         char *graph, *p;
 397
 398         assert(width >= TALLY_MIN_HISTO_WIDTH);
 399         assert(height >= TALLY_MIN_HISTO_HEIGHT);
 400
 401         /* Ignore unused buckets. */
 402         max_bucket = get_max_bucket(tally);
 403
 404         /* FIXME: It'd be nice to smooth here... */
 405         if (height >= max_bucket) {
 406                 height = max_bucket;
 407                 tmp = NULL;
 408         } else {
 409                 /* We create a temporary then renormalize so < height. */
 410                 /* FIXME: Antialias properly! */
 411                 tmp = tally_new(tally->buckets-1);
 412                 if (!tmp)
 413                         return NULL;
 414                 tmp->min = tally->min;
 415                 tmp->max = tally->max;
 416                 tmp->step_bits = tally->step_bits;
 417                 memcpy(tmp->counts, tally->counts,
 418                        sizeof(tally->counts[0]) * tmp->buckets);
 419                 while ((max_bucket = get_max_bucket(tmp)) >= height)
 420                         renormalize(tmp, tmp->min, tmp->max *= 2);
 421                 /* Restore max */
 422                 tmp->max = tally->max;
 423                 tally = tmp;
 424                 height = max_bucket;
 425         }
 426
 427         /* Figure out longest line, for scale. */
 428         largest_bucket = 0;
 429         for (i = 0; i < tally->buckets; i++) {
 430                 if (tally->counts[i] > largest_bucket)
 431                         largest_bucket = tally->counts[i];
 432         }
 433
 434         p = graph = malloc(height * (width + 1) + 1);
 435         if (!graph) {
 436                 free(tmp);
 437                 return NULL;
 438         }
 439         for (i = 0; i < height; i++) {
 440                 unsigned covered = 0;
 441                 count = (double)tally->counts[i] / largest_bucket * width;
 442
 443                 if (i == 0)
 444                         covered = snprintf(p, width, "%zi", tally->min);
 445                 else if (i == height - 1)
 446                         covered = snprintf(p, width, "%zi", tally->max);
 447                 if (covered) {
 448                         if (covered > width)
 449                                 covered = width;
 450                         p += covered;
 451                         if (count > covered)
 452                                 count -= covered;
 453                         else
 454                                 count = 0;
 455                 }
 456                 memset(p, '*', count);
 457                 p += count;
 458                 *p = '\n';
 459                 p++;
 460         }
 461         *p = '\0';
 462         free(tmp);
 463         return graph;
 464 }