tally: implement tally_total()
[ccan] / ccan / tally / tally.c
1 #include "config.h"
2 #include <ccan/tally/tally.h>
3 #include <ccan/build_assert/build_assert.h>
4 #include <ccan/likely/likely.h>
5 #include <stdint.h>
6 #include <limits.h>
7 #include <string.h>
8 #include <stdio.h>
9 #include <assert.h>
10
11 #define SIZET_BITS (sizeof(size_t)*CHAR_BIT)
12
13 /* We use power of 2 steps.  I tried being tricky, but it got buggy. */
14 struct tally {
15         ssize_t min, max;
16         size_t total[2];
17         /* This allows limited frequency analysis. */
18         size_t buckets;
19         size_t step_bits;
20         size_t counts[1 /* [buckets] */ ];
21 };
22
23 struct tally *tally_new(size_t buckets)
24 {
25         struct tally *tally;
26
27         /* Check for overflow. */
28         if (buckets && SIZE_MAX / buckets < sizeof(tally->counts[0]))
29                 return NULL;
30         tally = malloc(sizeof(*tally) + sizeof(tally->counts[0])*buckets);
31         if (tally) {
32                 /* SSIZE_MAX isn't portable, so make it one of these types. */
33                 BUILD_ASSERT(sizeof(tally->min) == sizeof(int)
34                              || sizeof(tally->min) == sizeof(long)
35                              || sizeof(tally->min) == sizeof(long long));
36                 if (sizeof(tally->min) == sizeof(int)) {
37                         tally->min = INT_MAX;
38                         tally->max = INT_MIN;
39                 } else if (sizeof(tally->min) == sizeof(long)) {
40                         tally->min = LONG_MAX;
41                         tally->max = LONG_MIN;
42                 } else if (sizeof(tally->min) == sizeof(long long)) {
43                         tally->min = (ssize_t)LLONG_MAX;
44                         tally->max = (ssize_t)LLONG_MIN;
45                 }
46                 tally->total[0] = tally->total[1] = 0;
47                 /* There is always 1 bucket. */
48                 tally->buckets = buckets+1;
49                 tally->step_bits = 0;
50                 memset(tally->counts, 0, sizeof(tally->counts[0])*(buckets+1));
51         }
52         return tally;
53 }
54
55 static unsigned bucket_of(ssize_t min, unsigned step_bits, ssize_t val)
56 {
57         /* Don't over-shift. */
58         if (step_bits == SIZET_BITS)
59                 return 0;
60         assert(step_bits < SIZET_BITS);
61         return (size_t)(val - min) >> step_bits;
62 }
63
64 /* Return the min value in bucket b. */
65 static ssize_t bucket_min(ssize_t min, unsigned step_bits, unsigned b)
66 {
67         /* Don't over-shift. */
68         if (step_bits == SIZET_BITS)
69                 return min;
70         assert(step_bits < SIZET_BITS);
71         return min + ((ssize_t)b << step_bits);
72 }
73
74 /* Does shifting by this many bits truncate the number? */
75 static bool shift_overflows(size_t num, unsigned bits)
76 {
77         if (bits == 0)
78                 return false;
79
80         return ((num << bits) >> 1) != (num << (bits - 1));
81 }
82
83 /* When min or max change, we may need to shuffle the frequency counts. */
84 static void renormalize(struct tally *tally,
85                         ssize_t new_min, ssize_t new_max)
86 {
87         size_t range, spill;
88         unsigned int i, old_min;
89
90         /* Uninitialized?  Don't do anything... */
91         if (tally->max < tally->min)
92                 goto update;
93
94         /* If we don't have sufficient range, increase step bits until
95          * buckets cover entire range of ssize_t anyway. */
96         range = (new_max - new_min) + 1;
97         while (!shift_overflows(tally->buckets, tally->step_bits)
98                && range > ((size_t)tally->buckets << tally->step_bits)) {
99                 /* Collapse down. */
100                 for (i = 1; i < tally->buckets; i++) {
101                         tally->counts[i/2] += tally->counts[i];
102                         tally->counts[i] = 0;
103                 }
104                 tally->step_bits++;
105         }
106
107         /* Now if minimum has dropped, move buckets up. */
108         old_min = bucket_of(new_min, tally->step_bits, tally->min);
109         memmove(tally->counts + old_min,
110                 tally->counts,
111                 sizeof(tally->counts[0]) * (tally->buckets - old_min));
112         memset(tally->counts, 0, sizeof(tally->counts[0]) * old_min);
113
114         /* If we moved boundaries, adjust buckets to that ratio. */
115         spill = (tally->min - new_min) % (1 << tally->step_bits);
116         for (i = 0; i < tally->buckets-1; i++) {
117                 size_t adjust = (tally->counts[i] >> tally->step_bits) * spill;
118                 tally->counts[i] -= adjust;
119                 tally->counts[i+1] += adjust;
120         }
121
122 update:
123         tally->min = new_min;
124         tally->max = new_max;
125 }
126
127 void tally_add(struct tally *tally, ssize_t val)
128 {
129         ssize_t new_min = tally->min, new_max = tally->max;
130         bool need_renormalize = false;
131
132         if (val < tally->min) {
133                 new_min = val;
134                 need_renormalize = true;
135         }
136         if (val > tally->max) {
137                 new_max = val;
138                 need_renormalize = true;
139         }
140         if (need_renormalize)
141                 renormalize(tally, new_min, new_max);
142
143         /* 128-bit arithmetic!  If we didn't want exact mean, we could just
144          * pull it out of counts. */
145         if (val > 0 && tally->total[0] + val < tally->total[0])
146                 tally->total[1]++;
147         else if (val < 0 && tally->total[0] + val > tally->total[0])
148                 tally->total[1]--;
149         tally->total[0] += val;
150         tally->counts[bucket_of(tally->min, tally->step_bits, val)]++;
151 }
152
153 size_t tally_num(const struct tally *tally)
154 {
155         size_t i, num = 0;
156         for (i = 0; i < tally->buckets; i++)
157                 num += tally->counts[i];
158         return num;
159 }
160
161 ssize_t tally_min(const struct tally *tally)
162 {
163         return tally->min;
164 }
165
166 ssize_t tally_max(const struct tally *tally)
167 {
168         return tally->max;
169 }
170
171 /* FIXME: Own ccan module please! */
172 static unsigned fls64(uint64_t val)
173 {
174 #if HAVE_BUILTIN_CLZL
175         if (val <= ULONG_MAX) {
176                 /* This is significantly faster! */
177                 return val ? sizeof(long) * CHAR_BIT - __builtin_clzl(val) : 0;
178         } else {
179 #endif
180         uint64_t r = 64;
181
182         if (!val)
183                 return 0;
184         if (!(val & 0xffffffff00000000ull)) {
185                 val <<= 32;
186                 r -= 32;
187         }
188         if (!(val & 0xffff000000000000ull)) {
189                 val <<= 16;
190                 r -= 16;
191         }
192         if (!(val & 0xff00000000000000ull)) {
193                 val <<= 8;
194                 r -= 8;
195         }
196         if (!(val & 0xf000000000000000ull)) {
197                 val <<= 4;
198                 r -= 4;
199         }
200         if (!(val & 0xc000000000000000ull)) {
201                 val <<= 2;
202                 r -= 2;
203         }
204         if (!(val & 0x8000000000000000ull)) {
205                 val <<= 1;
206                 r -= 1;
207         }
208         return r;
209 #if HAVE_BUILTIN_CLZL
210         }
211 #endif
212 }
213
214 /* This is stolen straight from Hacker's Delight. */
215 static uint64_t divlu64(uint64_t u1, uint64_t u0, uint64_t v)
216 {
217         const uint64_t b = 4294967296ULL; // Number base (32 bits).
218         uint32_t un[4],           // Dividend and divisor
219                 vn[2];            // normalized and broken
220                                   // up into halfwords.
221         uint32_t q[2];            // Quotient as halfwords.
222         uint64_t un1, un0,        // Dividend and divisor
223                 vn0;              // as fullwords.
224         uint64_t qhat;            // Estimated quotient digit.
225         uint64_t rhat;            // A remainder.
226         uint64_t p;               // Product of two digits.
227         int64_t s, i, j, t, k;
228
229         if (u1 >= v)              // If overflow, return the largest
230                 return (uint64_t)-1; // possible quotient.
231
232         s = 64 - fls64(v);                // 0 <= s <= 63.
233         vn0 = v << s;             // Normalize divisor.
234         vn[1] = vn0 >> 32;        // Break divisor up into
235         vn[0] = vn0 & 0xFFFFFFFF; // two 32-bit halves.
236
237         // Shift dividend left.
238         un1 = ((u1 << s) | (u0 >> (64 - s))) & (-s >> 63);
239         un0 = u0 << s;
240         un[3] = un1 >> 32;        // Break dividend up into
241         un[2] = un1;              // four 32-bit halfwords
242         un[1] = un0 >> 32;        // Note: storing into
243         un[0] = un0;              // halfwords truncates.
244
245         for (j = 1; j >= 0; j--) {
246                 // Compute estimate qhat of q[j].
247                 qhat = (un[j+2]*b + un[j+1])/vn[1];
248                 rhat = (un[j+2]*b + un[j+1]) - qhat*vn[1];
249         again:
250                 if (qhat >= b || qhat*vn[0] > b*rhat + un[j]) {
251                         qhat = qhat - 1;
252                         rhat = rhat + vn[1];
253                         if (rhat < b) goto again;
254                 }
255
256                 // Multiply and subtract.
257                 k = 0;
258                 for (i = 0; i < 2; i++) {
259                         p = qhat*vn[i];
260                         t = un[i+j] - k - (p & 0xFFFFFFFF);
261                         un[i+j] = t;
262                         k = (p >> 32) - (t >> 32);
263                 }
264                 t = un[j+2] - k;
265                 un[j+2] = t;
266
267                 q[j] = qhat;              // Store quotient digit.
268                 if (t < 0) {              // If we subtracted too
269                         q[j] = q[j] - 1;  // much, add back.
270                         k = 0;
271                         for (i = 0; i < 2; i++) {
272                                 t = un[i+j] + vn[i] + k;
273                                 un[i+j] = t;
274                                 k = t >> 32;
275                         }
276                         un[j+2] = un[j+2] + k;
277                 }
278         } // End j.
279
280         return q[1]*b + q[0];
281 }
282
283 static int64_t divls64(int64_t u1, uint64_t u0, int64_t v)
284 {
285         int64_t q, uneg, vneg, diff, borrow;
286
287         uneg = u1 >> 63;          // -1 if u < 0.
288         if (uneg) {               // Compute the absolute
289                 u0 = -u0;         // value of the dividend u.
290                 borrow = (u0 != 0);
291                 u1 = -u1 - borrow;
292         }
293
294         vneg = v >> 63;           // -1 if v < 0.
295         v = (v ^ vneg) - vneg;    // Absolute value of v.
296
297         if ((uint64_t)u1 >= (uint64_t)v)
298                 goto overflow;
299
300         q = divlu64(u1, u0, v);
301
302         diff = uneg ^ vneg;       // Negate q if signs of
303         q = (q ^ diff) - diff;    // u and v differed.
304
305         if ((diff ^ q) < 0 && q != 0) {    // If overflow, return the largest
306         overflow:                          // possible neg. quotient.
307                 q = 0x8000000000000000ULL;
308         }
309         return q;
310 }
311
312 ssize_t tally_mean(const struct tally *tally)
313 {
314         size_t count = tally_num(tally);
315         if (!count)
316                 return 0;
317
318         if (sizeof(tally->total[0]) == sizeof(uint32_t)) {
319                 /* Use standard 64-bit arithmetic. */
320                 int64_t total = tally->total[0]
321                         | (((uint64_t)tally->total[1]) << 32);
322                 return total / count;
323         }
324         return divls64(tally->total[1], tally->total[0], count);
325 }
326
327 ssize_t tally_total(const struct tally *tally, ssize_t *overflow)
328 {
329         if (overflow) {
330                 *overflow = tally->total[1];
331                 return tally->total[0];
332         }
333
334         /* If result is negative, make sure we can represent it. */
335         if (tally->total[1] & (1 << (SIZET_BITS-1))) {
336                 /* Must have only underflowed once, and must be able to
337                  * represent result at ssize_t. */
338                 if ((~tally->total[1])+1 != 0
339                     || (ssize_t)tally->total[0] >= 0) {
340                         /* Underflow, return minimum. */
341                         return (ssize_t)((size_t)1 << (SIZET_BITS - 1));
342                 }
343         } else {
344                 /* Result is positive, must not have overflowed, and must be
345                  * able to represent as ssize_t. */
346                 if (tally->total[1] || (ssize_t)tally->total[0] < 0) {
347                         /* Overflow.  Return maximum. */
348                         return (ssize_t)~((size_t)1 << (SIZET_BITS - 1));
349                 }
350         }
351         return tally->total[0];
352 }
353
354 static ssize_t bucket_range(const struct tally *tally, unsigned b, size_t *err)
355 {
356         ssize_t min, max;
357
358         min = bucket_min(tally->min, tally->step_bits, b);
359         if (b == tally->buckets - 1)
360                 max = tally->max;
361         else
362                 max = bucket_min(tally->min, tally->step_bits, b+1) - 1;
363
364         /* FIXME: Think harder about cumulative error; is this enough?. */
365         *err = (max - min + 1) / 2;
366         /* Avoid overflow. */
367         return min + (max - min) / 2;
368 }
369
370 ssize_t tally_approx_median(const struct tally *tally, size_t *err)
371 {
372         size_t count = tally_num(tally), total = 0;
373         unsigned int i;
374
375         for (i = 0; i < tally->buckets; i++) {
376                 total += tally->counts[i];
377                 if (total * 2 >= count)
378                         break;
379         }
380         return bucket_range(tally, i, err);
381 }
382
383 ssize_t tally_approx_mode(const struct tally *tally, size_t *err)
384 {
385         unsigned int i, min_best = 0, max_best = 0;
386
387         for (i = 0; i < tally->buckets; i++) {
388                 if (tally->counts[i] > tally->counts[min_best]) {
389                         min_best = max_best = i;
390                 } else if (tally->counts[i] == tally->counts[min_best]) {
391                         max_best = i;
392                 }
393         }
394
395         /* We can have more than one best, making our error huge. */
396         if (min_best != max_best) {
397                 ssize_t min, max;
398                 min = bucket_range(tally, min_best, err);
399                 max = bucket_range(tally, max_best, err);
400                 max += *err;
401                 *err += (size_t)(max - min);
402                 return min + (max - min) / 2;
403         }
404
405         return bucket_range(tally, min_best, err);
406 }
407
408 static unsigned get_max_bucket(const struct tally *tally)
409 {
410         unsigned int i;
411
412         for (i = tally->buckets; i > 0; i--)
413                 if (tally->counts[i-1])
414                         break;
415         return i;
416 }
417
418 char *tally_histogram(const struct tally *tally,
419                       unsigned width, unsigned height)
420 {
421         unsigned int i, count, max_bucket, largest_bucket;
422         struct tally *tmp;
423         char *graph, *p;
424
425         assert(width >= TALLY_MIN_HISTO_WIDTH);
426         assert(height >= TALLY_MIN_HISTO_HEIGHT);
427
428         /* Ignore unused buckets. */
429         max_bucket = get_max_bucket(tally);
430
431         /* FIXME: It'd be nice to smooth here... */
432         if (height >= max_bucket) {
433                 height = max_bucket;
434                 tmp = NULL;
435         } else {
436                 /* We create a temporary then renormalize so < height. */
437                 /* FIXME: Antialias properly! */
438                 tmp = tally_new(tally->buckets-1);
439                 if (!tmp)
440                         return NULL;
441                 tmp->min = tally->min;
442                 tmp->max = tally->max;
443                 tmp->step_bits = tally->step_bits;
444                 memcpy(tmp->counts, tally->counts,
445                        sizeof(tally->counts[0]) * tmp->buckets);
446                 while ((max_bucket = get_max_bucket(tmp)) >= height)
447                         renormalize(tmp, tmp->min, tmp->max *= 2);
448                 /* Restore max */
449                 tmp->max = tally->max;
450                 tally = tmp;
451                 height = max_bucket;
452         }
453
454         /* Figure out longest line, for scale. */
455         largest_bucket = 0;
456         for (i = 0; i < tally->buckets; i++) {
457                 if (tally->counts[i] > largest_bucket)
458                         largest_bucket = tally->counts[i];
459         }
460
461         p = graph = malloc(height * (width + 1) + 1);
462         if (!graph) {
463                 free(tmp);
464                 return NULL;
465         }
466         for (i = 0; i < height; i++) {
467                 unsigned covered = 0;
468                 count = (double)tally->counts[i] / largest_bucket * width;
469
470                 if (i == 0)
471                         covered = snprintf(p, width, "%zi", tally->min);
472                 else if (i == height - 1)
473                         covered = snprintf(p, width, "%zi", tally->max);
474                 if (covered) {
475                         if (covered > width)
476                                 covered = width;
477                         p += covered;
478                         if (count > covered)
479                                 count -= covered;
480                         else
481                                 count = 0;
482                 }
483                 memset(p, '*', count);
484                 p += count;
485                 *p = '\n';
486                 p++;
487         }
488         *p = '\0';
489         free(tmp);
490         return graph;
491 }