git.ozlabs.org Git - ccan/blob - ccan/timer/timer.c

   1 /* LGPL (v2.1 or any later version) - see LICENSE file for details */
   2 #include <ccan/timer/timer.h>
   3 #include <ccan/array_size/array_size.h>
   4 #include <ccan/ilog/ilog.h>
   5 #include <ccan/likely/likely.h>
   6 #include <stdlib.h>
   7 #include <stdio.h>
   8
   9 #define PER_LEVEL (1ULL << TIMER_LEVEL_BITS)
  10
  11 struct timer_level {
  12         struct list_head list[PER_LEVEL];
  13 };
  14
  15 static uint64_t time_to_grains(struct timespec ts)
  16 {
  17         return ts.tv_sec * ((uint64_t)1000000000 / TIMER_GRANULARITY)
  18                 + (ts.tv_nsec / TIMER_GRANULARITY);
  19 }
  20
  21 static struct timespec grains_to_time(uint64_t grains)
  22 {
  23         struct timespec ts;
  24
  25         ts.tv_sec = grains / (1000000000 / TIMER_GRANULARITY);
  26         ts.tv_nsec = (grains % (1000000000 / TIMER_GRANULARITY))
  27                 * TIMER_GRANULARITY;
  28         return ts;
  29 }
  30
  31 void timers_init(struct timers *timers, struct timespec start)
  32 {
  33         unsigned int i;
  34
  35         list_head_init(&timers->far);
  36         timers->base = time_to_grains(start);
  37         timers->first = -1ULL;
  38         for (i = 0; i < ARRAY_SIZE(timers->level); i++)
  39                 timers->level[i] = NULL;
  40 }
  41
  42 static unsigned int level_of(const struct timers *timers, uint64_t time)
  43 {
  44         uint64_t diff;
  45
  46         /* Level depends how far away it is. */
  47         diff = time - timers->base;
  48         return ilog64(diff / 2) / TIMER_LEVEL_BITS;
  49 }
  50
  51 static void timer_add_raw(struct timers *timers, struct timer *t)
  52 {
  53         struct list_head *l;
  54         unsigned int level = level_of(timers, t->time);
  55
  56         if (!timers->level[level])
  57                 l = &timers->far;
  58         else {
  59                 int off = (t->time >> (level*TIMER_LEVEL_BITS)) & (PER_LEVEL-1);
  60                 l = &timers->level[level]->list[off];
  61         }
  62
  63         list_add_tail(l, &t->list);
  64 }
  65
  66 void timer_add(struct timers *timers, struct timer *t, struct timespec when)
  67 {
  68         t->time = time_to_grains(when);
  69
  70         /* Added in the past?  Treat it as imminent. */
  71         if (t->time < timers->base)
  72                 t->time = timers->base;
  73         if (t->time < timers->first)
  74                 timers->first = t->time;
  75
  76         timer_add_raw(timers, t);
  77 }
  78
  79 /* FIXME: inline */
  80 void timer_del(struct timers *timers, struct timer *t)
  81 {
  82         list_del(&t->list);
  83 }
  84
  85 static void timers_far_get(struct timers *timers,
  86                            struct list_head *list,
  87                            uint64_t when)
  88 {
  89         struct timer *i, *next;
  90
  91         list_for_each_safe(&timers->far, i, next, list) {
  92                 if (i->time <= when) {
  93                         list_del_from(&timers->far, &i->list);
  94                         list_add_tail(list, &i->list);
  95                 }
  96         }
  97 }
  98
  99 static void add_level(struct timers *timers, unsigned int level)
 100 {
 101         struct timer_level *l;
 102         struct timer *t;
 103         unsigned int i;
 104         struct list_head from_far;
 105
 106         l = malloc(sizeof(*l));
 107         if (!l)
 108                 return;
 109
 110         for (i = 0; i < ARRAY_SIZE(l->list); i++)
 111                 list_head_init(&l->list[i]);
 112         timers->level[level] = l;
 113
 114         list_head_init(&from_far);
 115         timers_far_get(timers, &from_far,
 116                        timers->base + (1ULL << ((level+1)*TIMER_LEVEL_BITS)) - 1);
 117
 118         while ((t = list_pop(&from_far, struct timer, list)) != NULL)
 119                 timer_add_raw(timers, t);
 120 }
 121
 122 /* Take timers from level and distribute them down one. */
 123 static void cascade(struct timers *timers, unsigned int level)
 124 {
 125         struct timer *i;
 126         struct list_head from_far, *list;
 127
 128         if (level == ARRAY_SIZE(timers->level) || !timers->level[level]) {
 129                 list_head_init(&from_far);
 130                 timers_far_get(timers, &from_far,
 131                                timers->base
 132                                + (1ULL << (level*TIMER_LEVEL_BITS))-1);
 133                 list = &from_far;
 134                 if (level != ARRAY_SIZE(timers->level))
 135                         add_level(timers, level);
 136         } else {
 137                 unsigned src;
 138
 139                 src = (timers->base >> (level * TIMER_LEVEL_BITS)) % PER_LEVEL;
 140                 if (src == 0)
 141                         cascade(timers, level + 1);
 142                 list = &timers->level[level]->list[src];
 143         }
 144
 145         while ((i = list_pop(list, struct timer, list)) != NULL) {
 146                 unsigned dst;
 147
 148                 assert(i->time >= timers->base);
 149                 assert(i->time < (timers->base
 150                                   + (1ULL << ((level+1)*TIMER_LEVEL_BITS))));
 151
 152                 dst = (i->time >> ((level-1)*TIMER_LEVEL_BITS)) % PER_LEVEL;
 153                 list_add_tail(&timers->level[level-1]->list[dst], &i->list);
 154         }
 155 }
 156
 157 static const struct timer *find_first(const struct list_head *list,
 158                                       const struct timer *prev)
 159 {
 160         struct timer *t;
 161
 162         list_for_each(list, t, list) {
 163                 if (!prev || t->time < prev->time)
 164                         prev = t;
 165         }
 166         return prev;
 167 }
 168
 169 static const struct timer *get_first(const struct timers *timers)
 170 {
 171         unsigned int level, i, off;
 172         bool need_next;
 173         uint64_t base;
 174         const struct timer *found = NULL;
 175         struct list_head *h;
 176
 177         if (timers->first < timers->base) {
 178                 base = timers->base;
 179                 level = 0;
 180         } else {
 181                 /* May not be accurate, due to timer_del / expiry. */
 182                 level = level_of(timers, timers->first);
 183                 base = timers->first >> (TIMER_LEVEL_BITS * level);
 184         }
 185
 186 next:
 187         if (!timers->level[level])
 188                 return find_first(&timers->far, NULL);
 189
 190         need_next = false;
 191         off = base % PER_LEVEL;
 192         for (i = 0; i < PER_LEVEL; i++) {
 193                 h = &timers->level[level]->list[(i+off) % PER_LEVEL];
 194
 195                 if (!list_empty(h))
 196                         break;
 197
 198                 /* We haven't cascaded yet, so if we wrap, we'll need to
 199                  * check next level, too. */
 200                 if (i + off == PER_LEVEL)
 201                         need_next = true;
 202         }
 203         if (i == PER_LEVEL) {
 204                 level++;
 205                 base >>= TIMER_LEVEL_BITS;
 206                 goto next;
 207         }
 208
 209         /* Level 0 is exact, so they're all the same. */
 210         if (level == 0)
 211                 found = list_top(h, struct timer, list);
 212         else
 213                 found = find_first(h, NULL);
 214
 215         if (need_next) {
 216                 if (!timers->level[level+1]) {
 217                         found = find_first(&timers->far, found);
 218                 } else {
 219                         base >>= TIMER_LEVEL_BITS;
 220                         off = base % PER_LEVEL;
 221                         h = &timers->level[level+1]->list[off];
 222                         found = find_first(h, found);
 223                 }
 224         }
 225         return found;
 226 }
 227
 228 static bool update_first(struct timers *timers)
 229 {
 230         const struct timer *found = get_first(timers);
 231
 232         if (!found) {
 233                 timers->first = -1ULL;
 234                 return false;
 235         }
 236
 237         timers->first = found->time;
 238         return true;
 239 }
 240
 241 bool timer_earliest(struct timers *timers, struct timespec *first)
 242 {
 243         if (!update_first(timers))
 244                 return false;
 245
 246         *first = grains_to_time(timers->first);
 247         return true;
 248 }
 249
 250 /* Assume no timers before 'time', cascade down and update base time. */
 251 static void timer_fast_forward(struct timers *timers, uint64_t time)
 252 {
 253         unsigned int level, changed;
 254         int need_level = -1;
 255         struct list_head list;
 256         struct timer *i;
 257
 258         /* How many bits changed between base and time?
 259          * Each time we wrap, we need to empty buckets from above. */
 260         if (time == timers->base)
 261                 return;
 262
 263         changed = ilog64_nz(time ^ timers->base);
 264         level = (changed - 1) / TIMER_LEVEL_BITS;
 265
 266         /* Buckets always empty downwards, so we could cascade manually,
 267          * but it's rarely very many so we just remove and re-add */
 268         list_head_init(&list);
 269
 270         do {
 271                 if (!timers->level[level]) {
 272                         /* We need any which belong on this level. */
 273                         timers_far_get(timers, &list,
 274                                        timers->base
 275                                        + (1ULL << ((level+1)*TIMER_LEVEL_BITS))-1);
 276                         need_level = level;
 277                 } else {
 278                         unsigned src;
 279
 280                         /* Get all timers from this bucket. */
 281                         src = (time >> (level * TIMER_LEVEL_BITS)) % PER_LEVEL;
 282                         list_append_list(&list,
 283                                          &timers->level[level]->list[src]);
 284                 }
 285         } while (level--);
 286
 287         /* Did we hit the last level?  If so, add. */
 288         if (need_level != -1)
 289                 add_level(timers, need_level);
 290
 291         /* Fast-forward the time, and re-add everyone. */
 292         timers->base = time;
 293         while ((i = list_pop(&list, struct timer, list)) != NULL)
 294                 timer_add_raw(timers, i);
 295 }
 296
 297 /* Fills list of expired timers. */
 298 void timers_expire(struct timers *timers,
 299                    struct timespec expire,
 300                    struct list_head *list)
 301 {
 302         uint64_t now = time_to_grains(expire);
 303         unsigned int off;
 304
 305         assert(now >= timers->base);
 306
 307         list_head_init(list);
 308
 309         if (!timers->level[0]) {
 310                 if (list_empty(&timers->far))
 311                         return;
 312                 add_level(timers, 0);
 313         }
 314
 315         do {
 316                 if (timers->first > now) {
 317                         timer_fast_forward(timers, now);
 318                         break;
 319                 }
 320
 321                 timer_fast_forward(timers, timers->first);
 322                 off = timers->base % PER_LEVEL;
 323
 324                 list_append_list(list, &timers->level[0]->list[off]);
 325                 if (timers->base == now)
 326                         break;
 327         } while (update_first(timers));
 328 }
 329
 330 static bool timer_list_check(const struct list_head *l,
 331                              uint64_t min, uint64_t max, uint64_t first,
 332                              const char *abortstr)
 333 {
 334         const struct timer *t;
 335
 336         if (!list_check(l, abortstr))
 337                 return false;
 338
 339         list_for_each(l, t, list) {
 340                 if (t->time < min || t->time > max) {
 341                         if (abortstr) {
 342                                 fprintf(stderr,
 343                                         "%s: timer %p %llu not %llu-%llu\n",
 344                                         abortstr, t, t->time, min, max);
 345                                 abort();
 346                         }
 347                         return false;
 348                 }
 349                 if (t->time < first) {
 350                         if (abortstr) {
 351                                 fprintf(stderr,
 352                                         "%s: timer %p %llu < minimum %llu\n",
 353                                         abortstr, t, t->time, first);
 354                                 abort();
 355                         }
 356                         return false;
 357                 }
 358         }
 359         return true;
 360 }
 361
 362 struct timers *timers_check(const struct timers *timers, const char *abortstr)
 363 {
 364         unsigned int l, i, off;
 365         uint64_t base;
 366
 367         l = 0;
 368         if (!timers->level[0])
 369                 goto past_levels;
 370
 371         /* First level is simple. */
 372         off = timers->base % PER_LEVEL;
 373         for (i = 0; i < PER_LEVEL; i++) {
 374                 struct list_head *h;
 375
 376                 h = &timers->level[l]->list[(i+off) % PER_LEVEL];
 377                 if (!timer_list_check(h, timers->base + i, timers->base + i,
 378                                       timers->first, abortstr))
 379                         return NULL;
 380         }
 381
 382         /* For other levels, "current" bucket has been emptied, and may contain
 383          * entries for the current + level_size bucket. */
 384         for (l = 1; timers->level[l] && l < PER_LEVEL; l++) {
 385                 uint64_t per_bucket = 1ULL << (TIMER_LEVEL_BITS * l);
 386
 387                 off = ((timers->base >> (l*TIMER_LEVEL_BITS)) % PER_LEVEL);
 388                 /* We start at *next* bucket. */
 389                 base = (timers->base & ~(per_bucket - 1)) + per_bucket;
 390
 391                 for (i = 1; i <= PER_LEVEL; i++) {
 392                         struct list_head *h;
 393
 394                         h = &timers->level[l]->list[(i+off) % PER_LEVEL];
 395                         if (!timer_list_check(h, base, base + per_bucket - 1,
 396                                               timers->first, abortstr))
 397                                 return NULL;
 398                         base += per_bucket;
 399                 }
 400         }
 401
 402 past_levels:
 403         base = (timers->base & ~((1ULL << (TIMER_LEVEL_BITS * l)) - 1))
 404                 + (1ULL << (TIMER_LEVEL_BITS * l)) - 1;
 405         if (!timer_list_check(&timers->far, base, -1ULL, timers->first,
 406                               abortstr))
 407                 return NULL;
 408
 409         return (struct timers *)timers;
 410 }
 411
 412 //#ifdef CCAN_TIMER_DEBUG
 413 void timers_dump(const struct timers *timers, FILE *fp)
 414 {
 415         unsigned int l, i;
 416         uint64_t min, max, num;
 417         struct timer *t;
 418
 419         if (!fp)
 420                 fp = stderr;
 421
 422         fprintf(fp, "Base: %llu\n", timers->base);
 423
 424         for (l = 0; timers->level[l] && l < ARRAY_SIZE(timers->level); l++) {
 425                 fprintf(fp, "Level %i (+%llu):\n",
 426                         l, (uint64_t)1 << (TIMER_LEVEL_BITS * l));
 427                 for (i = 0; i < (1 << TIMER_LEVEL_BITS); i++) {
 428
 429                         if (list_empty(&timers->level[l]->list[i]))
 430                                 continue;
 431                         min = -1ULL;
 432                         max = 0;
 433                         num = 0;
 434                         list_for_each(&timers->level[l]->list[i], t, list) {
 435                                 if (t->time < min)
 436                                         min = t->time;
 437                                 if (t->time > max)
 438                                         max = t->time;
 439                                 num++;
 440                         }
 441                         fprintf(stderr, "  %llu (+%llu-+%llu)\n",
 442                                 num, min - timers->base, max - timers->base);
 443                 }
 444         }
 445
 446         min = -1ULL;
 447         max = 0;
 448         num = 0;
 449         list_for_each(&timers->far, t, list) {
 450                 if (t->time < min)
 451                         min = t->time;
 452                 if (t->time > max)
 453                         max = t->time;
 454                 num++;
 455         }
 456         fprintf(stderr, "Far: %llu (%llu-%llu)\n", num, min, max);
 457 }
 458 //#endif
 459
 460 void timers_cleanup(struct timers *timers)
 461 {
 462         unsigned int l;
 463
 464         for (l = 0; l < ARRAY_SIZE(timers->level); l++)
 465                 free(timers->level[l]);
 466 }