1 /* LGPL (v2.1 or any later version) - see LICENSE file for details */
2 #include <ccan/timer/timer.h>
3 #include <ccan/array_size/array_size.h>
4 #include <ccan/ilog/ilog.h>
5 #include <ccan/likely/likely.h>
9 #define PER_LEVEL (1ULL << TIMER_LEVEL_BITS)
12 struct list_head list[PER_LEVEL];
15 static uint64_t time_to_grains(struct timespec ts)
17 return ts.tv_sec * ((uint64_t)1000000000 / TIMER_GRANULARITY)
18 + (ts.tv_nsec / TIMER_GRANULARITY);
21 static struct timespec grains_to_time(uint64_t grains)
25 ts.tv_sec = grains / (1000000000 / TIMER_GRANULARITY);
26 ts.tv_nsec = (grains % (1000000000 / TIMER_GRANULARITY))
31 void timers_init(struct timers *timers, struct timespec start)
35 list_head_init(&timers->far);
36 timers->base = time_to_grains(start);
37 for (i = 0; i < ARRAY_SIZE(timers->level); i++)
38 timers->level[i] = NULL;
41 static void timer_add_raw(struct timers *timers, struct timer *t)
47 /* Level depends how far away it is. */
48 diff = t->time - timers->base;
49 level = ilog64(diff / 2) / TIMER_LEVEL_BITS;
51 if (!timers->level[level])
54 int off = (t->time >> (level*TIMER_LEVEL_BITS)) & (PER_LEVEL-1);
55 l = &timers->level[level]->list[off];
58 list_add_tail(l, &t->list);
61 void timer_add(struct timers *timers, struct timer *t, struct timespec when)
63 t->time = time_to_grains(when);
65 /* Added in the past? Treat it as imminent. */
66 if (t->time < timers->base)
67 t->time = timers->base;
69 timer_add_raw(timers, t);
73 void timer_del(struct timers *timers, struct timer *t)
78 static void timers_far_get(struct timers *timers,
79 struct list_head *list,
82 struct timer *i, *next;
84 list_for_each_safe(&timers->far, i, next, list) {
85 if (i->time <= when) {
86 list_del_from(&timers->far, &i->list);
87 list_add_tail(list, &i->list);
92 static void add_level(struct timers *timers, unsigned int level)
94 struct timer_level *l;
97 struct list_head from_far;
99 l = malloc(sizeof(*l));
103 for (i = 0; i < ARRAY_SIZE(l->list); i++)
104 list_head_init(&l->list[i]);
105 timers->level[level] = l;
107 list_head_init(&from_far);
108 timers_far_get(timers, &from_far,
109 timers->base + (1ULL << ((level+1)*TIMER_LEVEL_BITS)) - 1);
111 while ((t = list_pop(&from_far, struct timer, list)) != NULL)
112 timer_add_raw(timers, t);
115 /* Take timers from level and distribute them down one. */
116 static void cascade(struct timers *timers, unsigned int level)
119 struct list_head from_far, *list;
121 if (level == ARRAY_SIZE(timers->level) || !timers->level[level]) {
122 list_head_init(&from_far);
123 timers_far_get(timers, &from_far,
125 + (1ULL << (level*TIMER_LEVEL_BITS))-1);
127 if (level != ARRAY_SIZE(timers->level))
128 add_level(timers, level);
132 src = (timers->base >> (level * TIMER_LEVEL_BITS)) % PER_LEVEL;
134 cascade(timers, level + 1);
135 list = &timers->level[level]->list[src];
138 while ((i = list_pop(list, struct timer, list)) != NULL) {
141 assert(i->time >= timers->base);
142 assert(i->time < (timers->base
143 + (1ULL << ((level+1)*TIMER_LEVEL_BITS))));
145 dst = (i->time >> ((level-1)*TIMER_LEVEL_BITS)) % PER_LEVEL;
146 list_add_tail(&timers->level[level-1]->list[dst], &i->list);
150 static const struct timer *find_first(const struct list_head *list,
151 const struct timer *prev)
155 list_for_each(list, t, list) {
156 if (!prev || t->time < prev->time)
162 static struct timer *get_first(const struct timers *timers)
164 unsigned int level = 0, i, off;
166 uint64_t base = timers->base;
167 const struct timer *found = NULL;
171 if (!timers->level[level])
172 return (struct timer *)find_first(&timers->far, NULL);
175 off = base % PER_LEVEL;
176 for (i = 0; i < PER_LEVEL; i++) {
177 h = &timers->level[level]->list[(i+off) % PER_LEVEL];
182 /* We haven't cascaded yet, so if we wrap, we'll need to
183 * check next level, too. */
184 if (i + off == PER_LEVEL)
187 if (i == PER_LEVEL) {
189 base >>= TIMER_LEVEL_BITS;
193 /* Level 0 is exact, so they're all the same. */
195 found = list_top(h, struct timer, list);
197 found = find_first(h, NULL);
200 if (!timers->level[level+1]) {
201 found = find_first(&timers->far, found);
203 base >>= TIMER_LEVEL_BITS;
204 off = base % PER_LEVEL;
205 h = &timers->level[level+1]->list[off];
206 found = find_first(h, found);
210 return (struct timer *)found;
213 bool timer_earliest(const struct timers *timers, struct timespec *first)
215 struct timer *found = get_first(timers);
219 *first = grains_to_time(found->time);
223 /* Assume no timers before 'time', cascade down and update base time. */
224 static void timer_fast_forward(struct timers *timers, uint64_t time)
226 unsigned int level, changed;
228 struct list_head list;
231 /* How many bits changed between base and time?
232 * Each time we wrap, we need to empty buckets from above. */
233 if (time == timers->base)
236 changed = ilog64_nz(time ^ timers->base);
237 level = (changed - 1) / TIMER_LEVEL_BITS;
239 /* Buckets always empty downwards, so we could cascade manually,
240 * but it's rarely very many so we just remove and re-add */
241 list_head_init(&list);
244 if (!timers->level[level]) {
245 /* We need any which belong on this level. */
246 timers_far_get(timers, &list,
248 + (1ULL << ((level+1)*TIMER_LEVEL_BITS))-1);
253 /* Get all timers from this bucket. */
254 src = (time >> (level * TIMER_LEVEL_BITS)) % PER_LEVEL;
255 list_append_list(&list,
256 &timers->level[level]->list[src]);
260 /* Did we hit the last level? If so, add. */
261 if (need_level != -1)
262 add_level(timers, need_level);
264 /* Fast-forward the time, and re-add everyone. */
266 while ((i = list_pop(&list, struct timer, list)) != NULL)
267 timer_add_raw(timers, i);
270 /* Fills list of expired timers. */
271 void timers_expire(struct timers *timers,
272 struct timespec expire,
273 struct list_head *list)
275 uint64_t now = time_to_grains(expire);
277 const struct timer *first;
279 assert(now >= timers->base);
281 list_head_init(list);
283 if (!timers->level[0]) {
284 if (list_empty(&timers->far))
286 add_level(timers, 0);
289 while ((first = get_first(timers)) != NULL) {
290 assert(first->time >= timers->base);
291 if (first->time > now) {
292 timer_fast_forward(timers, now);
296 timer_fast_forward(timers, first->time);
297 off = timers->base % PER_LEVEL;
299 list_append_list(list, &timers->level[0]->list[off]);
300 if (timers->base == now)
305 static bool timer_list_check(const struct list_head *l,
306 uint64_t min, uint64_t max,
307 const char *abortstr)
309 const struct timer *t;
311 if (!list_check(l, abortstr))
314 list_for_each(l, t, list) {
315 if (t->time < min || t->time > max) {
318 "%s: timer %p %llu not %llu-%llu\n",
319 abortstr, t, t->time, min, max);
328 struct timers *timers_check(const struct timers *timers, const char *abortstr)
330 unsigned int l, i, off;
334 if (!timers->level[0])
337 /* First level is simple. */
338 off = timers->base % PER_LEVEL;
339 for (i = 0; i < PER_LEVEL; i++) {
342 h = &timers->level[l]->list[(i+off) % PER_LEVEL];
343 if (!timer_list_check(h, timers->base + i, timers->base + i,
348 /* For other levels, "current" bucket has been emptied, and may contain
349 * entries for the current + level_size bucket. */
350 for (l = 1; timers->level[l] && l < PER_LEVEL; l++) {
351 uint64_t per_bucket = 1ULL << (TIMER_LEVEL_BITS * l);
353 off = ((timers->base >> (l*TIMER_LEVEL_BITS)) % PER_LEVEL);
354 /* We start at *next* bucket. */
355 base = (timers->base & ~(per_bucket - 1)) + per_bucket;
357 for (i = 1; i <= PER_LEVEL; i++) {
360 h = &timers->level[l]->list[(i+off) % PER_LEVEL];
361 if (!timer_list_check(h, base, base + per_bucket - 1,
369 base = (timers->base & ~((1ULL << (TIMER_LEVEL_BITS * l)) - 1))
370 + (1ULL << (TIMER_LEVEL_BITS * l)) - 1;
371 if (!timer_list_check(&timers->far, base, -1ULL, abortstr))
374 return (struct timers *)timers;
377 //#ifdef CCAN_TIMER_DEBUG
378 void timers_dump(const struct timers *timers, FILE *fp)
381 uint64_t min, max, num;
387 fprintf(fp, "Base: %llu\n", timers->base);
389 for (l = 0; timers->level[l] && l < ARRAY_SIZE(timers->level); l++) {
390 fprintf(fp, "Level %i (+%llu):\n",
391 l, (uint64_t)1 << (TIMER_LEVEL_BITS * l));
392 for (i = 0; i < (1 << TIMER_LEVEL_BITS); i++) {
394 if (list_empty(&timers->level[l]->list[i]))
399 list_for_each(&timers->level[l]->list[i], t, list) {
406 fprintf(stderr, " %llu (+%llu-+%llu)\n",
407 num, min - timers->base, max - timers->base);
414 list_for_each(&timers->far, t, list) {
421 fprintf(stderr, "Far: %llu (%llu-%llu)\n", num, min, max);
425 void timers_cleanup(struct timers *timers)
429 for (l = 0; l < ARRAY_SIZE(timers->level); l++)
430 free(timers->level[l]);