We spend a lot of time searching for the next timer to expire: by caching
the minimum, we can skip most of this work. Even if timers are deleted,
the minimum will be a starting point for searching.
The expected-usage benchmark has to be increased by a factor of 100,
otherwise it's now too short.
Before:
$ ./expected-usage
1000000 in 12.
701647935
After:
$ ./expected-usage
1000000
1000000 in 0.
061095153
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
opt_parse(&argc, argv, opt_log_stderr_exit);
opt_parse(&argc, argv, opt_log_stderr_exit);
- num = argv[1] ? atoi(argv[1]) : (check ? 10000 : 1000000);
+ num = argv[1] ? atoi(argv[1]) : (check ? 100000 : 100000000);
list_head_init(&expired);
curr = start = time_now();
list_head_init(&expired);
curr = start = time_now();
for (timers.base = 0;
timers.base < (1ULL << MAX_ORD)+2;
timers.base = next(timers.base)) {
for (timers.base = 0;
timers.base < (1ULL << MAX_ORD)+2;
timers.base = next(timers.base)) {
- t.time = timers.base + diff;
- timer_add_raw(&timers, &t);
+ timer_add(&timers, &t, grains_to_time(timers.base + diff));
ok1(timers_check(&timers, NULL));
timer_del(&timers, &t);
}
ok1(timers_check(&timers, NULL));
timer_del(&timers, &t);
}
list_head_init(&timers->far);
timers->base = time_to_grains(start);
list_head_init(&timers->far);
timers->base = time_to_grains(start);
for (i = 0; i < ARRAY_SIZE(timers->level); i++)
timers->level[i] = NULL;
}
for (i = 0; i < ARRAY_SIZE(timers->level); i++)
timers->level[i] = NULL;
}
-static void timer_add_raw(struct timers *timers, struct timer *t)
+static unsigned int level_of(const struct timers *timers, uint64_t time)
/* Level depends how far away it is. */
/* Level depends how far away it is. */
- diff = t->time - timers->base;
- level = ilog64(diff / 2) / TIMER_LEVEL_BITS;
+ diff = time - timers->base;
+ return ilog64(diff / 2) / TIMER_LEVEL_BITS;
+}
+
+static void timer_add_raw(struct timers *timers, struct timer *t)
+{
+ struct list_head *l;
+ unsigned int level = level_of(timers, t->time);
if (!timers->level[level])
l = &timers->far;
if (!timers->level[level])
l = &timers->far;
/* Added in the past? Treat it as imminent. */
if (t->time < timers->base)
t->time = timers->base;
/* Added in the past? Treat it as imminent. */
if (t->time < timers->base)
t->time = timers->base;
+ if (t->time < timers->first)
+ timers->first = t->time;
timer_add_raw(timers, t);
}
timer_add_raw(timers, t);
}
-static struct timer *get_first(const struct timers *timers)
+static const struct timer *get_first(const struct timers *timers)
- unsigned int level = 0, i, off;
+ unsigned int level, i, off;
- uint64_t base = timers->base;
const struct timer *found = NULL;
struct list_head *h;
const struct timer *found = NULL;
struct list_head *h;
+ if (timers->first < timers->base) {
+ base = timers->base;
+ level = 0;
+ } else {
+ /* May not be accurate, due to timer_del / expiry. */
+ level = level_of(timers, timers->first);
+ base = timers->first >> (TIMER_LEVEL_BITS * level);
+ }
+
next:
if (!timers->level[level])
next:
if (!timers->level[level])
- return (struct timer *)find_first(&timers->far, NULL);
+ return find_first(&timers->far, NULL);
need_next = false;
off = base % PER_LEVEL;
need_next = false;
off = base % PER_LEVEL;
found = find_first(h, found);
}
}
found = find_first(h, found);
}
}
-
- return (struct timer *)found;
-bool timer_earliest(const struct timers *timers, struct timespec *first)
+static bool update_first(struct timers *timers)
- struct timer *found = get_first(timers);
+ const struct timer *found = get_first(timers);
+
+ if (!found) {
+ timers->first = -1ULL;
+ return false;
+ }
+
+ timers->first = found->time;
+ return true;
+}
+bool timer_earliest(struct timers *timers, struct timespec *first)
+{
+ if (!update_first(timers))
- *first = grains_to_time(found->time);
+
+ *first = grains_to_time(timers->first);
{
uint64_t now = time_to_grains(expire);
unsigned int off;
{
uint64_t now = time_to_grains(expire);
unsigned int off;
- const struct timer *first;
assert(now >= timers->base);
assert(now >= timers->base);
- while ((first = get_first(timers)) != NULL) {
- assert(first->time >= timers->base);
- if (first->time > now) {
+ do {
+ if (timers->first > now) {
timer_fast_forward(timers, now);
break;
}
timer_fast_forward(timers, now);
break;
}
- timer_fast_forward(timers, first->time);
+ timer_fast_forward(timers, timers->first);
off = timers->base % PER_LEVEL;
list_append_list(list, &timers->level[0]->list[off]);
if (timers->base == now)
break;
off = timers->base % PER_LEVEL;
list_append_list(list, &timers->level[0]->list[off]);
if (timers->base == now)
break;
+ } while (update_first(timers));
}
static bool timer_list_check(const struct list_head *l,
}
static bool timer_list_check(const struct list_head *l,
- uint64_t min, uint64_t max,
+ uint64_t min, uint64_t max, uint64_t first,
const char *abortstr)
{
const struct timer *t;
const char *abortstr)
{
const struct timer *t;
+ if (t->time < first) {
+ if (abortstr) {
+ fprintf(stderr,
+ "%s: timer %p %llu < minimum %llu\n",
+ abortstr, t, t->time, first);
+ abort();
+ }
+ return false;
+ }
h = &timers->level[l]->list[(i+off) % PER_LEVEL];
if (!timer_list_check(h, timers->base + i, timers->base + i,
h = &timers->level[l]->list[(i+off) % PER_LEVEL];
if (!timer_list_check(h, timers->base + i, timers->base + i,
+ timers->first, abortstr))
h = &timers->level[l]->list[(i+off) % PER_LEVEL];
if (!timer_list_check(h, base, base + per_bucket - 1,
h = &timers->level[l]->list[(i+off) % PER_LEVEL];
if (!timer_list_check(h, base, base + per_bucket - 1,
+ timers->first, abortstr))
return NULL;
base += per_bucket;
}
return NULL;
base += per_bucket;
}
past_levels:
base = (timers->base & ~((1ULL << (TIMER_LEVEL_BITS * l)) - 1))
+ (1ULL << (TIMER_LEVEL_BITS * l)) - 1;
past_levels:
base = (timers->base & ~((1ULL << (TIMER_LEVEL_BITS * l)) - 1))
+ (1ULL << (TIMER_LEVEL_BITS * l)) - 1;
- if (!timer_list_check(&timers->far, base, -1ULL, abortstr))
+ if (!timer_list_check(&timers->far, base, -1ULL, timers->first,
+ abortstr))
return NULL;
return (struct timers *)timers;
return NULL;
return (struct timers *)timers;
* timers. Otherwise, it sets @first to the expiry time of the first
* timer (rounded to TIMER_GRANULARITY nanoseconds), and returns true.
*/
* timers. Otherwise, it sets @first to the expiry time of the first
* timer (rounded to TIMER_GRANULARITY nanoseconds), and returns true.
*/
-bool timer_earliest(const struct timers *timers, struct timespec *first);
+bool timer_earliest(struct timers *timers, struct timespec *first);
/**
* timer_expire - update timers structure and remove expired timers.
/**
* timer_expire - update timers structure and remove expired timers.
/* Far in the future. */
struct list_head far;
uint64_t base;
/* Far in the future. */
struct list_head far;
uint64_t base;
struct timer_level *level[(64 + TIMER_LEVEL_BITS-1) / TIMER_LEVEL_BITS];
};
struct timer_level *level[(64 + TIMER_LEVEL_BITS-1) / TIMER_LEVEL_BITS];
};