openkylin
/
linux
mirror of https://gitee.com/openkylin/linux.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

cfq-iosched: Convert from jiffies to nanoseconds 

Convert all time-keeping in CFQ IO scheduler from jiffies to nanoseconds
so that we can later make the intervals more fine-grained than jiffies.
One jiffie is several miliseconds and even for today's rotating disks
that is a noticeable amount of time and thus we leave disk unnecessarily
idle.

Signed-off-by: Jeff Moyer <jmoyer@redhat.com>
Signed-off-by: Jan Kara <jack@suse.cz>
Signed-off-by: Jens Axboe <axboe@fb.com>
This commit is contained in:
Jeff Moyer 2016-06-08 08:55:34 -06:00 committed by Jens Axboe
parent 28a8f0d317
commit 9a7f38c42c
1 changed files with 136 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

273
block/cfq-iosched.c
View File

@ -10,7 +10,7 @@
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/blkdev.h> #include <linux/blkdev.h>
#include <linux/elevator.h> #include <linux/elevator.h>
#include <linux/jiffies.h> #include <linux/ktime.h>
#include <linux/rbtree.h> #include <linux/rbtree.h>
#include <linux/ioprio.h> #include <linux/ioprio.h>
#include <linux/blktrace_api.h> #include <linux/blktrace_api.h>
@ -22,28 +22,28 @@
*/ */
/* max queue in one round of service */ /* max queue in one round of service */
static const int cfq_quantum = 8; static const int cfq_quantum = 8;
static const int cfq_fifo_expire[2] = { HZ / 4, HZ / 8 }; static const u64 cfq_fifo_expire[2] = { NSEC_PER_SEC / 4, NSEC_PER_SEC / 8 };
/* maximum backwards seek, in KiB */ /* maximum backwards seek, in KiB */
static const int cfq_back_max = 16 * 1024; static const int cfq_back_max = 16 * 1024;
/* penalty of a backwards seek */ /* penalty of a backwards seek */
static const int cfq_back_penalty = 2; static const int cfq_back_penalty = 2;
static const int cfq_slice_sync = HZ / 10; static const u64 cfq_slice_sync = NSEC_PER_SEC / 10;
static int cfq_slice_async = HZ / 25; static u64 cfq_slice_async = NSEC_PER_SEC / 25;
static const int cfq_slice_async_rq = 2; static const int cfq_slice_async_rq = 2;
static int cfq_slice_idle = HZ / 125; static u64 cfq_slice_idle = NSEC_PER_SEC / 125;
static int cfq_group_idle = HZ / 125; static u64 cfq_group_idle = NSEC_PER_SEC / 125;
static const int cfq_target_latency = HZ * 3/10; /* 300 ms */ static const u64 cfq_target_latency = (u64)NSEC_PER_SEC * 3/10; /* 300 ms */
static const int cfq_hist_divisor = 4; static const int cfq_hist_divisor = 4;
/* /*
* offset from end of service tree * offset from end of service tree
*/ */
#define CFQ_IDLE_DELAY (HZ / 5) #define CFQ_IDLE_DELAY (NSEC_PER_SEC / 5)
/* /*
* below this threshold, we consider thinktime immediate * below this threshold, we consider thinktime immediate
*/ */
#define CFQ_MIN_TT (2) #define CFQ_MIN_TT (2 * NSEC_PER_SEC / HZ)
#define CFQ_SLICE_SCALE (5) #define CFQ_SLICE_SCALE (5)
#define CFQ_HW_QUEUE_MIN (5) #define CFQ_HW_QUEUE_MIN (5)
@ -73,11 +73,11 @@ static struct kmem_cache *cfq_pool;
#define CFQ_WEIGHT_LEGACY_MAX 1000 #define CFQ_WEIGHT_LEGACY_MAX 1000
struct cfq_ttime { struct cfq_ttime {
unsigned long last_end_request; u64 last_end_request;
unsigned long ttime_total; u64 ttime_total;
u64 ttime_mean;
unsigned long ttime_samples; unsigned long ttime_samples;
unsigned long ttime_mean;
}; };
/* /*
@ -94,7 +94,7 @@ struct cfq_rb_root {
struct cfq_ttime ttime; struct cfq_ttime ttime;
}; };
#define CFQ_RB_ROOT (struct cfq_rb_root) { .rb = RB_ROOT, \ #define CFQ_RB_ROOT (struct cfq_rb_root) { .rb = RB_ROOT, \
.ttime = {.last_end_request = jiffies,},} .ttime = {.last_end_request = ktime_get_ns(),},}
/* /*
* Per process-grouping structure * Per process-grouping structure
@ -109,7 +109,7 @@ struct cfq_queue {
/* service_tree member */ /* service_tree member */
struct rb_node rb_node; struct rb_node rb_node;
/* service_tree key */ /* service_tree key */
unsigned long rb_key; u64 rb_key;
/* prio tree member */ /* prio tree member */
struct rb_node p_node; struct rb_node p_node;
/* prio tree root we belong to, if any */ /* prio tree root we belong to, if any */
@ -126,13 +126,13 @@ struct cfq_queue {
struct list_head fifo; struct list_head fifo;
/* time when queue got scheduled in to dispatch first request. */ /* time when queue got scheduled in to dispatch first request. */
unsigned long dispatch_start; u64 dispatch_start;
unsigned int allocated_slice; u64 allocated_slice;
unsigned int slice_dispatch; u64 slice_dispatch;
/* time when first request from queue completed and slice started. */ /* time when first request from queue completed and slice started. */
unsigned long slice_start; u64 slice_start;
unsigned long slice_end; u64 slice_end;
long slice_resid; u64 slice_resid;
/* pending priority requests */ /* pending priority requests */
int prio_pending; int prio_pending;
@ -290,7 +290,7 @@ struct cfq_group {
struct cfq_rb_root service_trees[2][3]; struct cfq_rb_root service_trees[2][3];
struct cfq_rb_root service_tree_idle; struct cfq_rb_root service_tree_idle;
unsigned long saved_wl_slice; u64 saved_wl_slice;
enum wl_type_t saved_wl_type; enum wl_type_t saved_wl_type;
enum wl_class_t saved_wl_class; enum wl_class_t saved_wl_class;
@ -329,7 +329,7 @@ struct cfq_data {
*/ */
enum wl_class_t serving_wl_class; enum wl_class_t serving_wl_class;
enum wl_type_t serving_wl_type; enum wl_type_t serving_wl_type;
unsigned long workload_expires; u64 workload_expires;
struct cfq_group *serving_group; struct cfq_group *serving_group;
/* /*
@ -374,22 +374,22 @@ struct cfq_data {
* tunables, see top of file * tunables, see top of file
*/ */
unsigned int cfq_quantum; unsigned int cfq_quantum;
unsigned int cfq_fifo_expire[2];
unsigned int cfq_back_penalty; unsigned int cfq_back_penalty;
unsigned int cfq_back_max; unsigned int cfq_back_max;
unsigned int cfq_slice[2];
unsigned int cfq_slice_async_rq; unsigned int cfq_slice_async_rq;
unsigned int cfq_slice_idle;
unsigned int cfq_group_idle;
unsigned int cfq_latency; unsigned int cfq_latency;
unsigned int cfq_target_latency; u64 cfq_fifo_expire[2];
u64 cfq_slice[2];
u64 cfq_slice_idle;
u64 cfq_group_idle;
u64 cfq_target_latency;
/* /*
* Fallback dummy cfqq for extreme OOM conditions * Fallback dummy cfqq for extreme OOM conditions
*/ */
struct cfq_queue oom_cfqq; struct cfq_queue oom_cfqq;
unsigned long last_delayed_sync; u64 last_delayed_sync;
}; };
static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd); static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd);
@ -676,7 +676,7 @@ static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,
} }
static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg, static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,
unsigned long time, unsigned long unaccounted_time) uint64_t time, unsigned long unaccounted_time)
{ {
blkg_stat_add(&cfqg->stats.time, time); blkg_stat_add(&cfqg->stats.time, time);
#ifdef CONFIG_DEBUG_BLK_CGROUP #ifdef CONFIG_DEBUG_BLK_CGROUP
@ -788,7 +788,7 @@ static inline void cfqg_put(struct cfq_group *cfqg) { }
static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg, static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,
struct cfq_group *curr_cfqg, int op, int op_flags) { } struct cfq_group *curr_cfqg, int op, int op_flags) { }
static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg, static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,
unsigned long time, unsigned long unaccounted_time) { } uint64_t time, unsigned long unaccounted_time) { }
static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int op, static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int op,
int op_flags) { } int op_flags) { }
static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int op, static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int op,
@ -815,7 +815,7 @@ static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,
static inline bool cfq_io_thinktime_big(struct cfq_data *cfqd, static inline bool cfq_io_thinktime_big(struct cfq_data *cfqd,
struct cfq_ttime *ttime, bool group_idle) struct cfq_ttime *ttime, bool group_idle)
{ {
unsigned long slice; u64 slice;
if (!sample_valid(ttime->ttime_samples)) if (!sample_valid(ttime->ttime_samples))
return false; return false;
if (group_idle) if (group_idle)
@ -938,17 +938,18 @@ static inline void cfq_schedule_dispatch(struct cfq_data *cfqd)
* if a queue is marked sync and has sync io queued. A sync queue with async * if a queue is marked sync and has sync io queued. A sync queue with async
* io only, should not get full sync slice length. * io only, should not get full sync slice length.
*/ */
static inline int cfq_prio_slice(struct cfq_data *cfqd, bool sync, static inline u64 cfq_prio_slice(struct cfq_data *cfqd, bool sync,
unsigned short prio) unsigned short prio)
{ {
const int base_slice = cfqd->cfq_slice[sync]; u64 base_slice = cfqd->cfq_slice[sync];
u64 slice = div_u64(base_slice, CFQ_SLICE_SCALE);
WARN_ON(prio >= IOPRIO_BE_NR); WARN_ON(prio >= IOPRIO_BE_NR);
return base_slice + (base_slice/CFQ_SLICE_SCALE * (4 - prio)); return base_slice + (slice * (4 - prio));
} }
static inline int static inline u64
cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
{ {
return cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio); return cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio);
@ -966,15 +967,14 @@ cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
* *
* The result is also in fixed point w/ CFQ_SERVICE_SHIFT. * The result is also in fixed point w/ CFQ_SERVICE_SHIFT.
*/ */
static inline u64 cfqg_scale_charge(unsigned long charge, static inline u64 cfqg_scale_charge(u64 charge,
unsigned int vfraction) unsigned int vfraction)
{ {
u64 c = charge << CFQ_SERVICE_SHIFT; /* make it fixed point */ u64 c = charge << CFQ_SERVICE_SHIFT; /* make it fixed point */
/* charge / vfraction */ /* charge / vfraction */
c <<= CFQ_SERVICE_SHIFT; c <<= CFQ_SERVICE_SHIFT;
do_div(c, vfraction); return div_u64(c, vfraction);
return c;
} }
static inline u64 max_vdisktime(u64 min_vdisktime, u64 vdisktime) static inline u64 max_vdisktime(u64 min_vdisktime, u64 vdisktime)
@ -1027,16 +1027,16 @@ static inline unsigned cfq_group_get_avg_queues(struct cfq_data *cfqd,
return cfqg->busy_queues_avg[rt]; return cfqg->busy_queues_avg[rt];
} }
static inline unsigned static inline u64
cfq_group_slice(struct cfq_data *cfqd, struct cfq_group *cfqg) cfq_group_slice(struct cfq_data *cfqd, struct cfq_group *cfqg)
{ {
return cfqd->cfq_target_latency * cfqg->vfraction >> CFQ_SERVICE_SHIFT; return cfqd->cfq_target_latency * cfqg->vfraction >> CFQ_SERVICE_SHIFT;
} }
static inline unsigned static inline u64
cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
{ {
unsigned slice = cfq_prio_to_slice(cfqd, cfqq); u64 slice = cfq_prio_to_slice(cfqd, cfqq);
if (cfqd->cfq_latency) { if (cfqd->cfq_latency) {
/* /*
* interested queues (we consider only the ones with the same * interested queues (we consider only the ones with the same
@ -1044,20 +1044,22 @@ cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
*/ */
unsigned iq = cfq_group_get_avg_queues(cfqd, cfqq->cfqg, unsigned iq = cfq_group_get_avg_queues(cfqd, cfqq->cfqg,
cfq_class_rt(cfqq)); cfq_class_rt(cfqq));
unsigned sync_slice = cfqd->cfq_slice[1]; u64 sync_slice = cfqd->cfq_slice[1];
unsigned expect_latency = sync_slice * iq; u64 expect_latency = sync_slice * iq;
unsigned group_slice = cfq_group_slice(cfqd, cfqq->cfqg); u64 group_slice = cfq_group_slice(cfqd, cfqq->cfqg);
if (expect_latency > group_slice) { if (expect_latency > group_slice) {
unsigned base_low_slice = 2 * cfqd->cfq_slice_idle; u64 base_low_slice = 2 * cfqd->cfq_slice_idle;
u64 low_slice;
/* scale low_slice according to IO priority /* scale low_slice according to IO priority
* and sync vs async */ * and sync vs async */
unsigned low_slice = low_slice = div64_u64(base_low_slice*slice, sync_slice);
min(slice, base_low_slice * slice / sync_slice); low_slice = min(slice, low_slice);
/* the adapted slice value is scaled to fit all iqs /* the adapted slice value is scaled to fit all iqs
* into the target latency */ * into the target latency */
slice = max(slice * group_slice / expect_latency, slice = div64_u64(slice*group_slice, expect_latency);
low_slice); slice = max(slice, low_slice);
} }
} }
return slice; return slice;
@ -1066,12 +1068,13 @@ cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
static inline void static inline void
cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
{ {
unsigned slice = cfq_scaled_cfqq_slice(cfqd, cfqq); u64 slice = cfq_scaled_cfqq_slice(cfqd, cfqq);
u64 now = ktime_get_ns();
cfqq->slice_start = jiffies; cfqq->slice_start = now;
cfqq->slice_end = jiffies + slice; cfqq->slice_end = now + slice;
cfqq->allocated_slice = slice; cfqq->allocated_slice = slice;
cfq_log_cfqq(cfqd, cfqq, "set_slice=%lu", cfqq->slice_end - jiffies); cfq_log_cfqq(cfqd, cfqq, "set_slice=%llu", cfqq->slice_end - now);
} }
/* /*
@ -1083,7 +1086,7 @@ static inline bool cfq_slice_used(struct cfq_queue *cfqq)
{ {
if (cfq_cfqq_slice_new(cfqq)) if (cfq_cfqq_slice_new(cfqq))
return false; return false;
if (time_before(jiffies, cfqq->slice_end)) if (ktime_get_ns() < cfqq->slice_end)
return false; return false;
return true; return true;
@ -1249,8 +1252,8 @@ cfq_find_next_rq(struct cfq_data *cfqd, struct cfq_queue *cfqq,
return cfq_choose_req(cfqd, next, prev, blk_rq_pos(last)); return cfq_choose_req(cfqd, next, prev, blk_rq_pos(last));
} }
static unsigned long cfq_slice_offset(struct cfq_data *cfqd, static u64 cfq_slice_offset(struct cfq_data *cfqd,
struct cfq_queue *cfqq) struct cfq_queue *cfqq)
{ {
/* /*
* just an approximation, should be ok. * just an approximation, should be ok.
@ -1443,31 +1446,31 @@ cfq_group_notify_queue_del(struct cfq_data *cfqd, struct cfq_group *cfqg)
cfqg_stats_update_dequeue(cfqg); cfqg_stats_update_dequeue(cfqg);
} }
static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq, static inline u64 cfq_cfqq_slice_usage(struct cfq_queue *cfqq,
unsigned int *unaccounted_time) u64 *unaccounted_time)
{ {
unsigned int slice_used; u64 slice_used;
u64 now = ktime_get_ns();
/* /*
* Queue got expired before even a single request completed or * Queue got expired before even a single request completed or
* got expired immediately after first request completion. * got expired immediately after first request completion.
*/ */
if (!cfqq->slice_start || cfqq->slice_start == jiffies) { if (!cfqq->slice_start || cfqq->slice_start == now) {
/* /*
* Also charge the seek time incurred to the group, otherwise * Also charge the seek time incurred to the group, otherwise
* if there are mutiple queues in the group, each can dispatch * if there are mutiple queues in the group, each can dispatch
* a single request on seeky media and cause lots of seek time * a single request on seeky media and cause lots of seek time
* and group will never know it. * and group will never know it.
*/ */
slice_used = max_t(unsigned, (jiffies - cfqq->dispatch_start), slice_used = max_t(u64, (now - cfqq->dispatch_start), 1);
1);
} else { } else {
slice_used = jiffies - cfqq->slice_start; slice_used = now - cfqq->slice_start;
if (slice_used > cfqq->allocated_slice) { if (slice_used > cfqq->allocated_slice) {
*unaccounted_time = slice_used - cfqq->allocated_slice; *unaccounted_time = slice_used - cfqq->allocated_slice;
slice_used = cfqq->allocated_slice; slice_used = cfqq->allocated_slice;
} }
if (time_after(cfqq->slice_start, cfqq->dispatch_start)) if (cfqq->slice_start > cfqq->dispatch_start)
*unaccounted_time += cfqq->slice_start - *unaccounted_time += cfqq->slice_start -
cfqq->dispatch_start; cfqq->dispatch_start;
} }
@ -1479,10 +1482,11 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg,
struct cfq_queue *cfqq) struct cfq_queue *cfqq)
{ {
struct cfq_rb_root *st = &cfqd->grp_service_tree; struct cfq_rb_root *st = &cfqd->grp_service_tree;
unsigned int used_sl, charge, unaccounted_sl = 0; u64 used_sl, charge, unaccounted_sl = 0;
int nr_sync = cfqg->nr_cfqq - cfqg_busy_async_queues(cfqd, cfqg) int nr_sync = cfqg->nr_cfqq - cfqg_busy_async_queues(cfqd, cfqg)
- cfqg->service_tree_idle.count; - cfqg->service_tree_idle.count;
unsigned int vfr; unsigned int vfr;
u64 now = ktime_get_ns();
BUG_ON(nr_sync < 0); BUG_ON(nr_sync < 0);
used_sl = charge = cfq_cfqq_slice_usage(cfqq, &unaccounted_sl); used_sl = charge = cfq_cfqq_slice_usage(cfqq, &unaccounted_sl);
@ -1504,9 +1508,8 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg,
cfq_group_service_tree_add(st, cfqg); cfq_group_service_tree_add(st, cfqg);
/* This group is being expired. Save the context */ /* This group is being expired. Save the context */
if (time_after(cfqd->workload_expires, jiffies)) { if (cfqd->workload_expires > now) {
cfqg->saved_wl_slice = cfqd->workload_expires cfqg->saved_wl_slice = cfqd->workload_expires - now;
- jiffies;
cfqg->saved_wl_type = cfqd->serving_wl_type; cfqg->saved_wl_type = cfqd->serving_wl_type;
cfqg->saved_wl_class = cfqd->serving_wl_class; cfqg->saved_wl_class = cfqd->serving_wl_class;
} else } else
@ -1515,7 +1518,7 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg,
cfq_log_cfqg(cfqd, cfqg, "served: vt=%llu min_vt=%llu", cfqg->vdisktime, cfq_log_cfqg(cfqd, cfqg, "served: vt=%llu min_vt=%llu", cfqg->vdisktime,
st->min_vdisktime); st->min_vdisktime);
cfq_log_cfqq(cfqq->cfqd, cfqq, cfq_log_cfqq(cfqq->cfqd, cfqq,
"sl_used=%u disp=%u charge=%u iops=%u sect=%lu", "sl_used=%llu disp=%llu charge=%llu iops=%u sect=%lu",
used_sl, cfqq->slice_dispatch, charge, used_sl, cfqq->slice_dispatch, charge,
iops_mode(cfqd), cfqq->nr_sectors); iops_mode(cfqd), cfqq->nr_sectors);
cfqg_stats_update_timeslice_used(cfqg, used_sl, unaccounted_sl); cfqg_stats_update_timeslice_used(cfqg, used_sl, unaccounted_sl);
@ -1538,7 +1541,7 @@ static void cfq_init_cfqg_base(struct cfq_group *cfqg)
*st = CFQ_RB_ROOT; *st = CFQ_RB_ROOT;
RB_CLEAR_NODE(&cfqg->rb_node); RB_CLEAR_NODE(&cfqg->rb_node);
cfqg->ttime.last_end_request = jiffies; cfqg->ttime.last_end_request = ktime_get_ns();
} }
#ifdef CONFIG_CFQ_GROUP_IOSCHED #ifdef CONFIG_CFQ_GROUP_IOSCHED
@ -2221,10 +2224,11 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,
{ {
struct rb_node **p, *parent; struct rb_node **p, *parent;
struct cfq_queue *__cfqq; struct cfq_queue *__cfqq;
unsigned long rb_key; u64 rb_key;
struct cfq_rb_root *st; struct cfq_rb_root *st;
int left; int left;
int new_cfqq = 1; int new_cfqq = 1;
u64 now = ktime_get_ns();
st = st_for(cfqq->cfqg, cfqq_class(cfqq), cfqq_type(cfqq)); st = st_for(cfqq->cfqg, cfqq_class(cfqq), cfqq_type(cfqq));
if (cfq_class_idle(cfqq)) { if (cfq_class_idle(cfqq)) {
@ -2234,7 +2238,7 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,
__cfqq = rb_entry(parent, struct cfq_queue, rb_node); __cfqq = rb_entry(parent, struct cfq_queue, rb_node);
rb_key += __cfqq->rb_key; rb_key += __cfqq->rb_key;
} else } else
rb_key += jiffies; rb_key += now;
} else if (!add_front) { } else if (!add_front) {
/* /*
* Get our rb key offset. Subtract any residual slice * Get our rb key offset. Subtract any residual slice
@ -2242,13 +2246,13 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,
* count indicates slice overrun, and this should position * count indicates slice overrun, and this should position
* the next service time further away in the tree. * the next service time further away in the tree.
*/ */
rb_key = cfq_slice_offset(cfqd, cfqq) + jiffies; rb_key = cfq_slice_offset(cfqd, cfqq) + now;
rb_key -= cfqq->slice_resid; rb_key -= cfqq->slice_resid;
cfqq->slice_resid = 0; cfqq->slice_resid = 0;
} else { } else {
rb_key = -HZ; rb_key = -NSEC_PER_SEC;
__cfqq = cfq_rb_first(st); __cfqq = cfq_rb_first(st);
rb_key += __cfqq ? __cfqq->rb_key : jiffies; rb_key += __cfqq ? __cfqq->rb_key : now;
} }
if (!RB_EMPTY_NODE(&cfqq->rb_node)) { if (!RB_EMPTY_NODE(&cfqq->rb_node)) {
@ -2274,7 +2278,7 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,
/* /*
* sort by key, that represents service time. * sort by key, that represents service time.
*/ */
if (time_before(rb_key, __cfqq->rb_key)) if (rb_key < __cfqq->rb_key)
p = &parent->rb_left; p = &parent->rb_left;
else { else {
p = &parent->rb_right; p = &parent->rb_right;
@ -2574,7 +2578,7 @@ cfq_merged_requests(struct request_queue *q, struct request *rq,
* reposition in fifo if next is older than rq * reposition in fifo if next is older than rq
*/ */
if (!list_empty(&rq->queuelist) && !list_empty(&next->queuelist) && if (!list_empty(&rq->queuelist) && !list_empty(&next->queuelist) &&
time_before(next->fifo_time, rq->fifo_time) && next->fifo_time < rq->fifo_time &&
cfqq == RQ_CFQQ(next)) { cfqq == RQ_CFQQ(next)) {
list_move(&rq->queuelist, &next->queuelist); list_move(&rq->queuelist, &next->queuelist);
rq->fifo_time = next->fifo_time; rq->fifo_time = next->fifo_time;
@ -2635,7 +2639,7 @@ static void __cfq_set_active_queue(struct cfq_data *cfqd,
cfqd->serving_wl_class, cfqd->serving_wl_type); cfqd->serving_wl_class, cfqd->serving_wl_type);
cfqg_stats_update_avg_queue_size(cfqq->cfqg); cfqg_stats_update_avg_queue_size(cfqq->cfqg);
cfqq->slice_start = 0; cfqq->slice_start = 0;
cfqq->dispatch_start = jiffies; cfqq->dispatch_start = ktime_get_ns();
cfqq->allocated_slice = 0; cfqq->allocated_slice = 0;
cfqq->slice_end = 0; cfqq->slice_end = 0;
cfqq->slice_dispatch = 0; cfqq->slice_dispatch = 0;
@ -2684,8 +2688,8 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
if (cfq_cfqq_slice_new(cfqq)) if (cfq_cfqq_slice_new(cfqq))
cfqq->slice_resid = cfq_scaled_cfqq_slice(cfqd, cfqq); cfqq->slice_resid = cfq_scaled_cfqq_slice(cfqd, cfqq);
else else
cfqq->slice_resid = cfqq->slice_end - jiffies; cfqq->slice_resid = cfqq->slice_end - ktime_get_ns();
cfq_log_cfqq(cfqd, cfqq, "resid=%ld", cfqq->slice_resid); cfq_log_cfqq(cfqd, cfqq, "resid=%llu", cfqq->slice_resid);
} }
cfq_group_served(cfqd, cfqq->cfqg, cfqq); cfq_group_served(cfqd, cfqq->cfqg, cfqq);
@ -2919,7 +2923,8 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd)
struct cfq_queue *cfqq = cfqd->active_queue; struct cfq_queue *cfqq = cfqd->active_queue;
struct cfq_rb_root *st = cfqq->service_tree; struct cfq_rb_root *st = cfqq->service_tree;
struct cfq_io_cq *cic; struct cfq_io_cq *cic;
unsigned long sl, group_idle = 0; u64 sl, group_idle = 0;
u64 now = ktime_get_ns();
/* /*
* SSD device without seek penalty, disable idling. But only do so * SSD device without seek penalty, disable idling. But only do so
@ -2962,8 +2967,8 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd)
* time slice. * time slice.
*/ */
if (sample_valid(cic->ttime.ttime_samples) && if (sample_valid(cic->ttime.ttime_samples) &&
(cfqq->slice_end - jiffies < cic->ttime.ttime_mean)) { (cfqq->slice_end - now < cic->ttime.ttime_mean)) {
cfq_log_cfqq(cfqd, cfqq, "Not idling. think_time:%lu", cfq_log_cfqq(cfqd, cfqq, "Not idling. think_time:%llu",
cic->ttime.ttime_mean); cic->ttime.ttime_mean);
return; return;
} }
@ -2984,9 +2989,9 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd)
else else
sl = cfqd->cfq_slice_idle; sl = cfqd->cfq_slice_idle;
mod_timer(&cfqd->idle_slice_timer, jiffies + sl); mod_timer(&cfqd->idle_slice_timer, now + sl);
cfqg_stats_set_start_idle_time(cfqq->cfqg); cfqg_stats_set_start_idle_time(cfqq->cfqg);
cfq_log_cfqq(cfqd, cfqq, "arm_idle: %lu group_idle: %d", sl, cfq_log_cfqq(cfqd, cfqq, "arm_idle: %llu group_idle: %d", sl,
group_idle ? 1 : 0); group_idle ? 1 : 0);
} }
@ -3026,7 +3031,7 @@ static struct request *cfq_check_fifo(struct cfq_queue *cfqq)
return NULL; return NULL;
rq = rq_entry_fifo(cfqq->fifo.next); rq = rq_entry_fifo(cfqq->fifo.next);
if (time_before(jiffies, rq->fifo_time)) if (ktime_get_ns() < rq->fifo_time)
rq = NULL; rq = NULL;
cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq); cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq);
@ -3104,14 +3109,14 @@ static enum wl_type_t cfq_choose_wl_type(struct cfq_data *cfqd,
struct cfq_queue *queue; struct cfq_queue *queue;
int i; int i;
bool key_valid = false; bool key_valid = false;
unsigned long lowest_key = 0; u64 lowest_key = 0;
enum wl_type_t cur_best = SYNC_NOIDLE_WORKLOAD; enum wl_type_t cur_best = SYNC_NOIDLE_WORKLOAD;
for (i = 0; i <= SYNC_WORKLOAD; ++i) { for (i = 0; i <= SYNC_WORKLOAD; ++i) {
/* select the one with lowest rb_key */ /* select the one with lowest rb_key */
queue = cfq_rb_first(st_for(cfqg, wl_class, i)); queue = cfq_rb_first(st_for(cfqg, wl_class, i));
if (queue && if (queue &&
(!key_valid || time_before(queue->rb_key, lowest_key))) { (!key_valid || queue->rb_key < lowest_key)) {
lowest_key = queue->rb_key; lowest_key = queue->rb_key;
cur_best = i; cur_best = i;
key_valid = true; key_valid = true;
@ -3124,11 +3129,12 @@ static enum wl_type_t cfq_choose_wl_type(struct cfq_data *cfqd,
static void static void
choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg) choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg)
{ {
unsigned slice; u64 slice;
unsigned count; unsigned count;
struct cfq_rb_root *st; struct cfq_rb_root *st;
unsigned group_slice; u64 group_slice;
enum wl_class_t original_class = cfqd->serving_wl_class; enum wl_class_t original_class = cfqd->serving_wl_class;
u64 now = ktime_get_ns();
/* Choose next priority. RT > BE > IDLE */ /* Choose next priority. RT > BE > IDLE */
if (cfq_group_busy_queues_wl(RT_WORKLOAD, cfqd, cfqg)) if (cfq_group_busy_queues_wl(RT_WORKLOAD, cfqd, cfqg))
@ -3137,7 +3143,7 @@ choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg)
cfqd->serving_wl_class = BE_WORKLOAD; cfqd->serving_wl_class = BE_WORKLOAD;
else { else {
cfqd->serving_wl_class = IDLE_WORKLOAD; cfqd->serving_wl_class = IDLE_WORKLOAD;
cfqd->workload_expires = jiffies + 1; cfqd->workload_expires = now + jiffies_to_nsecs(1);
return; return;
} }
@ -3155,7 +3161,7 @@ choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg)
/* /*
* check workload expiration, and that we still have other queues ready * check workload expiration, and that we still have other queues ready
*/ */
if (count && !time_after(jiffies, cfqd->workload_expires)) if (count && !(now > cfqd->workload_expires))
return; return;
new_workload: new_workload:
@ -3172,13 +3178,13 @@ choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg)
*/ */
group_slice = cfq_group_slice(cfqd, cfqg); group_slice = cfq_group_slice(cfqd, cfqg);
slice = group_slice * count / slice = div_u64(group_slice * count,
max_t(unsigned, cfqg->busy_queues_avg[cfqd->serving_wl_class], max_t(unsigned, cfqg->busy_queues_avg[cfqd->serving_wl_class],
cfq_group_busy_queues_wl(cfqd->serving_wl_class, cfqd, cfq_group_busy_queues_wl(cfqd->serving_wl_class, cfqd,
cfqg)); cfqg)));
if (cfqd->serving_wl_type == ASYNC_WORKLOAD) { if (cfqd->serving_wl_type == ASYNC_WORKLOAD) {
unsigned int tmp; u64 tmp;
/* /*
* Async queues are currently system wide. Just taking * Async queues are currently system wide. Just taking
@ -3189,19 +3195,19 @@ choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg)
*/ */
tmp = cfqd->cfq_target_latency * tmp = cfqd->cfq_target_latency *
cfqg_busy_async_queues(cfqd, cfqg); cfqg_busy_async_queues(cfqd, cfqg);
tmp = tmp/cfqd->busy_queues; tmp = div_u64(tmp, cfqd->busy_queues);
slice = min_t(unsigned, slice, tmp); slice = min_t(u64, slice, tmp);
/* async workload slice is scaled down according to /* async workload slice is scaled down according to
* the sync/async slice ratio. */ * the sync/async slice ratio. */
slice = slice * cfqd->cfq_slice[0] / cfqd->cfq_slice[1]; slice = div64_u64(slice*cfqd->cfq_slice[0], cfqd->cfq_slice[1]);
} else } else
/* sync workload slice is at least 2 * cfq_slice_idle */ /* sync workload slice is at least 2 * cfq_slice_idle */
slice = max(slice, 2 * cfqd->cfq_slice_idle); slice = max(slice, 2 * cfqd->cfq_slice_idle);
slice = max_t(unsigned, slice, CFQ_MIN_TT); slice = max_t(u64, slice, CFQ_MIN_TT);
cfq_log(cfqd, "workload slice:%d", slice); cfq_log(cfqd, "workload slice:%llu", slice);
cfqd->workload_expires = jiffies + slice; cfqd->workload_expires = now + slice;
} }
static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd) static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd)
@ -3219,16 +3225,17 @@ static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd)
static void cfq_choose_cfqg(struct cfq_data *cfqd) static void cfq_choose_cfqg(struct cfq_data *cfqd)
{ {
struct cfq_group *cfqg = cfq_get_next_cfqg(cfqd); struct cfq_group *cfqg = cfq_get_next_cfqg(cfqd);
u64 now = ktime_get_ns();
cfqd->serving_group = cfqg; cfqd->serving_group = cfqg;
/* Restore the workload type data */ /* Restore the workload type data */
if (cfqg->saved_wl_slice) { if (cfqg->saved_wl_slice) {
cfqd->workload_expires = jiffies + cfqg->saved_wl_slice; cfqd->workload_expires = now + cfqg->saved_wl_slice;
cfqd->serving_wl_type = cfqg->saved_wl_type; cfqd->serving_wl_type = cfqg->saved_wl_type;
cfqd->serving_wl_class = cfqg->saved_wl_class; cfqd->serving_wl_class = cfqg->saved_wl_class;
} else } else
cfqd->workload_expires = jiffies - 1; cfqd->workload_expires = now - 1;
choose_wl_class_and_type(cfqd, cfqg); choose_wl_class_and_type(cfqd, cfqg);
} }
@ -3240,6 +3247,7 @@ static void cfq_choose_cfqg(struct cfq_data *cfqd)
static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd) static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)
{ {
struct cfq_queue *cfqq, *new_cfqq = NULL; struct cfq_queue *cfqq, *new_cfqq = NULL;
u64 now = ktime_get_ns();
cfqq = cfqd->active_queue; cfqq = cfqd->active_queue;
if (!cfqq) if (!cfqq)
@ -3311,7 +3319,7 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)
**/ **/
if (CFQQ_SEEKY(cfqq) && cfq_cfqq_idle_window(cfqq) && if (CFQQ_SEEKY(cfqq) && cfq_cfqq_idle_window(cfqq) &&
(cfq_cfqq_slice_new(cfqq) || (cfq_cfqq_slice_new(cfqq) ||
(cfqq->slice_end - jiffies > jiffies - cfqq->slice_start))) { (cfqq->slice_end - now > now - cfqq->slice_start))) {
cfq_clear_cfqq_deep(cfqq); cfq_clear_cfqq_deep(cfqq);
cfq_clear_cfqq_idle_window(cfqq); cfq_clear_cfqq_idle_window(cfqq);
} }
@ -3389,11 +3397,12 @@ static int cfq_forced_dispatch(struct cfq_data *cfqd)
static inline bool cfq_slice_used_soon(struct cfq_data *cfqd, static inline bool cfq_slice_used_soon(struct cfq_data *cfqd,
struct cfq_queue *cfqq) struct cfq_queue *cfqq)
{ {
u64 now = ktime_get_ns();
/* the queue hasn't finished any request, can't estimate */ /* the queue hasn't finished any request, can't estimate */
if (cfq_cfqq_slice_new(cfqq)) if (cfq_cfqq_slice_new(cfqq))
return true; return true;
if (time_after(jiffies + cfqd->cfq_slice_idle * cfqq->dispatched, if (now + cfqd->cfq_slice_idle * cfqq->dispatched > cfqq->slice_end)
cfqq->slice_end))
return true; return true;
return false; return false;
@ -3468,10 +3477,10 @@ static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq)
* based on the last sync IO we serviced * based on the last sync IO we serviced
*/ */
if (!cfq_cfqq_sync(cfqq) && cfqd->cfq_latency) { if (!cfq_cfqq_sync(cfqq) && cfqd->cfq_latency) {
unsigned long last_sync = jiffies - cfqd->last_delayed_sync; u64 last_sync = ktime_get_ns() - cfqd->last_delayed_sync;
unsigned int depth; unsigned int depth;
depth = last_sync / cfqd->cfq_slice[1]; depth = div64_u64(last_sync, cfqd->cfq_slice[1]);
if (!depth && !cfqq->dispatched) if (!depth && !cfqq->dispatched)
depth = 1; depth = 1;
if (depth < max_dispatch) if (depth < max_dispatch)
@ -3554,7 +3563,7 @@ static int cfq_dispatch_requests(struct request_queue *q, int force)
if (cfqd->busy_queues > 1 && ((!cfq_cfqq_sync(cfqq) && if (cfqd->busy_queues > 1 && ((!cfq_cfqq_sync(cfqq) &&
cfqq->slice_dispatch >= cfq_prio_to_maxrq(cfqd, cfqq)) || cfqq->slice_dispatch >= cfq_prio_to_maxrq(cfqd, cfqq)) ||
cfq_class_idle(cfqq))) { cfq_class_idle(cfqq))) {
cfqq->slice_end = jiffies + 1; cfqq->slice_end = ktime_get_ns() + 1;
cfq_slice_expired(cfqd, 0); cfq_slice_expired(cfqd, 0);
} }
@ -3632,7 +3641,7 @@ static void cfq_init_icq(struct io_cq *icq)
{ {
struct cfq_io_cq *cic = icq_to_cic(icq); struct cfq_io_cq *cic = icq_to_cic(icq);
cic->ttime.last_end_request = jiffies; cic->ttime.last_end_request = ktime_get_ns();
} }
static void cfq_exit_icq(struct io_cq *icq) static void cfq_exit_icq(struct io_cq *icq)
@ -3853,14 +3862,15 @@ cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic,
} }
static void static void
__cfq_update_io_thinktime(struct cfq_ttime *ttime, unsigned long slice_idle) __cfq_update_io_thinktime(struct cfq_ttime *ttime, u64 slice_idle)
{ {
unsigned long elapsed = jiffies - ttime->last_end_request; u64 elapsed = ktime_get_ns() - ttime->last_end_request;
elapsed = min(elapsed, 2UL * slice_idle); elapsed = min(elapsed, 2UL * slice_idle);
ttime->ttime_samples = (7*ttime->ttime_samples + 256) / 8; ttime->ttime_samples = (7*ttime->ttime_samples + 256) / 8;
ttime->ttime_total = (7*ttime->ttime_total + 256*elapsed) / 8; ttime->ttime_total = div_u64(7*ttime->ttime_total + 256*elapsed, 8);
ttime->ttime_mean = (ttime->ttime_total + 128) / ttime->ttime_samples; ttime->ttime_mean = div64_ul(ttime->ttime_total + 128,
ttime->ttime_samples);
} }
static void static void
@ -4113,7 +4123,7 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq)
cfq_log_cfqq(cfqd, cfqq, "insert_request"); cfq_log_cfqq(cfqd, cfqq, "insert_request");
cfq_init_prio_data(cfqq, RQ_CIC(rq)); cfq_init_prio_data(cfqq, RQ_CIC(rq));
rq->fifo_time = jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)]; rq->fifo_time = ktime_get_ns() + cfqd->cfq_fifo_expire[rq_is_sync(rq)];
list_add_tail(&rq->queuelist, &cfqq->fifo); list_add_tail(&rq->queuelist, &cfqq->fifo);
cfq_add_rq_rb(rq); cfq_add_rq_rb(rq);
cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group, req_op(rq), cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group, req_op(rq),
@ -4161,6 +4171,7 @@ static void cfq_update_hw_tag(struct cfq_data *cfqd)
static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq) static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq)
{ {
struct cfq_io_cq *cic = cfqd->active_cic; struct cfq_io_cq *cic = cfqd->active_cic;
u64 now = ktime_get_ns();
/* If the queue already has requests, don't wait */ /* If the queue already has requests, don't wait */
if (!RB_EMPTY_ROOT(&cfqq->sort_list)) if (!RB_EMPTY_ROOT(&cfqq->sort_list))
@ -4179,7 +4190,7 @@ static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq)
/* if slice left is less than think time, wait busy */ /* if slice left is less than think time, wait busy */
if (cic && sample_valid(cic->ttime.ttime_samples) if (cic && sample_valid(cic->ttime.ttime_samples)
&& (cfqq->slice_end - jiffies < cic->ttime.ttime_mean)) && (cfqq->slice_end - now < cic->ttime.ttime_mean))
return true; return true;
/* /*
@ -4189,7 +4200,7 @@ static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq)
* case where think time is less than a jiffy, mark the queue wait * case where think time is less than a jiffy, mark the queue wait
* busy if only 1 jiffy is left in the slice. * busy if only 1 jiffy is left in the slice.
*/ */
if (cfqq->slice_end - jiffies == 1) if (cfqq->slice_end - now <= jiffies_to_nsecs(1))
return true; return true;
return false; return false;
@ -4200,9 +4211,8 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
struct cfq_queue *cfqq = RQ_CFQQ(rq); struct cfq_queue *cfqq = RQ_CFQQ(rq);
struct cfq_data *cfqd = cfqq->cfqd; struct cfq_data *cfqd = cfqq->cfqd;
const int sync = rq_is_sync(rq); const int sync = rq_is_sync(rq);
unsigned long now; u64 now = ktime_get_ns();
now = jiffies;
cfq_log_cfqq(cfqd, cfqq, "complete rqnoidle %d", cfq_log_cfqq(cfqd, cfqq, "complete rqnoidle %d",
!!(rq->cmd_flags & REQ_NOIDLE)); !!(rq->cmd_flags & REQ_NOIDLE));
@ -4231,7 +4241,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
cfqq_type(cfqq)); cfqq_type(cfqq));
st->ttime.last_end_request = now; st->ttime.last_end_request = now;
if (!time_after(rq->start_time + cfqd->cfq_fifo_expire[1], now)) if (!(rq->start_time + cfqd->cfq_fifo_expire[1] > now))
cfqd->last_delayed_sync = now; cfqd->last_delayed_sync = now;
} }
@ -4256,10 +4266,10 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
* the queue. * the queue.
*/ */
if (cfq_should_wait_busy(cfqd, cfqq)) { if (cfq_should_wait_busy(cfqd, cfqq)) {
unsigned long extend_sl = cfqd->cfq_slice_idle; u64 extend_sl = cfqd->cfq_slice_idle;
if (!cfqd->cfq_slice_idle) if (!cfqd->cfq_slice_idle)
extend_sl = cfqd->cfq_group_idle; extend_sl = cfqd->cfq_group_idle;
cfqq->slice_end = jiffies + extend_sl; cfqq->slice_end = now + extend_sl;
cfq_mark_cfqq_wait_busy(cfqq); cfq_mark_cfqq_wait_busy(cfqq);
cfq_log_cfqq(cfqd, cfqq, "will busy wait"); cfq_log_cfqq(cfqd, cfqq, "will busy wait");
} }
@ -4618,7 +4628,7 @@ static int cfq_init_queue(struct request_queue *q, struct elevator_type *e)
* we optimistically start assuming sync ops weren't delayed in last * we optimistically start assuming sync ops weren't delayed in last
* second, in order to have larger depth for async operations. * second, in order to have larger depth for async operations.
*/ */
cfqd->last_delayed_sync = jiffies - HZ; cfqd->last_delayed_sync = ktime_get_ns() - NSEC_PER_SEC;
return 0; return 0;
out_free: out_free:
@ -4661,9 +4671,9 @@ cfq_var_store(unsigned int *var, const char *page, size_t count)
static ssize_t __FUNC(struct elevator_queue *e, char *page) \ static ssize_t __FUNC(struct elevator_queue *e, char *page) \
{ \ { \
struct cfq_data *cfqd = e->elevator_data; \ struct cfq_data *cfqd = e->elevator_data; \
unsigned int __data = __VAR; \ u64 __data = __VAR; \
if (__CONV) \ if (__CONV) \
__data = jiffies_to_msecs(__data); \ __data = div_u64(__data, NSEC_PER_MSEC); \
return cfq_var_show(__data, (page)); \ return cfq_var_show(__data, (page)); \
} }
SHOW_FUNCTION(cfq_quantum_show, cfqd->cfq_quantum, 0); SHOW_FUNCTION(cfq_quantum_show, cfqd->cfq_quantum, 0);
@ -4691,7 +4701,7 @@ static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count)
else if (__data > (MAX)) \ else if (__data > (MAX)) \
__data = (MAX); \ __data = (MAX); \
if (__CONV) \ if (__CONV) \
*(__PTR) = msecs_to_jiffies(__data); \ *(__PTR) = (u64)__data * NSEC_PER_MSEC; \
else \ else \
*(__PTR) = __data; \ *(__PTR) = __data; \
return ret; \ return ret; \
@ -4785,18 +4795,7 @@ static int __init cfq_init(void)
{ {
int ret; int ret;
/*
* could be 0 on HZ < 1000 setups
*/
if (!cfq_slice_async)
cfq_slice_async = 1;
if (!cfq_slice_idle)
cfq_slice_idle = 1;
#ifdef CONFIG_CFQ_GROUP_IOSCHED #ifdef CONFIG_CFQ_GROUP_IOSCHED
if (!cfq_group_idle)
cfq_group_idle = 1;
ret = blkcg_policy_register(&blkcg_policy_cfq); ret = blkcg_policy_register(&blkcg_policy_cfq);
if (ret) if (ret)
return ret; return ret;