diff options
| -rw-r--r-- | include/linux/workqueue.h | 14 | ||||
| -rw-r--r-- | kernel/workqueue.c | 36 |
2 files changed, 21 insertions, 29 deletions
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index d74a529ed13e..5f76001c4e6d 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -25,17 +25,19 @@ typedef void (*work_func_t)(struct work_struct *work); enum { WORK_STRUCT_PENDING_BIT = 0, /* work item is pending execution */ - WORK_STRUCT_LINKED_BIT = 1, /* next work is linked to this one */ + WORK_STRUCT_CWQ_BIT = 1, /* data points to cwq */ + WORK_STRUCT_LINKED_BIT = 2, /* next work is linked to this one */ #ifdef CONFIG_DEBUG_OBJECTS_WORK - WORK_STRUCT_STATIC_BIT = 2, /* static initializer (debugobjects) */ - WORK_STRUCT_COLOR_SHIFT = 3, /* color for workqueue flushing */ + WORK_STRUCT_STATIC_BIT = 3, /* static initializer (debugobjects) */ + WORK_STRUCT_COLOR_SHIFT = 4, /* color for workqueue flushing */ #else - WORK_STRUCT_COLOR_SHIFT = 2, /* color for workqueue flushing */ + WORK_STRUCT_COLOR_SHIFT = 3, /* color for workqueue flushing */ #endif WORK_STRUCT_COLOR_BITS = 4, WORK_STRUCT_PENDING = 1 << WORK_STRUCT_PENDING_BIT, + WORK_STRUCT_CWQ = 1 << WORK_STRUCT_CWQ_BIT, WORK_STRUCT_LINKED = 1 << WORK_STRUCT_LINKED_BIT, #ifdef CONFIG_DEBUG_OBJECTS_WORK WORK_STRUCT_STATIC = 1 << WORK_STRUCT_STATIC_BIT, @@ -56,8 +58,8 @@ enum { WORK_CPU_LAST = WORK_CPU_NONE, /* - * Reserve 6 bits off of cwq pointer w/ debugobjects turned - * off. This makes cwqs aligned to 64 bytes which isn't too + * Reserve 7 bits off of cwq pointer w/ debugobjects turned + * off. This makes cwqs aligned to 128 bytes which isn't too * excessive while allowing 15 workqueue flush colors. */ WORK_STRUCT_FLAG_BITS = WORK_STRUCT_COLOR_SHIFT + diff --git a/kernel/workqueue.c b/kernel/workqueue.c index c11edc9c9365..e5cb7faac58e 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -468,10 +468,9 @@ static int work_next_color(int color) } /* - * Work data points to the cwq while a work is on queue. Once - * execution starts, it points to the cpu the work was last on. This - * can be distinguished by comparing the data value against - * PAGE_OFFSET. + * A work's data points to the cwq with WORK_STRUCT_CWQ set while the + * work is on queue. Once execution starts, WORK_STRUCT_CWQ is + * cleared and the work data contains the cpu number it was last on. * * set_work_{cwq|cpu}() and clear_work_data() can be used to set the * cwq, cpu or clear work->data. These functions should only be @@ -494,7 +493,7 @@ static void set_work_cwq(struct work_struct *work, unsigned long extra_flags) { set_work_data(work, (unsigned long)cwq, - WORK_STRUCT_PENDING | extra_flags); + WORK_STRUCT_PENDING | WORK_STRUCT_CWQ | extra_flags); } static void set_work_cpu(struct work_struct *work, unsigned int cpu) @@ -507,25 +506,24 @@ static void clear_work_data(struct work_struct *work) set_work_data(work, WORK_STRUCT_NO_CPU, 0); } -static inline unsigned long get_work_data(struct work_struct *work) -{ - return atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK; -} - static struct cpu_workqueue_struct *get_work_cwq(struct work_struct *work) { - unsigned long data = get_work_data(work); + unsigned long data = atomic_long_read(&work->data); - return data >= PAGE_OFFSET ? (void *)data : NULL; + if (data & WORK_STRUCT_CWQ) + return (void *)(data & WORK_STRUCT_WQ_DATA_MASK); + else + return NULL; } static struct global_cwq *get_work_gcwq(struct work_struct *work) { - unsigned long data = get_work_data(work); + unsigned long data = atomic_long_read(&work->data); unsigned int cpu; - if (data >= PAGE_OFFSET) - return ((struct cpu_workqueue_struct *)data)->gcwq; + if (data & WORK_STRUCT_CWQ) + return ((struct cpu_workqueue_struct *) + (data & WORK_STRUCT_WQ_DATA_MASK))->gcwq; cpu = data >> WORK_STRUCT_FLAG_BITS; if (cpu == WORK_CPU_NONE) @@ -3501,14 +3499,6 @@ void __init init_workqueues(void) unsigned int cpu; int i; - /* - * The pointer part of work->data is either pointing to the - * cwq or contains the cpu number the work ran last on. Make - * sure cpu number won't overflow into kernel pointer area so - * that they can be distinguished. - */ - BUILD_BUG_ON(WORK_CPU_LAST << WORK_STRUCT_FLAG_BITS >= PAGE_OFFSET); - hotcpu_notifier(workqueue_cpu_callback, CPU_PRI_WORKQUEUE); /* initialize gcwqs */ |