blob: c8496f98b1391a48b545a766c1ebff4ed34fc6d7 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/ipc/sem.c
4 * Copyright (C) 1992 Krishna Balasubramanian
5 * Copyright (C) 1995 Eric Schenk, Bruno Haible
6 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * /proc/sysvipc/sem support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
8 *
9 * SMP-threaded, sysctl's added
Christian Kujau624dffc2006-01-15 02:43:54 +010010 * (c) 1999 Manfred Spraul <manfred@colorfullife.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Enforced range limit on SEM_UNDO
Alan Cox046c6882009-01-05 14:06:29 +000012 * (c) 2001 Red Hat Inc
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 * Lockless wakeup
14 * (c) 2003 Manfred Spraul <manfred@colorfullife.com>
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -080015 * (c) 2016 Davidlohr Bueso <dave@stgolabs.net>
Manfred Spraulc5cf6352010-05-26 14:43:43 -070016 * Further wakeup optimizations, documentation
17 * (c) 2010 Manfred Spraul <manfred@colorfullife.com>
Steve Grubb073115d2006-04-02 17:07:33 -040018 *
19 * support for audit of ipc object properties and permission changes
20 * Dustin Kirkland <dustin.kirkland@us.ibm.com>
Kirill Korotaeve3893532006-10-02 02:18:22 -070021 *
22 * namespaces support
23 * OpenVZ, SWsoft Inc.
24 * Pavel Emelianov <xemul@openvz.org>
Manfred Spraulc5cf6352010-05-26 14:43:43 -070025 *
26 * Implementation notes: (May 2010)
27 * This file implements System V semaphores.
28 *
29 * User space visible behavior:
30 * - FIFO ordering for semop() operations (just FIFO, not starvation
31 * protection)
32 * - multiple semaphore operations that alter the same semaphore in
33 * one semop() are handled.
34 * - sem_ctime (time of last semctl()) is updated in the IPC_SET, SETVAL and
35 * SETALL calls.
36 * - two Linux specific semctl() commands: SEM_STAT, SEM_INFO.
37 * - undo adjustments at process exit are limited to 0..SEMVMX.
38 * - namespace are supported.
Bhaskar Chowdhuryb1989a32021-05-06 18:05:48 -070039 * - SEMMSL, SEMMNS, SEMOPM and SEMMNI can be configured at runtime by writing
Manfred Spraulc5cf6352010-05-26 14:43:43 -070040 * to /proc/sys/kernel/sem.
41 * - statistics about the usage are reported in /proc/sysvipc/sem.
42 *
43 * Internals:
44 * - scalability:
45 * - all global variables are read-mostly.
46 * - semop() calls and semctl(RMID) are synchronized by RCU.
47 * - most operations do write operations (actually: spin_lock calls) to
48 * the per-semaphore array structure.
49 * Thus: Perfect SMP scaling between independent semaphore arrays.
50 * If multiple semaphores in one array are used, then cache line
51 * trashing on the semaphore array spinlock will limit the scaling.
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -070052 * - semncnt and semzcnt are calculated on demand in count_semcnt()
Manfred Spraulc5cf6352010-05-26 14:43:43 -070053 * - the task that performs a successful semop() scans the list of all
54 * sleeping tasks and completes any pending operations that can be fulfilled.
55 * Semaphores are actively given to waiting tasks (necessary for FIFO).
56 * (see update_queue())
57 * - To improve the scalability, the actual wake-up calls are performed after
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -080058 * dropping all locks. (see wake_up_sem_queue_prepare())
Manfred Spraulc5cf6352010-05-26 14:43:43 -070059 * - All work is done by the waker, the woken up task does not have to do
60 * anything - not even acquiring a lock or dropping a refcount.
61 * - A woken up task may not even touch the semaphore array anymore, it may
62 * have been destroyed already by a semctl(RMID).
Manfred Spraulc5cf6352010-05-26 14:43:43 -070063 * - UNDO values are stored in an array (one per process and per
64 * semaphore array, lazily allocated). For backwards compatibility, multiple
65 * modes for the UNDO variables are supported (per process, per thread)
66 * (see copy_semundo, CLONE_SYSVSEM)
67 * - There are two lists of the pending operations: a per-array list
68 * and per-semaphore list (stored in the array). This allows to achieve FIFO
69 * ordering without always scanning all pending operations.
70 * The worst-case behavior is nevertheless O(N^2) for N wakeups.
Linus Torvalds1da177e2005-04-16 15:20:36 -070071 */
72
Arnd Bergmannb0d17572018-04-13 13:58:23 +020073#include <linux/compat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070074#include <linux/slab.h>
75#include <linux/spinlock.h>
76#include <linux/init.h>
77#include <linux/proc_fs.h>
78#include <linux/time.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070079#include <linux/security.h>
80#include <linux/syscalls.h>
81#include <linux/audit.h>
Randy.Dunlapc59ede72006-01-11 12:17:46 -080082#include <linux/capability.h>
Mike Waychison19b49462005-09-06 15:17:10 -070083#include <linux/seq_file.h>
Nadia Derbey3e148c72007-10-18 23:40:54 -070084#include <linux/rwsem.h>
Kirill Korotaeve3893532006-10-02 02:18:22 -070085#include <linux/nsproxy.h>
Pavel Emelyanovae5e1b22008-02-08 04:18:22 -080086#include <linux/ipc_namespace.h>
Ingo Molnar84f001e2017-02-01 16:36:40 +010087#include <linux/sched/wake_q.h>
Davidlohr Buesoec67aaa2018-06-14 15:27:51 -070088#include <linux/nospec.h>
NeilBrown0eb71a92018-06-18 12:52:50 +100089#include <linux/rhashtable.h>
Ingo Molnar5f921ae2006-03-26 01:37:17 -080090
Paul McQuade7153e402014-06-06 14:37:37 -070091#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070092#include "util.h"
93
Eric W. Biederman1a5c1342018-03-22 21:30:56 -050094/* One semaphore structure for each semaphore in the system. */
95struct sem {
96 int semval; /* current value */
97 /*
98 * PID of the process that last modified the semaphore. For
99 * Linux, specifically these are:
100 * - semop
101 * - semctl, via SETVAL and SETALL.
102 * - at task exit when performing undo adjustments (see exit_sem).
103 */
Eric W. Biederman51d6f262018-03-23 01:11:29 -0500104 struct pid *sempid;
Eric W. Biederman1a5c1342018-03-22 21:30:56 -0500105 spinlock_t lock; /* spinlock for fine-grained semtimedop */
106 struct list_head pending_alter; /* pending single-sop operations */
107 /* that alter the semaphore */
108 struct list_head pending_const; /* pending single-sop operations */
109 /* that do not alter the semaphore*/
Arnd Bergmann2a70b782018-04-12 15:19:44 +0200110 time64_t sem_otime; /* candidate for sem_otime */
Eric W. Biederman1a5c1342018-03-22 21:30:56 -0500111} ____cacheline_aligned_in_smp;
112
113/* One sem_array data structure for each set of semaphores in the system. */
114struct sem_array {
115 struct kern_ipc_perm sem_perm; /* permissions .. see ipc.h */
116 time64_t sem_ctime; /* create/last semctl() time */
117 struct list_head pending_alter; /* pending operations */
118 /* that alter the array */
119 struct list_head pending_const; /* pending complex operations */
120 /* that do not alter semvals */
121 struct list_head list_id; /* undo requests on this array */
122 int sem_nsems; /* no. of semaphores in array */
123 int complex_count; /* pending complex operations */
124 unsigned int use_global_lock;/* >0: global lock required */
125
126 struct sem sems[];
127} __randomize_layout;
Manfred Spraule57940d2011-11-02 13:38:54 -0700128
129/* One queue for each sleeping process in the system. */
130struct sem_queue {
Manfred Spraule57940d2011-11-02 13:38:54 -0700131 struct list_head list; /* queue of pending operations */
132 struct task_struct *sleeper; /* this process */
133 struct sem_undo *undo; /* undo structure */
Eric W. Biederman51d6f262018-03-23 01:11:29 -0500134 struct pid *pid; /* process id of requesting process */
Manfred Spraule57940d2011-11-02 13:38:54 -0700135 int status; /* completion status of operation */
136 struct sembuf *sops; /* array of pending operations */
Manfred Sprauled247b72014-06-06 14:37:49 -0700137 struct sembuf *blocking; /* the operation that blocked */
Manfred Spraule57940d2011-11-02 13:38:54 -0700138 int nsops; /* number of operations */
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -0800139 bool alter; /* does *sops alter the array? */
140 bool dupsop; /* sops on more than one sem_num */
Manfred Spraule57940d2011-11-02 13:38:54 -0700141};
142
143/* Each task has a list of undo requests. They are executed automatically
144 * when the process exits.
145 */
146struct sem_undo {
147 struct list_head list_proc; /* per-process list: *
148 * all undos from one process
149 * rcu protected */
150 struct rcu_head rcu; /* rcu struct for sem_undo */
151 struct sem_undo_list *ulp; /* back ptr to sem_undo_list */
152 struct list_head list_id; /* per semaphore array list:
153 * all undos for one array */
154 int semid; /* semaphore set identifier */
155 short *semadj; /* array of adjustments */
156 /* one per semaphore */
157};
158
159/* sem_undo_list controls shared access to the list of sem_undo structures
160 * that may be shared among all a CLONE_SYSVSEM task group.
161 */
162struct sem_undo_list {
Elena Reshetovaf74370b2017-09-08 16:17:42 -0700163 refcount_t refcnt;
Manfred Spraule57940d2011-11-02 13:38:54 -0700164 spinlock_t lock;
165 struct list_head list_proc;
166};
167
168
Pierre Peiffered2ddbf2008-02-08 04:18:57 -0800169#define sem_ids(ns) ((ns)->ids[IPC_SEM_IDS])
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700171static int newary(struct ipc_namespace *, struct ipc_params *);
Pierre Peiffer01b8b072008-02-08 04:18:57 -0800172static void freeary(struct ipc_namespace *, struct kern_ipc_perm *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173#ifdef CONFIG_PROC_FS
Mike Waychison19b49462005-09-06 15:17:10 -0700174static int sysvipc_sem_proc_show(struct seq_file *s, void *it);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175#endif
176
177#define SEMMSL_FAST 256 /* 512 bytes on stack */
178#define SEMOPM_FAST 64 /* ~ 372 bytes on stack */
179
180/*
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800181 * Switching from the mode suitable for simple ops
182 * to the mode for complex ops is costly. Therefore:
183 * use some hysteresis
184 */
185#define USE_GLOBAL_LOCK_HYSTERESIS 10
186
187/*
Manfred Spraul758a6ba32013-07-08 16:01:26 -0700188 * Locking:
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700189 * a) global sem_lock() for read/write
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 * sem_undo.id_next,
Manfred Spraul758a6ba32013-07-08 16:01:26 -0700191 * sem_array.complex_count,
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700192 * sem_array.pending{_alter,_const},
193 * sem_array.sem_undo
Paul McQuade46c0a8c2014-06-06 14:37:37 -0700194 *
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700195 * b) global or semaphore sem_lock() for read/write:
Manfred Spraul1a233952017-07-12 14:34:38 -0700196 * sem_array.sems[i].pending_{const,alter}:
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700197 *
198 * c) special:
199 * sem_undo_list.list_proc:
200 * * undo_list->lock for write
201 * * rcu for read
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800202 * use_global_lock:
203 * * global sem_lock() for write
204 * * either local or global sem_lock() for read.
205 *
206 * Memory ordering:
207 * Most ordering is enforced by using spin_lock() and spin_unlock().
Manfred Spraul8116b542020-02-03 17:34:42 -0800208 *
209 * Exceptions:
210 * 1) use_global_lock: (SEM_BARRIER_1)
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800211 * Setting it from non-zero to 0 is a RELEASE, this is ensured by
Manfred Spraul8116b542020-02-03 17:34:42 -0800212 * using smp_store_release(): Immediately after setting it to 0,
213 * a simple op can start.
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800214 * Testing if it is non-zero is an ACQUIRE, this is ensured by using
215 * smp_load_acquire().
216 * Setting it from 0 to non-zero must be ordered with regards to
217 * this smp_load_acquire(), this is guaranteed because the smp_load_acquire()
218 * is inside a spin_lock() and after a write from 0 to non-zero a
219 * spin_lock()+spin_unlock() is done.
Manfred Spraul17d056e2021-06-30 18:57:15 -0700220 * To prevent the compiler/cpu temporarily writing 0 to use_global_lock,
221 * READ_ONCE()/WRITE_ONCE() is used.
Manfred Spraul8116b542020-02-03 17:34:42 -0800222 *
223 * 2) queue.status: (SEM_BARRIER_2)
224 * Initialization is done while holding sem_lock(), so no further barrier is
225 * required.
226 * Setting it to a result code is a RELEASE, this is ensured by both a
227 * smp_store_release() (for case a) and while holding sem_lock()
228 * (for case b).
Bhaskar Chowdhuryb1989a32021-05-06 18:05:48 -0700229 * The ACQUIRE when reading the result code without holding sem_lock() is
Manfred Spraul8116b542020-02-03 17:34:42 -0800230 * achieved by using READ_ONCE() + smp_acquire__after_ctrl_dep().
231 * (case a above).
232 * Reading the result code while holding sem_lock() needs no further barriers,
233 * the locks inside sem_lock() enforce ordering (case b above)
234 *
235 * 3) current->state:
236 * current->state is set to TASK_INTERRUPTIBLE while holding sem_lock().
237 * The wakeup is handled using the wake_q infrastructure. wake_q wakeups may
238 * happen immediately after calling wake_q_add. As wake_q_add_safe() is called
239 * when holding sem_lock(), no further barriers are required.
240 *
241 * See also ipc/mqueue.c for more details on the covered races.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242 */
243
Kirill Korotaeve3893532006-10-02 02:18:22 -0700244#define sc_semmsl sem_ctls[0]
245#define sc_semmns sem_ctls[1]
246#define sc_semopm sem_ctls[2]
247#define sc_semmni sem_ctls[3]
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248
Davidlohr Buesoeae04d22018-08-21 22:01:56 -0700249void sem_init_ns(struct ipc_namespace *ns)
Kirill Korotaeve3893532006-10-02 02:18:22 -0700250{
Kirill Korotaeve3893532006-10-02 02:18:22 -0700251 ns->sc_semmsl = SEMMSL;
252 ns->sc_semmns = SEMMNS;
253 ns->sc_semopm = SEMOPM;
254 ns->sc_semmni = SEMMNI;
255 ns->used_sems = 0;
Davidlohr Buesoeae04d22018-08-21 22:01:56 -0700256 ipc_init_ids(&ns->ids[IPC_SEM_IDS]);
Kirill Korotaeve3893532006-10-02 02:18:22 -0700257}
258
Pavel Emelyanovae5e1b22008-02-08 04:18:22 -0800259#ifdef CONFIG_IPC_NS
Kirill Korotaeve3893532006-10-02 02:18:22 -0700260void sem_exit_ns(struct ipc_namespace *ns)
261{
Pierre Peiffer01b8b072008-02-08 04:18:57 -0800262 free_ipcs(ns, &sem_ids(ns), freeary);
Serge E. Hallyn7d6feeb2009-12-15 16:47:27 -0800263 idr_destroy(&ns->ids[IPC_SEM_IDS].ipcs_idr);
Guillaume Knispel0cfb6ae2017-09-08 16:17:55 -0700264 rhashtable_destroy(&ns->ids[IPC_SEM_IDS].key_ht);
Kirill Korotaeve3893532006-10-02 02:18:22 -0700265}
Pavel Emelyanovae5e1b22008-02-08 04:18:22 -0800266#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267
Davidlohr Buesoeae04d22018-08-21 22:01:56 -0700268void __init sem_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269{
Davidlohr Buesoeae04d22018-08-21 22:01:56 -0700270 sem_init_ns(&init_ipc_ns);
Mike Waychison19b49462005-09-06 15:17:10 -0700271 ipc_init_proc_interface("sysvipc/sem",
272 " key semid perms nsems uid gid cuid cgid otime ctime\n",
Kirill Korotaeve3893532006-10-02 02:18:22 -0700273 IPC_SEM_IDS, sysvipc_sem_proc_show);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274}
275
Manfred Spraulf269f402013-07-08 16:01:24 -0700276/**
277 * unmerge_queues - unmerge queues, if possible.
278 * @sma: semaphore array
279 *
280 * The function unmerges the wait queues if complex_count is 0.
281 * It must be called prior to dropping the global semaphore array lock.
282 */
283static void unmerge_queues(struct sem_array *sma)
284{
285 struct sem_queue *q, *tq;
286
287 /* complex operations still around? */
288 if (sma->complex_count)
289 return;
290 /*
291 * We will switch back to simple mode.
292 * Move all pending operation back into the per-semaphore
293 * queues.
294 */
295 list_for_each_entry_safe(q, tq, &sma->pending_alter, list) {
296 struct sem *curr;
Manfred Spraul1a233952017-07-12 14:34:38 -0700297 curr = &sma->sems[q->sops[0].sem_num];
Manfred Spraulf269f402013-07-08 16:01:24 -0700298
299 list_add_tail(&q->list, &curr->pending_alter);
300 }
301 INIT_LIST_HEAD(&sma->pending_alter);
302}
303
304/**
Davidlohr Bueso8001c852014-01-27 17:07:05 -0800305 * merge_queues - merge single semop queues into global queue
Manfred Spraulf269f402013-07-08 16:01:24 -0700306 * @sma: semaphore array
307 *
308 * This function merges all per-semaphore queues into the global queue.
309 * It is necessary to achieve FIFO ordering for the pending single-sop
310 * operations when a multi-semop operation must sleep.
311 * Only the alter operations must be moved, the const operations can stay.
312 */
313static void merge_queues(struct sem_array *sma)
314{
315 int i;
316 for (i = 0; i < sma->sem_nsems; i++) {
Manfred Spraul1a233952017-07-12 14:34:38 -0700317 struct sem *sem = &sma->sems[i];
Manfred Spraulf269f402013-07-08 16:01:24 -0700318
319 list_splice_init(&sem->pending_alter, &sma->pending_alter);
320 }
321}
322
Davidlohr Bueso53dad6d2013-09-23 17:04:45 -0700323static void sem_rcu_free(struct rcu_head *head)
324{
Manfred Sprauldba4cdd2017-07-12 14:34:41 -0700325 struct kern_ipc_perm *p = container_of(head, struct kern_ipc_perm, rcu);
326 struct sem_array *sma = container_of(p, struct sem_array, sem_perm);
Davidlohr Bueso53dad6d2013-09-23 17:04:45 -0700327
Eric W. Biedermanaefad952018-03-22 20:52:43 -0500328 security_sem_free(&sma->sem_perm);
Kees Cooke2029df2017-07-12 14:35:31 -0700329 kvfree(sma);
Davidlohr Bueso53dad6d2013-09-23 17:04:45 -0700330}
331
Nadia Derbey3e148c72007-10-18 23:40:54 -0700332/*
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700333 * Enter the mode suitable for non-simple operations:
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700334 * Caller must own sem_perm.lock.
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700335 */
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700336static void complexmode_enter(struct sem_array *sma)
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700337{
338 int i;
339 struct sem *sem;
340
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800341 if (sma->use_global_lock > 0) {
342 /*
343 * We are already in global lock mode.
344 * Nothing to do, just reset the
345 * counter until we return to simple mode.
346 */
Manfred Spraul17d056e2021-06-30 18:57:15 -0700347 WRITE_ONCE(sma->use_global_lock, USE_GLOBAL_LOCK_HYSTERESIS);
Manfred Spraul6d07b682013-09-30 13:45:06 -0700348 return;
349 }
Manfred Spraul17d056e2021-06-30 18:57:15 -0700350 WRITE_ONCE(sma->use_global_lock, USE_GLOBAL_LOCK_HYSTERESIS);
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700351
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700352 for (i = 0; i < sma->sem_nsems; i++) {
Manfred Spraul1a233952017-07-12 14:34:38 -0700353 sem = &sma->sems[i];
Manfred Spraul27d7be12017-02-27 14:28:15 -0800354 spin_lock(&sem->lock);
355 spin_unlock(&sem->lock);
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700356 }
357}
358
359/*
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700360 * Try to leave the mode that disallows simple operations:
361 * Caller must own sem_perm.lock.
362 */
363static void complexmode_tryleave(struct sem_array *sma)
364{
365 if (sma->complex_count) {
366 /* Complex ops are sleeping.
367 * We must stay in complex mode
368 */
369 return;
370 }
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800371 if (sma->use_global_lock == 1) {
Manfred Spraul8116b542020-02-03 17:34:42 -0800372
373 /* See SEM_BARRIER_1 for purpose/pairing */
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800374 smp_store_release(&sma->use_global_lock, 0);
375 } else {
Manfred Spraul17d056e2021-06-30 18:57:15 -0700376 WRITE_ONCE(sma->use_global_lock,
377 sma->use_global_lock-1);
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800378 }
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700379}
380
381#define SEM_GLOBAL_LOCK (-1)
382/*
Rik van Riel6062a8d2013-04-30 19:15:44 -0700383 * If the request contains only one semaphore operation, and there are
384 * no complex transactions pending, lock only the semaphore involved.
385 * Otherwise, lock the entire semaphore array, since we either have
386 * multiple semaphores in our own semops, or we need to look at
387 * semaphores from other pending complex operations.
Rik van Riel6062a8d2013-04-30 19:15:44 -0700388 */
389static inline int sem_lock(struct sem_array *sma, struct sembuf *sops,
390 int nsops)
391{
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700392 struct sem *sem;
Davidlohr Buesoec67aaa2018-06-14 15:27:51 -0700393 int idx;
Rik van Riel6062a8d2013-04-30 19:15:44 -0700394
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700395 if (nsops != 1) {
396 /* Complex operation - acquire a full lock */
397 ipc_lock_object(&sma->sem_perm);
398
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700399 /* Prevent parallel simple ops */
400 complexmode_enter(sma);
401 return SEM_GLOBAL_LOCK;
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700402 }
403
404 /*
405 * Only one semaphore affected - try to optimize locking.
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700406 * Optimized locking is possible if no complex operation
407 * is either enqueued or processed right now.
408 *
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800409 * Both facts are tracked by use_global_mode.
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700410 */
Davidlohr Buesoec67aaa2018-06-14 15:27:51 -0700411 idx = array_index_nospec(sops->sem_num, sma->sem_nsems);
412 sem = &sma->sems[idx];
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700413
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700414 /*
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800415 * Initial check for use_global_lock. Just an optimization,
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700416 * no locking, no memory barrier.
417 */
Manfred Spraul17d056e2021-06-30 18:57:15 -0700418 if (!READ_ONCE(sma->use_global_lock)) {
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700419 /*
420 * It appears that no complex operation is around.
421 * Acquire the per-semaphore lock.
422 */
Rik van Riel6062a8d2013-04-30 19:15:44 -0700423 spin_lock(&sem->lock);
424
Manfred Spraul8116b542020-02-03 17:34:42 -0800425 /* see SEM_BARRIER_1 for purpose/pairing */
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800426 if (!smp_load_acquire(&sma->use_global_lock)) {
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700427 /* fast path successful! */
428 return sops->sem_num;
Rik van Riel6062a8d2013-04-30 19:15:44 -0700429 }
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700430 spin_unlock(&sem->lock);
Rik van Riel6062a8d2013-04-30 19:15:44 -0700431 }
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700432
433 /* slow path: acquire the full lock */
434 ipc_lock_object(&sma->sem_perm);
435
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800436 if (sma->use_global_lock == 0) {
437 /*
438 * The use_global_lock mode ended while we waited for
439 * sma->sem_perm.lock. Thus we must switch to locking
440 * with sem->lock.
441 * Unlike in the fast path, there is no need to recheck
442 * sma->use_global_lock after we have acquired sem->lock:
443 * We own sma->sem_perm.lock, thus use_global_lock cannot
444 * change.
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700445 */
446 spin_lock(&sem->lock);
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800447
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700448 ipc_unlock_object(&sma->sem_perm);
449 return sops->sem_num;
450 } else {
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800451 /*
452 * Not a false alarm, thus continue to use the global lock
453 * mode. No need for complexmode_enter(), this was done by
454 * the caller that has set use_global_mode to non-zero.
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700455 */
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700456 return SEM_GLOBAL_LOCK;
Manfred Spraul5e9d5272013-09-30 13:45:04 -0700457 }
Rik van Riel6062a8d2013-04-30 19:15:44 -0700458}
459
460static inline void sem_unlock(struct sem_array *sma, int locknum)
461{
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700462 if (locknum == SEM_GLOBAL_LOCK) {
Manfred Spraulf269f402013-07-08 16:01:24 -0700463 unmerge_queues(sma);
Manfred Spraul5864a2f2016-10-11 13:54:50 -0700464 complexmode_tryleave(sma);
Davidlohr Buesocf9d5d72013-07-08 16:01:11 -0700465 ipc_unlock_object(&sma->sem_perm);
Rik van Riel6062a8d2013-04-30 19:15:44 -0700466 } else {
Manfred Spraul1a233952017-07-12 14:34:38 -0700467 struct sem *sem = &sma->sems[locknum];
Rik van Riel6062a8d2013-04-30 19:15:44 -0700468 spin_unlock(&sem->lock);
469 }
Rik van Riel6062a8d2013-04-30 19:15:44 -0700470}
471
472/*
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700473 * sem_lock_(check_) routines are called in the paths where the rwsem
Nadia Derbey3e148c72007-10-18 23:40:54 -0700474 * is not held.
Linus Torvalds321310c2013-05-04 10:47:57 -0700475 *
476 * The caller holds the RCU read lock.
Nadia Derbey3e148c72007-10-18 23:40:54 -0700477 */
Davidlohr Bueso16df3672013-04-30 19:15:29 -0700478static inline struct sem_array *sem_obtain_object(struct ipc_namespace *ns, int id)
479{
Davidlohr Bueso55b7ae52015-06-30 14:58:42 -0700480 struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&sem_ids(ns), id);
Davidlohr Bueso16df3672013-04-30 19:15:29 -0700481
482 if (IS_ERR(ipcp))
483 return ERR_CAST(ipcp);
484
485 return container_of(ipcp, struct sem_array, sem_perm);
486}
487
Davidlohr Bueso16df3672013-04-30 19:15:29 -0700488static inline struct sem_array *sem_obtain_object_check(struct ipc_namespace *ns,
489 int id)
490{
491 struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&sem_ids(ns), id);
492
493 if (IS_ERR(ipcp))
494 return ERR_CAST(ipcp);
Pierre Peifferb1ed88b2008-02-06 01:36:23 -0800495
Nadia Derbey03f02c72007-10-18 23:40:51 -0700496 return container_of(ipcp, struct sem_array, sem_perm);
Nadia Derbey023a5352007-10-18 23:40:51 -0700497}
498
Pierre Peiffer6ff37972008-04-29 01:00:46 -0700499static inline void sem_lock_and_putref(struct sem_array *sma)
500{
Rik van Riel6062a8d2013-04-30 19:15:44 -0700501 sem_lock(sma, NULL, -1);
Manfred Sprauldba4cdd2017-07-12 14:34:41 -0700502 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
Pierre Peiffer6ff37972008-04-29 01:00:46 -0700503}
504
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700505static inline void sem_rmid(struct ipc_namespace *ns, struct sem_array *s)
506{
507 ipc_rmid(&sem_ids(ns), &s->sem_perm);
508}
509
Kees Cook101ede02017-07-12 14:35:02 -0700510static struct sem_array *sem_alloc(size_t nsems)
511{
512 struct sem_array *sma;
Kees Cook101ede02017-07-12 14:35:02 -0700513
514 if (nsems > (INT_MAX - sizeof(*sma)) / sizeof(sma->sems[0]))
515 return NULL;
516
Vasily Averin18319492021-09-02 14:55:31 -0700517 sma = kvzalloc(struct_size(sma, sems, nsems), GFP_KERNEL_ACCOUNT);
Kees Cook101ede02017-07-12 14:35:02 -0700518 if (unlikely(!sma))
519 return NULL;
520
Kees Cook101ede02017-07-12 14:35:02 -0700521 return sma;
522}
523
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700524/**
525 * newary - Create a new semaphore set
526 * @ns: namespace
527 * @params: ptr to the structure that contains key, semflg and nsems
528 *
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700529 * Called with sem_ids.rwsem held (as a writer)
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700530 */
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700531static int newary(struct ipc_namespace *ns, struct ipc_params *params)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 int retval;
534 struct sem_array *sma;
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700535 key_t key = params->key;
536 int nsems = params->u.nsems;
537 int semflg = params->flg;
Manfred Spraulb97e8202009-12-15 16:47:32 -0800538 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539
540 if (!nsems)
541 return -EINVAL;
Kirill Korotaeve3893532006-10-02 02:18:22 -0700542 if (ns->used_sems + nsems > ns->sc_semmns)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 return -ENOSPC;
544
Kees Cook101ede02017-07-12 14:35:02 -0700545 sma = sem_alloc(nsems);
Davidlohr Bueso3ab08fe2014-01-27 17:07:06 -0800546 if (!sma)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547 return -ENOMEM;
Davidlohr Bueso3ab08fe2014-01-27 17:07:06 -0800548
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 sma->sem_perm.mode = (semflg & S_IRWXUGO);
550 sma->sem_perm.key = key;
551
552 sma->sem_perm.security = NULL;
Eric W. Biedermanaefad952018-03-22 20:52:43 -0500553 retval = security_sem_alloc(&sma->sem_perm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 if (retval) {
Kees Cooke2029df2017-07-12 14:35:31 -0700555 kvfree(sma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556 return retval;
557 }
558
Rik van Riel6062a8d2013-04-30 19:15:44 -0700559 for (i = 0; i < nsems; i++) {
Manfred Spraul1a233952017-07-12 14:34:38 -0700560 INIT_LIST_HEAD(&sma->sems[i].pending_alter);
561 INIT_LIST_HEAD(&sma->sems[i].pending_const);
562 spin_lock_init(&sma->sems[i].lock);
Rik van Riel6062a8d2013-04-30 19:15:44 -0700563 }
Manfred Spraulb97e8202009-12-15 16:47:32 -0800564
565 sma->complex_count = 0;
Manfred Spraul9de5ab82017-02-27 14:28:18 -0800566 sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS;
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700567 INIT_LIST_HEAD(&sma->pending_alter);
568 INIT_LIST_HEAD(&sma->pending_const);
Manfred Spraul4daa28f2008-07-25 01:48:04 -0700569 INIT_LIST_HEAD(&sma->list_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 sma->sem_nsems = nsems;
Deepa Dinamanie54d02b2017-08-02 19:51:13 -0700571 sma->sem_ctime = ktime_get_real_seconds();
Manfred Spraule8577d12014-12-02 15:59:34 -0800572
Davidlohr Bueso39c96a12017-11-17 15:31:11 -0800573 /* ipc_addid() locks sma upon success. */
Manfred Spraul2ec55f82017-07-12 14:35:13 -0700574 retval = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni);
575 if (retval < 0) {
Manfred Spraul39cfffd2018-08-21 22:01:29 -0700576 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
Manfred Spraul2ec55f82017-07-12 14:35:13 -0700577 return retval;
Manfred Spraule8577d12014-12-02 15:59:34 -0800578 }
579 ns->used_sems += nsems;
580
Rik van Riel6062a8d2013-04-30 19:15:44 -0700581 sem_unlock(sma, -1);
Linus Torvalds6d49dab2013-05-03 15:04:40 -0700582 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700584 return sma->sem_perm.id;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585}
586
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700587
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700588/*
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700589 * Called with sem_ids.rwsem and ipcp locked.
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700590 */
Alexey Dobriyan00898e82020-08-11 18:37:05 -0700591static int sem_more_checks(struct kern_ipc_perm *ipcp, struct ipc_params *params)
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700592{
Nadia Derbey03f02c72007-10-18 23:40:51 -0700593 struct sem_array *sma;
594
595 sma = container_of(ipcp, struct sem_array, sem_perm);
596 if (params->u.nsems > sma->sem_nsems)
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700597 return -EINVAL;
598
599 return 0;
600}
601
Dominik Brodowski69894712018-03-20 19:53:58 +0100602long ksys_semget(key_t key, int nsems, int semflg)
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700603{
Kirill Korotaeve3893532006-10-02 02:18:22 -0700604 struct ipc_namespace *ns;
Mathias Krauseeb66ec42014-06-06 14:37:36 -0700605 static const struct ipc_ops sem_ops = {
606 .getnew = newary,
Eric W. Biederman50ab44b2018-03-23 23:41:55 -0500607 .associate = security_sem_associate,
Mathias Krauseeb66ec42014-06-06 14:37:36 -0700608 .more_checks = sem_more_checks,
609 };
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700610 struct ipc_params sem_params;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611
Kirill Korotaeve3893532006-10-02 02:18:22 -0700612 ns = current->nsproxy->ipc_ns;
613
614 if (nsems < 0 || nsems > ns->sc_semmsl)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 return -EINVAL;
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700616
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700617 sem_params.key = key;
618 sem_params.flg = semflg;
619 sem_params.u.nsems = nsems;
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700620
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700621 return ipcget(ns, &sem_ids(ns), &sem_ops, &sem_params);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622}
623
Dominik Brodowski69894712018-03-20 19:53:58 +0100624SYSCALL_DEFINE3(semget, key_t, key, int, nsems, int, semflg)
625{
626 return ksys_semget(key, nsems, semflg);
627}
628
Petr Mladek78f50092014-01-27 17:07:00 -0800629/**
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -0800630 * perform_atomic_semop[_slow] - Attempt to perform semaphore
631 * operations on a given array.
Manfred Spraul758a6ba32013-07-08 16:01:26 -0700632 * @sma: semaphore array
Manfred Sprauld198cd62014-06-06 14:37:49 -0700633 * @q: struct sem_queue that describes the operation
Manfred Spraul758a6ba32013-07-08 16:01:26 -0700634 *
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -0800635 * Caller blocking are as follows, based the value
636 * indicated by the semaphore operation (sem_op):
637 *
638 * (1) >0 never blocks.
639 * (2) 0 (wait-for-zero operation): semval is non-zero.
640 * (3) <0 attempting to decrement semval to a value smaller than zero.
641 *
Manfred Spraul758a6ba32013-07-08 16:01:26 -0700642 * Returns 0 if the operation was possible.
643 * Returns 1 if the operation is impossible, the caller must sleep.
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -0800644 * Returns <0 for error codes.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645 */
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -0800646static int perform_atomic_semop_slow(struct sem_array *sma, struct sem_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647{
Eric W. Biederman51d6f262018-03-23 01:11:29 -0500648 int result, sem_op, nsops;
649 struct pid *pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650 struct sembuf *sop;
Manfred Spraul239521f2014-01-27 17:07:04 -0800651 struct sem *curr;
Manfred Sprauld198cd62014-06-06 14:37:49 -0700652 struct sembuf *sops;
653 struct sem_undo *un;
654
655 sops = q->sops;
656 nsops = q->nsops;
657 un = q->undo;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658
659 for (sop = sops; sop < sops + nsops; sop++) {
Davidlohr Buesoec67aaa2018-06-14 15:27:51 -0700660 int idx = array_index_nospec(sop->sem_num, sma->sem_nsems);
661 curr = &sma->sems[idx];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 sem_op = sop->sem_op;
663 result = curr->semval;
Petr Mladek78f50092014-01-27 17:07:00 -0800664
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665 if (!sem_op && result)
666 goto would_block;
667
668 result += sem_op;
669 if (result < 0)
670 goto would_block;
671 if (result > SEMVMX)
672 goto out_of_range;
Petr Mladek78f50092014-01-27 17:07:00 -0800673
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674 if (sop->sem_flg & SEM_UNDO) {
675 int undo = un->semadj[sop->sem_num] - sem_op;
Petr Mladek78f50092014-01-27 17:07:00 -0800676 /* Exceeding the undo range is an error. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 if (undo < (-SEMAEM - 1) || undo > SEMAEM)
678 goto out_of_range;
Petr Mladek78f50092014-01-27 17:07:00 -0800679 un->semadj[sop->sem_num] = undo;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680 }
Petr Mladek78f50092014-01-27 17:07:00 -0800681
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682 curr->semval = result;
683 }
684
685 sop--;
Manfred Sprauld198cd62014-06-06 14:37:49 -0700686 pid = q->pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687 while (sop >= sops) {
Eric W. Biederman51d6f262018-03-23 01:11:29 -0500688 ipc_update_pid(&sma->sems[sop->sem_num].sempid, pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689 sop--;
690 }
Petr Mladek78f50092014-01-27 17:07:00 -0800691
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692 return 0;
693
694out_of_range:
695 result = -ERANGE;
696 goto undo;
697
698would_block:
Manfred Sprauled247b72014-06-06 14:37:49 -0700699 q->blocking = sop;
700
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701 if (sop->sem_flg & IPC_NOWAIT)
702 result = -EAGAIN;
703 else
704 result = 1;
705
706undo:
707 sop--;
708 while (sop >= sops) {
Petr Mladek78f50092014-01-27 17:07:00 -0800709 sem_op = sop->sem_op;
Manfred Spraul1a233952017-07-12 14:34:38 -0700710 sma->sems[sop->sem_num].semval -= sem_op;
Petr Mladek78f50092014-01-27 17:07:00 -0800711 if (sop->sem_flg & SEM_UNDO)
712 un->semadj[sop->sem_num] += sem_op;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713 sop--;
714 }
715
716 return result;
717}
718
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -0800719static int perform_atomic_semop(struct sem_array *sma, struct sem_queue *q)
720{
721 int result, sem_op, nsops;
722 struct sembuf *sop;
723 struct sem *curr;
724 struct sembuf *sops;
725 struct sem_undo *un;
726
727 sops = q->sops;
728 nsops = q->nsops;
729 un = q->undo;
730
731 if (unlikely(q->dupsop))
732 return perform_atomic_semop_slow(sma, q);
733
734 /*
735 * We scan the semaphore set twice, first to ensure that the entire
736 * operation can succeed, therefore avoiding any pointless writes
737 * to shared memory and having to undo such changes in order to block
738 * until the operations can go through.
739 */
740 for (sop = sops; sop < sops + nsops; sop++) {
Davidlohr Buesoec67aaa2018-06-14 15:27:51 -0700741 int idx = array_index_nospec(sop->sem_num, sma->sem_nsems);
742
743 curr = &sma->sems[idx];
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -0800744 sem_op = sop->sem_op;
745 result = curr->semval;
746
747 if (!sem_op && result)
748 goto would_block; /* wait-for-zero */
749
750 result += sem_op;
751 if (result < 0)
752 goto would_block;
753
754 if (result > SEMVMX)
755 return -ERANGE;
756
757 if (sop->sem_flg & SEM_UNDO) {
758 int undo = un->semadj[sop->sem_num] - sem_op;
759
760 /* Exceeding the undo range is an error. */
761 if (undo < (-SEMAEM - 1) || undo > SEMAEM)
762 return -ERANGE;
763 }
764 }
765
766 for (sop = sops; sop < sops + nsops; sop++) {
Manfred Spraul1a233952017-07-12 14:34:38 -0700767 curr = &sma->sems[sop->sem_num];
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -0800768 sem_op = sop->sem_op;
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -0800769
770 if (sop->sem_flg & SEM_UNDO) {
771 int undo = un->semadj[sop->sem_num] - sem_op;
772
773 un->semadj[sop->sem_num] = undo;
774 }
775 curr->semval += sem_op;
Eric W. Biederman51d6f262018-03-23 01:11:29 -0500776 ipc_update_pid(&curr->sempid, q->pid);
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -0800777 }
778
779 return 0;
780
781would_block:
782 q->blocking = sop;
783 return sop->sem_flg & IPC_NOWAIT ? -EAGAIN : 1;
784}
785
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800786static inline void wake_up_sem_queue_prepare(struct sem_queue *q, int error,
787 struct wake_q_head *wake_q)
Nick Piggind4212092009-12-15 16:47:30 -0800788{
Varad Gautama11ddb32021-05-22 17:41:49 -0700789 struct task_struct *sleeper;
790
791 sleeper = get_task_struct(q->sleeper);
Manfred Spraul8116b542020-02-03 17:34:42 -0800792
Bhaskar Chowdhury74978352021-05-06 18:06:41 -0700793 /* see SEM_BARRIER_2 for purpose/pairing */
Manfred Spraul8116b542020-02-03 17:34:42 -0800794 smp_store_release(&q->status, error);
795
Varad Gautama11ddb32021-05-22 17:41:49 -0700796 wake_q_add_safe(wake_q, sleeper);
Nick Piggind4212092009-12-15 16:47:30 -0800797}
798
Manfred Spraulb97e8202009-12-15 16:47:32 -0800799static void unlink_queue(struct sem_array *sma, struct sem_queue *q)
800{
801 list_del(&q->list);
Rik van Riel9f1bc2c92013-04-30 19:15:39 -0700802 if (q->nsops > 1)
Manfred Spraulb97e8202009-12-15 16:47:32 -0800803 sma->complex_count--;
804}
805
Manfred Spraulfd5db422010-05-26 14:43:40 -0700806/** check_restart(sma, q)
807 * @sma: semaphore array
808 * @q: the operation that just completed
809 *
810 * update_queue is O(N^2) when it restarts scanning the whole queue of
811 * waiting operations. Therefore this function checks if the restart is
812 * really necessary. It is called after a previously waiting operation
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700813 * modified the array.
814 * Note that wait-for-zero operations are handled without restart.
Manfred Spraulfd5db422010-05-26 14:43:40 -0700815 */
Davidlohr Bueso4663d3e2016-12-14 15:06:40 -0800816static inline int check_restart(struct sem_array *sma, struct sem_queue *q)
Manfred Spraulfd5db422010-05-26 14:43:40 -0700817{
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700818 /* pending complex alter operations are too difficult to analyse */
819 if (!list_empty(&sma->pending_alter))
Manfred Spraulfd5db422010-05-26 14:43:40 -0700820 return 1;
821
822 /* we were a sleeping complex operation. Too difficult */
823 if (q->nsops > 1)
824 return 1;
825
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700826 /* It is impossible that someone waits for the new value:
827 * - complex operations always restart.
Bhaskar Chowdhuryb1989a32021-05-06 18:05:48 -0700828 * - wait-for-zero are handled separately.
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700829 * - q is a previously sleeping simple operation that
830 * altered the array. It must be a decrement, because
831 * simple increments never sleep.
832 * - If there are older (higher priority) decrements
833 * in the queue, then they have observed the original
834 * semval value and couldn't proceed. The operation
835 * decremented to value - thus they won't proceed either.
836 */
837 return 0;
838}
Manfred Spraulfd5db422010-05-26 14:43:40 -0700839
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700840/**
Davidlohr Bueso8001c852014-01-27 17:07:05 -0800841 * wake_const_ops - wake up non-alter tasks
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700842 * @sma: semaphore array.
843 * @semnum: semaphore that was modified.
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800844 * @wake_q: lockless wake-queue head.
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700845 *
846 * wake_const_ops must be called after a semaphore in a semaphore array
847 * was set to 0. If complex const operations are pending, wake_const_ops must
848 * be called with semnum = -1, as well as with the number of each modified
849 * semaphore.
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800850 * The tasks that must be woken up are added to @wake_q. The return code
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700851 * is stored in q->pid.
852 * The function returns 1 if at least one operation was completed successfully.
853 */
854static int wake_const_ops(struct sem_array *sma, int semnum,
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800855 struct wake_q_head *wake_q)
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700856{
Davidlohr Buesof150f022016-12-14 15:06:43 -0800857 struct sem_queue *q, *tmp;
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700858 struct list_head *pending_list;
859 int semop_completed = 0;
Manfred Spraulfd5db422010-05-26 14:43:40 -0700860
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700861 if (semnum == -1)
862 pending_list = &sma->pending_const;
863 else
Manfred Spraul1a233952017-07-12 14:34:38 -0700864 pending_list = &sma->sems[semnum].pending_const;
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700865
Davidlohr Buesof150f022016-12-14 15:06:43 -0800866 list_for_each_entry_safe(q, tmp, pending_list, list) {
867 int error = perform_atomic_semop(sma, q);
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700868
Davidlohr Buesof150f022016-12-14 15:06:43 -0800869 if (error > 0)
870 continue;
871 /* operation completed, remove from queue & wakeup */
872 unlink_queue(sma, q);
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700873
Davidlohr Buesof150f022016-12-14 15:06:43 -0800874 wake_up_sem_queue_prepare(q, error, wake_q);
875 if (error == 0)
876 semop_completed = 1;
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700877 }
Davidlohr Buesof150f022016-12-14 15:06:43 -0800878
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700879 return semop_completed;
880}
881
882/**
Davidlohr Bueso8001c852014-01-27 17:07:05 -0800883 * do_smart_wakeup_zero - wakeup all wait for zero tasks
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700884 * @sma: semaphore array
885 * @sops: operations that were performed
886 * @nsops: number of operations
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800887 * @wake_q: lockless wake-queue head
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700888 *
Davidlohr Bueso8001c852014-01-27 17:07:05 -0800889 * Checks all required queue for wait-for-zero operations, based
890 * on the actual changes that were performed on the semaphore array.
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700891 * The function returns 1 if at least one operation was completed successfully.
892 */
893static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops,
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800894 int nsops, struct wake_q_head *wake_q)
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700895{
896 int i;
897 int semop_completed = 0;
898 int got_zero = 0;
899
900 /* first: the per-semaphore queues, if known */
901 if (sops) {
902 for (i = 0; i < nsops; i++) {
903 int num = sops[i].sem_num;
904
Manfred Spraul1a233952017-07-12 14:34:38 -0700905 if (sma->sems[num].semval == 0) {
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700906 got_zero = 1;
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800907 semop_completed |= wake_const_ops(sma, num, wake_q);
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700908 }
909 }
910 } else {
911 /*
912 * No sops means modified semaphores not known.
913 * Assume all were changed.
Manfred Spraulfd5db422010-05-26 14:43:40 -0700914 */
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700915 for (i = 0; i < sma->sem_nsems; i++) {
Manfred Spraul1a233952017-07-12 14:34:38 -0700916 if (sma->sems[i].semval == 0) {
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700917 got_zero = 1;
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800918 semop_completed |= wake_const_ops(sma, i, wake_q);
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700919 }
920 }
Manfred Spraulfd5db422010-05-26 14:43:40 -0700921 }
922 /*
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700923 * If one of the modified semaphores got 0,
924 * then check the global queue, too.
Manfred Spraulfd5db422010-05-26 14:43:40 -0700925 */
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700926 if (got_zero)
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800927 semop_completed |= wake_const_ops(sma, -1, wake_q);
Manfred Spraulfd5db422010-05-26 14:43:40 -0700928
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700929 return semop_completed;
Manfred Spraulfd5db422010-05-26 14:43:40 -0700930}
931
Manfred Spraul636c6be2009-12-15 16:47:33 -0800932
933/**
Davidlohr Bueso8001c852014-01-27 17:07:05 -0800934 * update_queue - look for tasks that can be completed.
Manfred Spraul636c6be2009-12-15 16:47:33 -0800935 * @sma: semaphore array.
936 * @semnum: semaphore that was modified.
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800937 * @wake_q: lockless wake-queue head.
Manfred Spraul636c6be2009-12-15 16:47:33 -0800938 *
939 * update_queue must be called after a semaphore in a semaphore array
Rik van Riel9f1bc2c92013-04-30 19:15:39 -0700940 * was modified. If multiple semaphores were modified, update_queue must
941 * be called with semnum = -1, as well as with the number of each modified
942 * semaphore.
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800943 * The tasks that must be woken up are added to @wake_q. The return code
Manfred Spraul0a2b9d42010-05-26 14:43:41 -0700944 * is stored in q->pid.
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700945 * The function internally checks if const operations can now succeed.
946 *
Manfred Spraul0a2b9d42010-05-26 14:43:41 -0700947 * The function return 1 if at least one semop was completed successfully.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948 */
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800949static int update_queue(struct sem_array *sma, int semnum, struct wake_q_head *wake_q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950{
Davidlohr Buesof150f022016-12-14 15:06:43 -0800951 struct sem_queue *q, *tmp;
Manfred Spraul636c6be2009-12-15 16:47:33 -0800952 struct list_head *pending_list;
Manfred Spraul0a2b9d42010-05-26 14:43:41 -0700953 int semop_completed = 0;
Manfred Spraul636c6be2009-12-15 16:47:33 -0800954
Rik van Riel9f1bc2c92013-04-30 19:15:39 -0700955 if (semnum == -1)
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700956 pending_list = &sma->pending_alter;
Rik van Riel9f1bc2c92013-04-30 19:15:39 -0700957 else
Manfred Spraul1a233952017-07-12 14:34:38 -0700958 pending_list = &sma->sems[semnum].pending_alter;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959
Nick Piggin9cad2002009-12-15 16:47:29 -0800960again:
Davidlohr Buesof150f022016-12-14 15:06:43 -0800961 list_for_each_entry_safe(q, tmp, pending_list, list) {
Manfred Spraulfd5db422010-05-26 14:43:40 -0700962 int error, restart;
Manfred Spraul636c6be2009-12-15 16:47:33 -0800963
Manfred Sprauld987f8b22009-12-15 16:47:34 -0800964 /* If we are scanning the single sop, per-semaphore list of
965 * one semaphore and that semaphore is 0, then it is not
Manfred Spraul1a82e9e2013-07-08 16:01:23 -0700966 * necessary to scan further: simple increments
Manfred Sprauld987f8b22009-12-15 16:47:34 -0800967 * that affect only one entry succeed immediately and cannot
968 * be in the per semaphore pending queue, and decrements
969 * cannot be successful if the value is already 0.
970 */
Manfred Spraul1a233952017-07-12 14:34:38 -0700971 if (semnum != -1 && sma->sems[semnum].semval == 0)
Manfred Sprauld987f8b22009-12-15 16:47:34 -0800972 break;
973
Manfred Sprauld198cd62014-06-06 14:37:49 -0700974 error = perform_atomic_semop(sma, q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975
976 /* Does q->sleeper still need to sleep? */
Nick Piggin9cad2002009-12-15 16:47:29 -0800977 if (error > 0)
978 continue;
Manfred Spraula1193f82008-07-25 01:48:06 -0700979
Manfred Spraulb97e8202009-12-15 16:47:32 -0800980 unlink_queue(sma, q);
Manfred Spraula1193f82008-07-25 01:48:06 -0700981
Manfred Spraul0a2b9d42010-05-26 14:43:41 -0700982 if (error) {
Manfred Spraulfd5db422010-05-26 14:43:40 -0700983 restart = 0;
Manfred Spraul0a2b9d42010-05-26 14:43:41 -0700984 } else {
985 semop_completed = 1;
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800986 do_smart_wakeup_zero(sma, q->sops, q->nsops, wake_q);
Manfred Spraulfd5db422010-05-26 14:43:40 -0700987 restart = check_restart(sma, q);
Manfred Spraul0a2b9d42010-05-26 14:43:41 -0700988 }
Manfred Spraulfd5db422010-05-26 14:43:40 -0700989
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -0800990 wake_up_sem_queue_prepare(q, error, wake_q);
Manfred Spraulfd5db422010-05-26 14:43:40 -0700991 if (restart)
Nick Piggin9cad2002009-12-15 16:47:29 -0800992 goto again;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 }
Manfred Spraul0a2b9d42010-05-26 14:43:41 -0700994 return semop_completed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995}
996
Manfred Spraul0a2b9d42010-05-26 14:43:41 -0700997/**
Davidlohr Bueso8001c852014-01-27 17:07:05 -0800998 * set_semotime - set sem_otime
Manfred Spraul0e8c6652013-09-30 13:45:25 -0700999 * @sma: semaphore array
1000 * @sops: operations that modified the array, may be NULL
1001 *
1002 * sem_otime is replicated to avoid cache line trashing.
1003 * This function sets one instance to the current time.
1004 */
1005static void set_semotime(struct sem_array *sma, struct sembuf *sops)
1006{
1007 if (sops == NULL) {
Arnd Bergmann2a70b782018-04-12 15:19:44 +02001008 sma->sems[0].sem_otime = ktime_get_real_seconds();
Manfred Spraul0e8c6652013-09-30 13:45:25 -07001009 } else {
Manfred Spraul1a233952017-07-12 14:34:38 -07001010 sma->sems[sops[0].sem_num].sem_otime =
Arnd Bergmann2a70b782018-04-12 15:19:44 +02001011 ktime_get_real_seconds();
Manfred Spraul0e8c6652013-09-30 13:45:25 -07001012 }
1013}
1014
1015/**
Davidlohr Bueso8001c852014-01-27 17:07:05 -08001016 * do_smart_update - optimized update_queue
Manfred Spraulfd5db422010-05-26 14:43:40 -07001017 * @sma: semaphore array
1018 * @sops: operations that were performed
1019 * @nsops: number of operations
Manfred Spraul0a2b9d42010-05-26 14:43:41 -07001020 * @otime: force setting otime
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001021 * @wake_q: lockless wake-queue head
Manfred Spraulfd5db422010-05-26 14:43:40 -07001022 *
Manfred Spraul1a82e9e2013-07-08 16:01:23 -07001023 * do_smart_update() does the required calls to update_queue and wakeup_zero,
1024 * based on the actual changes that were performed on the semaphore array.
Manfred Spraul0a2b9d42010-05-26 14:43:41 -07001025 * Note that the function does not do the actual wake-up: the caller is
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001026 * responsible for calling wake_up_q().
Manfred Spraul0a2b9d42010-05-26 14:43:41 -07001027 * It is safe to perform this call after dropping all locks.
Manfred Spraulfd5db422010-05-26 14:43:40 -07001028 */
Manfred Spraul0a2b9d42010-05-26 14:43:41 -07001029static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsops,
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001030 int otime, struct wake_q_head *wake_q)
Manfred Spraulfd5db422010-05-26 14:43:40 -07001031{
1032 int i;
1033
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001034 otime |= do_smart_wakeup_zero(sma, sops, nsops, wake_q);
Manfred Spraul1a82e9e2013-07-08 16:01:23 -07001035
Manfred Spraulf269f402013-07-08 16:01:24 -07001036 if (!list_empty(&sma->pending_alter)) {
1037 /* semaphore array uses the global queue - just process it. */
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001038 otime |= update_queue(sma, -1, wake_q);
Manfred Spraulf269f402013-07-08 16:01:24 -07001039 } else {
1040 if (!sops) {
1041 /*
1042 * No sops, thus the modified semaphores are not
1043 * known. Check all.
1044 */
1045 for (i = 0; i < sma->sem_nsems; i++)
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001046 otime |= update_queue(sma, i, wake_q);
Manfred Spraulf269f402013-07-08 16:01:24 -07001047 } else {
1048 /*
1049 * Check the semaphores that were increased:
1050 * - No complex ops, thus all sleeping ops are
1051 * decrease.
1052 * - if we decreased the value, then any sleeping
Bhaskar Chowdhuryb1989a32021-05-06 18:05:48 -07001053 * semaphore ops won't be able to run: If the
Manfred Spraulf269f402013-07-08 16:01:24 -07001054 * previous value was too small, then the new
1055 * value will be too small, too.
1056 */
1057 for (i = 0; i < nsops; i++) {
1058 if (sops[i].sem_op > 0) {
1059 otime |= update_queue(sma,
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001060 sops[i].sem_num, wake_q);
Manfred Spraulf269f402013-07-08 16:01:24 -07001061 }
Manfred Spraulab465df2013-05-26 11:08:52 +02001062 }
Rik van Riel9f1bc2c92013-04-30 19:15:39 -07001063 }
Manfred Spraulfd5db422010-05-26 14:43:40 -07001064 }
Manfred Spraul0e8c6652013-09-30 13:45:25 -07001065 if (otime)
1066 set_semotime(sma, sops);
Manfred Spraulfd5db422010-05-26 14:43:40 -07001067}
1068
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001069/*
Manfred Spraulb220c572014-06-06 14:37:51 -07001070 * check_qop: Test if a queued operation sleeps on the semaphore semnum
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001071 */
1072static int check_qop(struct sem_array *sma, int semnum, struct sem_queue *q,
1073 bool count_zero)
1074{
Manfred Spraulb220c572014-06-06 14:37:51 -07001075 struct sembuf *sop = q->blocking;
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001076
Manfred Spraul9b44ee22014-06-06 14:37:52 -07001077 /*
1078 * Linux always (since 0.99.10) reported a task as sleeping on all
1079 * semaphores. This violates SUS, therefore it was changed to the
1080 * standard compliant behavior.
1081 * Give the administrators a chance to notice that an application
1082 * might misbehave because it relies on the Linux behavior.
1083 */
1084 pr_info_once("semctl(GETNCNT/GETZCNT) is since 3.16 Single Unix Specification compliant.\n"
1085 "The task %s (%d) triggered the difference, watch for misbehavior.\n",
1086 current->comm, task_pid_nr(current));
1087
Manfred Spraulb220c572014-06-06 14:37:51 -07001088 if (sop->sem_num != semnum)
1089 return 0;
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001090
Manfred Spraulb220c572014-06-06 14:37:51 -07001091 if (count_zero && sop->sem_op == 0)
1092 return 1;
1093 if (!count_zero && sop->sem_op < 0)
1094 return 1;
1095
1096 return 0;
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001097}
1098
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099/* The following counts are associated to each semaphore:
1100 * semncnt number of tasks waiting on semval being nonzero
1101 * semzcnt number of tasks waiting on semval being zero
Manfred Spraulb220c572014-06-06 14:37:51 -07001102 *
1103 * Per definition, a task waits only on the semaphore of the first semop
1104 * that cannot proceed, even if additional operation would block, too.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105 */
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001106static int count_semcnt(struct sem_array *sma, ushort semnum,
1107 bool count_zero)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108{
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001109 struct list_head *l;
Manfred Spraul239521f2014-01-27 17:07:04 -08001110 struct sem_queue *q;
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001111 int semcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001112
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001113 semcnt = 0;
1114 /* First: check the simple operations. They are easy to evaluate */
1115 if (count_zero)
Manfred Spraul1a233952017-07-12 14:34:38 -07001116 l = &sma->sems[semnum].pending_const;
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001117 else
Manfred Spraul1a233952017-07-12 14:34:38 -07001118 l = &sma->sems[semnum].pending_alter;
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001119
1120 list_for_each_entry(q, l, list) {
1121 /* all task on a per-semaphore list sleep on exactly
1122 * that semaphore
1123 */
1124 semcnt++;
Rik van Rielde2657f2013-05-09 16:59:59 -04001125 }
1126
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001127 /* Then: check the complex operations. */
Manfred Spraul1a82e9e2013-07-08 16:01:23 -07001128 list_for_each_entry(q, &sma->pending_alter, list) {
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001129 semcnt += check_qop(sma, semnum, q, count_zero);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001130 }
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001131 if (count_zero) {
1132 list_for_each_entry(q, &sma->pending_const, list) {
1133 semcnt += check_qop(sma, semnum, q, count_zero);
1134 }
Rik van Rielebc2e5e2013-05-09 16:53:28 -04001135 }
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001136 return semcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137}
1138
Davidlohr Buesod9a605e2013-09-11 14:26:24 -07001139/* Free a semaphore set. freeary() is called with sem_ids.rwsem locked
1140 * as a writer and the spinlock for this semaphore set hold. sem_ids.rwsem
Nadia Derbey3e148c72007-10-18 23:40:54 -07001141 * remains locked on exit.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001142 */
Pierre Peiffer01b8b072008-02-08 04:18:57 -08001143static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001144{
Manfred Spraul380af1b2008-07-25 01:48:06 -07001145 struct sem_undo *un, *tu;
1146 struct sem_queue *q, *tq;
Pierre Peiffer01b8b072008-02-08 04:18:57 -08001147 struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm);
Rik van Riel9f1bc2c92013-04-30 19:15:39 -07001148 int i;
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001149 DEFINE_WAKE_Q(wake_q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150
Manfred Spraul380af1b2008-07-25 01:48:06 -07001151 /* Free the existing undo structures for this semaphore set. */
Davidlohr Buesocf9d5d72013-07-08 16:01:11 -07001152 ipc_assert_locked_object(&sma->sem_perm);
Manfred Spraul380af1b2008-07-25 01:48:06 -07001153 list_for_each_entry_safe(un, tu, &sma->list_id, list_id) {
1154 list_del(&un->list_id);
1155 spin_lock(&un->ulp->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001156 un->semid = -1;
Manfred Spraul380af1b2008-07-25 01:48:06 -07001157 list_del_rcu(&un->list_proc);
1158 spin_unlock(&un->ulp->lock);
Vasily Averinfc37a3b2021-06-30 18:57:09 -07001159 kvfree_rcu(un, rcu);
Manfred Spraul380af1b2008-07-25 01:48:06 -07001160 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161
1162 /* Wake up all pending processes and let them fail with EIDRM. */
Manfred Spraul1a82e9e2013-07-08 16:01:23 -07001163 list_for_each_entry_safe(q, tq, &sma->pending_const, list) {
1164 unlink_queue(sma, q);
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001165 wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
Manfred Spraul1a82e9e2013-07-08 16:01:23 -07001166 }
1167
1168 list_for_each_entry_safe(q, tq, &sma->pending_alter, list) {
Manfred Spraulb97e8202009-12-15 16:47:32 -08001169 unlink_queue(sma, q);
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001170 wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001171 }
Rik van Riel9f1bc2c92013-04-30 19:15:39 -07001172 for (i = 0; i < sma->sem_nsems; i++) {
Manfred Spraul1a233952017-07-12 14:34:38 -07001173 struct sem *sem = &sma->sems[i];
Manfred Spraul1a82e9e2013-07-08 16:01:23 -07001174 list_for_each_entry_safe(q, tq, &sem->pending_const, list) {
1175 unlink_queue(sma, q);
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001176 wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
Manfred Spraul1a82e9e2013-07-08 16:01:23 -07001177 }
1178 list_for_each_entry_safe(q, tq, &sem->pending_alter, list) {
Rik van Riel9f1bc2c92013-04-30 19:15:39 -07001179 unlink_queue(sma, q);
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001180 wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
Rik van Riel9f1bc2c92013-04-30 19:15:39 -07001181 }
Eric W. Biederman51d6f262018-03-23 01:11:29 -05001182 ipc_update_pid(&sem->sempid, NULL);
Rik van Riel9f1bc2c92013-04-30 19:15:39 -07001183 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184
Nadia Derbey7ca7e562007-10-18 23:40:48 -07001185 /* Remove the semaphore set from the IDR */
1186 sem_rmid(ns, sma);
Rik van Riel6062a8d2013-04-30 19:15:44 -07001187 sem_unlock(sma, -1);
Linus Torvalds6d49dab2013-05-03 15:04:40 -07001188 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001190 wake_up_q(&wake_q);
Kirill Korotaeve3893532006-10-02 02:18:22 -07001191 ns->used_sems -= sma->sem_nsems;
Manfred Sprauldba4cdd2017-07-12 14:34:41 -07001192 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001193}
1194
1195static unsigned long copy_semid_to_user(void __user *buf, struct semid64_ds *in, int version)
1196{
Manfred Spraul239521f2014-01-27 17:07:04 -08001197 switch (version) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198 case IPC_64:
1199 return copy_to_user(buf, in, sizeof(*in));
1200 case IPC_OLD:
1201 {
1202 struct semid_ds out;
1203
Dan Rosenberg982f7c22010-09-30 15:15:31 -07001204 memset(&out, 0, sizeof(out));
1205
Linus Torvalds1da177e2005-04-16 15:20:36 -07001206 ipc64_perm_to_ipc_perm(&in->sem_perm, &out.sem_perm);
1207
1208 out.sem_otime = in->sem_otime;
1209 out.sem_ctime = in->sem_ctime;
1210 out.sem_nsems = in->sem_nsems;
1211
1212 return copy_to_user(buf, &out, sizeof(out));
1213 }
1214 default:
1215 return -EINVAL;
1216 }
1217}
1218
Deepa Dinamanie54d02b2017-08-02 19:51:13 -07001219static time64_t get_semotime(struct sem_array *sma)
Manfred Sprauld12e1e52013-07-08 16:01:25 -07001220{
1221 int i;
Deepa Dinamanie54d02b2017-08-02 19:51:13 -07001222 time64_t res;
Manfred Sprauld12e1e52013-07-08 16:01:25 -07001223
Manfred Spraul1a233952017-07-12 14:34:38 -07001224 res = sma->sems[0].sem_otime;
Manfred Sprauld12e1e52013-07-08 16:01:25 -07001225 for (i = 1; i < sma->sem_nsems; i++) {
Deepa Dinamanie54d02b2017-08-02 19:51:13 -07001226 time64_t to = sma->sems[i].sem_otime;
Manfred Sprauld12e1e52013-07-08 16:01:25 -07001227
1228 if (to > res)
1229 res = to;
1230 }
1231 return res;
1232}
1233
Al Viro45a4a642017-07-09 09:11:00 -04001234static int semctl_stat(struct ipc_namespace *ns, int semid,
1235 int cmd, struct semid64_ds *semid64)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001236{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001237 struct sem_array *sma;
Arnd Bergmannc2ab9752015-04-28 21:39:50 +02001238 time64_t semotime;
Al Viro45a4a642017-07-09 09:11:00 -04001239 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240
Al Viro45a4a642017-07-09 09:11:00 -04001241 memset(semid64, 0, sizeof(*semid64));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242
Al Viro45a4a642017-07-09 09:11:00 -04001243 rcu_read_lock();
Davidlohr Buesoa280d6d2018-04-10 16:35:26 -07001244 if (cmd == SEM_STAT || cmd == SEM_STAT_ANY) {
Al Viro45a4a642017-07-09 09:11:00 -04001245 sma = sem_obtain_object(ns, semid);
1246 if (IS_ERR(sma)) {
1247 err = PTR_ERR(sma);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001248 goto out_unlock;
Al Viro45a4a642017-07-09 09:11:00 -04001249 }
Davidlohr Buesoa280d6d2018-04-10 16:35:26 -07001250 } else { /* IPC_STAT */
Al Viro45a4a642017-07-09 09:11:00 -04001251 sma = sem_obtain_object_check(ns, semid);
1252 if (IS_ERR(sma)) {
1253 err = PTR_ERR(sma);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001254 goto out_unlock;
Al Viro45a4a642017-07-09 09:11:00 -04001255 }
1256 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001257
Davidlohr Buesoa280d6d2018-04-10 16:35:26 -07001258 /* see comment for SHM_STAT_ANY */
1259 if (cmd == SEM_STAT_ANY)
1260 audit_ipc_obj(&sma->sem_perm);
1261 else {
1262 err = -EACCES;
1263 if (ipcperms(ns, &sma->sem_perm, S_IRUGO))
1264 goto out_unlock;
1265 }
Al Viro45a4a642017-07-09 09:11:00 -04001266
Eric W. Biedermanaefad952018-03-22 20:52:43 -05001267 err = security_sem_semctl(&sma->sem_perm, cmd);
Al Viro45a4a642017-07-09 09:11:00 -04001268 if (err)
1269 goto out_unlock;
1270
Philippe Mikoyan87ad4b02018-02-06 15:40:49 -08001271 ipc_lock_object(&sma->sem_perm);
1272
1273 if (!ipc_valid_object(&sma->sem_perm)) {
1274 ipc_unlock_object(&sma->sem_perm);
1275 err = -EIDRM;
1276 goto out_unlock;
1277 }
1278
Al Viro45a4a642017-07-09 09:11:00 -04001279 kernel_to_ipc64_perm(&sma->sem_perm, &semid64->sem_perm);
Arnd Bergmannc2ab9752015-04-28 21:39:50 +02001280 semotime = get_semotime(sma);
1281 semid64->sem_otime = semotime;
Al Viro45a4a642017-07-09 09:11:00 -04001282 semid64->sem_ctime = sma->sem_ctime;
Arnd Bergmannc2ab9752015-04-28 21:39:50 +02001283#ifndef CONFIG_64BIT
1284 semid64->sem_otime_high = semotime >> 32;
1285 semid64->sem_ctime_high = sma->sem_ctime >> 32;
1286#endif
Al Viro45a4a642017-07-09 09:11:00 -04001287 semid64->sem_nsems = sma->sem_nsems;
Philippe Mikoyan87ad4b02018-02-06 15:40:49 -08001288
Manfred Spraul615c9992018-08-21 22:01:21 -07001289 if (cmd == IPC_STAT) {
1290 /*
1291 * As defined in SUS:
1292 * Return 0 on success
1293 */
1294 err = 0;
1295 } else {
1296 /*
1297 * SEM_STAT and SEM_STAT_ANY (both Linux specific)
1298 * Return the full id, including the sequence number
1299 */
1300 err = sma->sem_perm.id;
1301 }
Philippe Mikoyan87ad4b02018-02-06 15:40:49 -08001302 ipc_unlock_object(&sma->sem_perm);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001303out_unlock:
Davidlohr Bueso16df3672013-04-30 19:15:29 -07001304 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001305 return err;
1306}
1307
Al Viro45a4a642017-07-09 09:11:00 -04001308static int semctl_info(struct ipc_namespace *ns, int semid,
1309 int cmd, void __user *p)
1310{
1311 struct seminfo seminfo;
Manfred Spraul27c331a2018-08-21 22:02:00 -07001312 int max_idx;
Al Viro45a4a642017-07-09 09:11:00 -04001313 int err;
1314
1315 err = security_sem_semctl(NULL, cmd);
1316 if (err)
1317 return err;
1318
1319 memset(&seminfo, 0, sizeof(seminfo));
1320 seminfo.semmni = ns->sc_semmni;
1321 seminfo.semmns = ns->sc_semmns;
1322 seminfo.semmsl = ns->sc_semmsl;
1323 seminfo.semopm = ns->sc_semopm;
1324 seminfo.semvmx = SEMVMX;
1325 seminfo.semmnu = SEMMNU;
1326 seminfo.semmap = SEMMAP;
1327 seminfo.semume = SEMUME;
1328 down_read(&sem_ids(ns).rwsem);
1329 if (cmd == SEM_INFO) {
1330 seminfo.semusz = sem_ids(ns).in_use;
1331 seminfo.semaem = ns->used_sems;
1332 } else {
1333 seminfo.semusz = SEMUSZ;
1334 seminfo.semaem = SEMAEM;
1335 }
Manfred Spraul27c331a2018-08-21 22:02:00 -07001336 max_idx = ipc_get_maxidx(&sem_ids(ns));
Al Viro45a4a642017-07-09 09:11:00 -04001337 up_read(&sem_ids(ns).rwsem);
1338 if (copy_to_user(p, &seminfo, sizeof(struct seminfo)))
1339 return -EFAULT;
Manfred Spraul27c331a2018-08-21 22:02:00 -07001340 return (max_idx < 0) ? 0 : max_idx;
Al Viro45a4a642017-07-09 09:11:00 -04001341}
1342
Al Viroe1fd1f42013-03-05 15:04:55 -05001343static int semctl_setval(struct ipc_namespace *ns, int semid, int semnum,
Al Viro45a4a642017-07-09 09:11:00 -04001344 int val)
Al Viroe1fd1f42013-03-05 15:04:55 -05001345{
1346 struct sem_undo *un;
1347 struct sem_array *sma;
Manfred Spraul239521f2014-01-27 17:07:04 -08001348 struct sem *curr;
Al Viro45a4a642017-07-09 09:11:00 -04001349 int err;
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001350 DEFINE_WAKE_Q(wake_q);
1351
Rik van Riel6062a8d2013-04-30 19:15:44 -07001352 if (val > SEMVMX || val < 0)
1353 return -ERANGE;
Al Viroe1fd1f42013-03-05 15:04:55 -05001354
Rik van Riel6062a8d2013-04-30 19:15:44 -07001355 rcu_read_lock();
1356 sma = sem_obtain_object_check(ns, semid);
1357 if (IS_ERR(sma)) {
1358 rcu_read_unlock();
1359 return PTR_ERR(sma);
1360 }
1361
1362 if (semnum < 0 || semnum >= sma->sem_nsems) {
1363 rcu_read_unlock();
1364 return -EINVAL;
1365 }
1366
1367
1368 if (ipcperms(ns, &sma->sem_perm, S_IWUGO)) {
1369 rcu_read_unlock();
1370 return -EACCES;
1371 }
Al Viroe1fd1f42013-03-05 15:04:55 -05001372
Eric W. Biedermanaefad952018-03-22 20:52:43 -05001373 err = security_sem_semctl(&sma->sem_perm, SETVAL);
Rik van Riel6062a8d2013-04-30 19:15:44 -07001374 if (err) {
1375 rcu_read_unlock();
1376 return -EACCES;
1377 }
Al Viroe1fd1f42013-03-05 15:04:55 -05001378
Rik van Riel6062a8d2013-04-30 19:15:44 -07001379 sem_lock(sma, NULL, -1);
Al Viroe1fd1f42013-03-05 15:04:55 -05001380
Rafael Aquini0f3d2b02014-01-27 17:07:01 -08001381 if (!ipc_valid_object(&sma->sem_perm)) {
Manfred Spraul6e224f92013-10-16 13:46:45 -07001382 sem_unlock(sma, -1);
1383 rcu_read_unlock();
1384 return -EIDRM;
1385 }
1386
Davidlohr Buesoec67aaa2018-06-14 15:27:51 -07001387 semnum = array_index_nospec(semnum, sma->sem_nsems);
Manfred Spraul1a233952017-07-12 14:34:38 -07001388 curr = &sma->sems[semnum];
Al Viroe1fd1f42013-03-05 15:04:55 -05001389
Davidlohr Buesocf9d5d72013-07-08 16:01:11 -07001390 ipc_assert_locked_object(&sma->sem_perm);
Al Viroe1fd1f42013-03-05 15:04:55 -05001391 list_for_each_entry(un, &sma->list_id, list_id)
1392 un->semadj[semnum] = 0;
1393
1394 curr->semval = val;
Eric W. Biederman51d6f262018-03-23 01:11:29 -05001395 ipc_update_pid(&curr->sempid, task_tgid(current));
Deepa Dinamanie54d02b2017-08-02 19:51:13 -07001396 sma->sem_ctime = ktime_get_real_seconds();
Al Viroe1fd1f42013-03-05 15:04:55 -05001397 /* maybe some queued-up processes were waiting for this */
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001398 do_smart_update(sma, NULL, 0, 0, &wake_q);
Rik van Riel6062a8d2013-04-30 19:15:44 -07001399 sem_unlock(sma, -1);
Linus Torvalds6d49dab2013-05-03 15:04:40 -07001400 rcu_read_unlock();
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001401 wake_up_q(&wake_q);
Rik van Riel6062a8d2013-04-30 19:15:44 -07001402 return 0;
Al Viroe1fd1f42013-03-05 15:04:55 -05001403}
1404
Kirill Korotaeve3893532006-10-02 02:18:22 -07001405static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
Al Viroe1fd1f42013-03-05 15:04:55 -05001406 int cmd, void __user *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407{
1408 struct sem_array *sma;
Manfred Spraul239521f2014-01-27 17:07:04 -08001409 struct sem *curr;
Davidlohr Bueso16df3672013-04-30 19:15:29 -07001410 int err, nsems;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001411 ushort fast_sem_io[SEMMSL_FAST];
Manfred Spraul239521f2014-01-27 17:07:04 -08001412 ushort *sem_io = fast_sem_io;
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001413 DEFINE_WAKE_Q(wake_q);
Davidlohr Bueso16df3672013-04-30 19:15:29 -07001414
1415 rcu_read_lock();
1416 sma = sem_obtain_object_check(ns, semid);
1417 if (IS_ERR(sma)) {
1418 rcu_read_unlock();
1419 return PTR_ERR(sma);
1420 }
1421
Linus Torvalds1da177e2005-04-16 15:20:36 -07001422 nsems = sma->sem_nsems;
1423
Linus Torvalds1da177e2005-04-16 15:20:36 -07001424 err = -EACCES;
Linus Torvaldsc728b9c2013-05-04 11:04:29 -07001425 if (ipcperms(ns, &sma->sem_perm, cmd == SETALL ? S_IWUGO : S_IRUGO))
1426 goto out_rcu_wakeup;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001427
Eric W. Biedermanaefad952018-03-22 20:52:43 -05001428 err = security_sem_semctl(&sma->sem_perm, cmd);
Linus Torvaldsc728b9c2013-05-04 11:04:29 -07001429 if (err)
1430 goto out_rcu_wakeup;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001431
Linus Torvalds1da177e2005-04-16 15:20:36 -07001432 switch (cmd) {
1433 case GETALL:
1434 {
Al Viroe1fd1f42013-03-05 15:04:55 -05001435 ushort __user *array = p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436 int i;
1437
Al Viroce857222013-05-03 00:30:49 +01001438 sem_lock(sma, NULL, -1);
Rafael Aquini0f3d2b02014-01-27 17:07:01 -08001439 if (!ipc_valid_object(&sma->sem_perm)) {
Manfred Spraul6e224f92013-10-16 13:46:45 -07001440 err = -EIDRM;
1441 goto out_unlock;
1442 }
Manfred Spraul239521f2014-01-27 17:07:04 -08001443 if (nsems > SEMMSL_FAST) {
Manfred Sprauldba4cdd2017-07-12 14:34:41 -07001444 if (!ipc_rcu_getref(&sma->sem_perm)) {
Al Viroce857222013-05-03 00:30:49 +01001445 err = -EIDRM;
Manfred Spraul6e224f92013-10-16 13:46:45 -07001446 goto out_unlock;
Al Viroce857222013-05-03 00:30:49 +01001447 }
1448 sem_unlock(sma, -1);
Linus Torvalds6d49dab2013-05-03 15:04:40 -07001449 rcu_read_unlock();
Kees Cookf8dbe8d2017-07-12 14:34:47 -07001450 sem_io = kvmalloc_array(nsems, sizeof(ushort),
1451 GFP_KERNEL);
Manfred Spraul239521f2014-01-27 17:07:04 -08001452 if (sem_io == NULL) {
Manfred Sprauldba4cdd2017-07-12 14:34:41 -07001453 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454 return -ENOMEM;
1455 }
1456
Linus Torvalds4091fd92013-05-04 10:13:40 -07001457 rcu_read_lock();
Pierre Peiffer6ff37972008-04-29 01:00:46 -07001458 sem_lock_and_putref(sma);
Rafael Aquini0f3d2b02014-01-27 17:07:01 -08001459 if (!ipc_valid_object(&sma->sem_perm)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001460 err = -EIDRM;
Manfred Spraul6e224f92013-10-16 13:46:45 -07001461 goto out_unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001462 }
Al Viroce857222013-05-03 00:30:49 +01001463 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001464 for (i = 0; i < sma->sem_nsems; i++)
Manfred Spraul1a233952017-07-12 14:34:38 -07001465 sem_io[i] = sma->sems[i].semval;
Rik van Riel6062a8d2013-04-30 19:15:44 -07001466 sem_unlock(sma, -1);
Linus Torvalds6d49dab2013-05-03 15:04:40 -07001467 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001468 err = 0;
Manfred Spraul239521f2014-01-27 17:07:04 -08001469 if (copy_to_user(array, sem_io, nsems*sizeof(ushort)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001470 err = -EFAULT;
1471 goto out_free;
1472 }
1473 case SETALL:
1474 {
1475 int i;
1476 struct sem_undo *un;
1477
Manfred Sprauldba4cdd2017-07-12 14:34:41 -07001478 if (!ipc_rcu_getref(&sma->sem_perm)) {
Manfred Spraul6e224f92013-10-16 13:46:45 -07001479 err = -EIDRM;
1480 goto out_rcu_wakeup;
Rik van Riel6062a8d2013-04-30 19:15:44 -07001481 }
Davidlohr Bueso16df3672013-04-30 19:15:29 -07001482 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001483
Manfred Spraul239521f2014-01-27 17:07:04 -08001484 if (nsems > SEMMSL_FAST) {
Kees Cookf8dbe8d2017-07-12 14:34:47 -07001485 sem_io = kvmalloc_array(nsems, sizeof(ushort),
1486 GFP_KERNEL);
Manfred Spraul239521f2014-01-27 17:07:04 -08001487 if (sem_io == NULL) {
Manfred Sprauldba4cdd2017-07-12 14:34:41 -07001488 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489 return -ENOMEM;
1490 }
1491 }
1492
Manfred Spraul239521f2014-01-27 17:07:04 -08001493 if (copy_from_user(sem_io, p, nsems*sizeof(ushort))) {
Manfred Sprauldba4cdd2017-07-12 14:34:41 -07001494 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495 err = -EFAULT;
1496 goto out_free;
1497 }
1498
1499 for (i = 0; i < nsems; i++) {
1500 if (sem_io[i] > SEMVMX) {
Manfred Sprauldba4cdd2017-07-12 14:34:41 -07001501 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502 err = -ERANGE;
1503 goto out_free;
1504 }
1505 }
Linus Torvalds4091fd92013-05-04 10:13:40 -07001506 rcu_read_lock();
Pierre Peiffer6ff37972008-04-29 01:00:46 -07001507 sem_lock_and_putref(sma);
Rafael Aquini0f3d2b02014-01-27 17:07:01 -08001508 if (!ipc_valid_object(&sma->sem_perm)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509 err = -EIDRM;
Manfred Spraul6e224f92013-10-16 13:46:45 -07001510 goto out_unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511 }
1512
Davidlohr Buesoa5f4db82016-03-22 14:27:48 -07001513 for (i = 0; i < nsems; i++) {
Manfred Spraul1a233952017-07-12 14:34:38 -07001514 sma->sems[i].semval = sem_io[i];
Eric W. Biederman51d6f262018-03-23 01:11:29 -05001515 ipc_update_pid(&sma->sems[i].sempid, task_tgid(current));
Davidlohr Buesoa5f4db82016-03-22 14:27:48 -07001516 }
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001517
Davidlohr Buesocf9d5d72013-07-08 16:01:11 -07001518 ipc_assert_locked_object(&sma->sem_perm);
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001519 list_for_each_entry(un, &sma->list_id, list_id) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520 for (i = 0; i < nsems; i++)
1521 un->semadj[i] = 0;
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001522 }
Deepa Dinamanie54d02b2017-08-02 19:51:13 -07001523 sma->sem_ctime = ktime_get_real_seconds();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001524 /* maybe some queued-up processes were waiting for this */
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001525 do_smart_update(sma, NULL, 0, 0, &wake_q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526 err = 0;
1527 goto out_unlock;
1528 }
Al Viroe1fd1f42013-03-05 15:04:55 -05001529 /* GETVAL, GETPID, GETNCTN, GETZCNT: fall-through */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530 }
1531 err = -EINVAL;
Linus Torvaldsc728b9c2013-05-04 11:04:29 -07001532 if (semnum < 0 || semnum >= nsems)
1533 goto out_rcu_wakeup;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534
Rik van Riel6062a8d2013-04-30 19:15:44 -07001535 sem_lock(sma, NULL, -1);
Rafael Aquini0f3d2b02014-01-27 17:07:01 -08001536 if (!ipc_valid_object(&sma->sem_perm)) {
Manfred Spraul6e224f92013-10-16 13:46:45 -07001537 err = -EIDRM;
1538 goto out_unlock;
1539 }
Davidlohr Buesoec67aaa2018-06-14 15:27:51 -07001540
1541 semnum = array_index_nospec(semnum, nsems);
Manfred Spraul1a233952017-07-12 14:34:38 -07001542 curr = &sma->sems[semnum];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001543
1544 switch (cmd) {
1545 case GETVAL:
1546 err = curr->semval;
1547 goto out_unlock;
1548 case GETPID:
Eric W. Biederman51d6f262018-03-23 01:11:29 -05001549 err = pid_vnr(curr->sempid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001550 goto out_unlock;
1551 case GETNCNT:
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001552 err = count_semcnt(sma, semnum, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001553 goto out_unlock;
1554 case GETZCNT:
Manfred Spraul2f2ed41d2014-06-06 14:37:48 -07001555 err = count_semcnt(sma, semnum, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001556 goto out_unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557 }
Davidlohr Bueso16df3672013-04-30 19:15:29 -07001558
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559out_unlock:
Rik van Riel6062a8d2013-04-30 19:15:44 -07001560 sem_unlock(sma, -1);
Linus Torvaldsc728b9c2013-05-04 11:04:29 -07001561out_rcu_wakeup:
Linus Torvalds6d49dab2013-05-03 15:04:40 -07001562 rcu_read_unlock();
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08001563 wake_up_q(&wake_q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001564out_free:
Manfred Spraul239521f2014-01-27 17:07:04 -08001565 if (sem_io != fast_sem_io)
Kees Cookf8dbe8d2017-07-12 14:34:47 -07001566 kvfree(sem_io);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001567 return err;
1568}
1569
Pierre Peiffer016d7132008-04-29 01:00:50 -07001570static inline unsigned long
1571copy_semid_from_user(struct semid64_ds *out, void __user *buf, int version)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572{
Manfred Spraul239521f2014-01-27 17:07:04 -08001573 switch (version) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001574 case IPC_64:
Pierre Peiffer016d7132008-04-29 01:00:50 -07001575 if (copy_from_user(out, buf, sizeof(*out)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001576 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001577 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578 case IPC_OLD:
1579 {
1580 struct semid_ds tbuf_old;
1581
Manfred Spraul239521f2014-01-27 17:07:04 -08001582 if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583 return -EFAULT;
1584
Pierre Peiffer016d7132008-04-29 01:00:50 -07001585 out->sem_perm.uid = tbuf_old.sem_perm.uid;
1586 out->sem_perm.gid = tbuf_old.sem_perm.gid;
1587 out->sem_perm.mode = tbuf_old.sem_perm.mode;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001588
1589 return 0;
1590 }
1591 default:
1592 return -EINVAL;
1593 }
1594}
1595
Pierre Peiffer522bb2a2008-04-29 01:00:49 -07001596/*
Davidlohr Buesod9a605e2013-09-11 14:26:24 -07001597 * This function handles some semctl commands which require the rwsem
Pierre Peiffer522bb2a2008-04-29 01:00:49 -07001598 * to be held in write mode.
Davidlohr Buesod9a605e2013-09-11 14:26:24 -07001599 * NOTE: no locks must be held, the rwsem is taken inside this function.
Pierre Peiffer522bb2a2008-04-29 01:00:49 -07001600 */
Pierre Peiffer21a48262008-04-29 01:00:49 -07001601static int semctl_down(struct ipc_namespace *ns, int semid,
Al Viro45a4a642017-07-09 09:11:00 -04001602 int cmd, struct semid64_ds *semid64)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001603{
1604 struct sem_array *sma;
1605 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606 struct kern_ipc_perm *ipcp;
1607
Davidlohr Buesod9a605e2013-09-11 14:26:24 -07001608 down_write(&sem_ids(ns).rwsem);
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -07001609 rcu_read_lock();
1610
Manfred Spraul4241c1a2018-08-21 22:01:34 -07001611 ipcp = ipcctl_obtain_check(ns, &sem_ids(ns), semid, cmd,
Al Viro45a4a642017-07-09 09:11:00 -04001612 &semid64->sem_perm, 0);
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -07001613 if (IS_ERR(ipcp)) {
1614 err = PTR_ERR(ipcp);
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -07001615 goto out_unlock1;
1616 }
Steve Grubb073115d2006-04-02 17:07:33 -04001617
Pierre Peiffera5f75e72008-04-29 01:00:54 -07001618 sma = container_of(ipcp, struct sem_array, sem_perm);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619
Eric W. Biedermanaefad952018-03-22 20:52:43 -05001620 err = security_sem_semctl(&sma->sem_perm, cmd);
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -07001621 if (err)
1622 goto out_unlock1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001623
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -07001624 switch (cmd) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001625 case IPC_RMID:
Rik van Riel6062a8d2013-04-30 19:15:44 -07001626 sem_lock(sma, NULL, -1);
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -07001627 /* freeary unlocks the ipc object and rcu */
Pierre Peiffer01b8b072008-02-08 04:18:57 -08001628 freeary(ns, ipcp);
Pierre Peiffer522bb2a2008-04-29 01:00:49 -07001629 goto out_up;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630 case IPC_SET:
Rik van Riel6062a8d2013-04-30 19:15:44 -07001631 sem_lock(sma, NULL, -1);
Al Viro45a4a642017-07-09 09:11:00 -04001632 err = ipc_update_perm(&semid64->sem_perm, ipcp);
Eric W. Biederman1efdb692012-02-07 16:54:11 -08001633 if (err)
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -07001634 goto out_unlock0;
Deepa Dinamanie54d02b2017-08-02 19:51:13 -07001635 sma->sem_ctime = ktime_get_real_seconds();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001636 break;
1637 default:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001638 err = -EINVAL;
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -07001639 goto out_unlock1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001640 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001641
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -07001642out_unlock0:
Rik van Riel6062a8d2013-04-30 19:15:44 -07001643 sem_unlock(sma, -1);
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -07001644out_unlock1:
Linus Torvalds6d49dab2013-05-03 15:04:40 -07001645 rcu_read_unlock();
Pierre Peiffer522bb2a2008-04-29 01:00:49 -07001646out_up:
Davidlohr Buesod9a605e2013-09-11 14:26:24 -07001647 up_write(&sem_ids(ns).rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648 return err;
1649}
1650
Arnd Bergmann275f2212018-12-31 22:22:40 +01001651static long ksys_semctl(int semid, int semnum, int cmd, unsigned long arg, int version)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652{
Kirill Korotaeve3893532006-10-02 02:18:22 -07001653 struct ipc_namespace *ns;
Al Viroe1fd1f42013-03-05 15:04:55 -05001654 void __user *p = (void __user *)arg;
Al Viro45a4a642017-07-09 09:11:00 -04001655 struct semid64_ds semid64;
1656 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657
1658 if (semid < 0)
1659 return -EINVAL;
1660
Kirill Korotaeve3893532006-10-02 02:18:22 -07001661 ns = current->nsproxy->ipc_ns;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001662
Manfred Spraul239521f2014-01-27 17:07:04 -08001663 switch (cmd) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001664 case IPC_INFO:
1665 case SEM_INFO:
Al Viro45a4a642017-07-09 09:11:00 -04001666 return semctl_info(ns, semid, cmd, p);
Pierre Peiffer4b9fcb02008-02-08 04:18:56 -08001667 case IPC_STAT:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001668 case SEM_STAT:
Davidlohr Buesoa280d6d2018-04-10 16:35:26 -07001669 case SEM_STAT_ANY:
Al Viro45a4a642017-07-09 09:11:00 -04001670 err = semctl_stat(ns, semid, cmd, &semid64);
1671 if (err < 0)
1672 return err;
1673 if (copy_semid_to_user(p, &semid64, version))
1674 err = -EFAULT;
1675 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001676 case GETALL:
1677 case GETVAL:
1678 case GETPID:
1679 case GETNCNT:
1680 case GETZCNT:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001681 case SETALL:
Al Viroe1fd1f42013-03-05 15:04:55 -05001682 return semctl_main(ns, semid, semnum, cmd, p);
Al Viro45a4a642017-07-09 09:11:00 -04001683 case SETVAL: {
1684 int val;
1685#if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN)
1686 /* big-endian 64bit */
1687 val = arg >> 32;
1688#else
1689 /* 32bit or little-endian 64bit */
1690 val = arg;
1691#endif
1692 return semctl_setval(ns, semid, semnum, val);
1693 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001694 case IPC_SET:
Al Viro45a4a642017-07-09 09:11:00 -04001695 if (copy_semid_from_user(&semid64, p, version))
1696 return -EFAULT;
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05001697 fallthrough;
Al Viro45a4a642017-07-09 09:11:00 -04001698 case IPC_RMID:
1699 return semctl_down(ns, semid, cmd, &semid64);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001700 default:
1701 return -EINVAL;
1702 }
1703}
1704
Dominik Brodowskid969c6f2018-03-20 20:00:39 +01001705SYSCALL_DEFINE4(semctl, int, semid, int, semnum, int, cmd, unsigned long, arg)
1706{
Arnd Bergmann275f2212018-12-31 22:22:40 +01001707 return ksys_semctl(semid, semnum, cmd, arg, IPC_64);
Dominik Brodowskid969c6f2018-03-20 20:00:39 +01001708}
1709
Arnd Bergmann275f2212018-12-31 22:22:40 +01001710#ifdef CONFIG_ARCH_WANT_IPC_PARSE_VERSION
1711long ksys_old_semctl(int semid, int semnum, int cmd, unsigned long arg)
1712{
1713 int version = ipc_parse_version(&cmd);
1714
1715 return ksys_semctl(semid, semnum, cmd, arg, version);
1716}
1717
1718SYSCALL_DEFINE4(old_semctl, int, semid, int, semnum, int, cmd, unsigned long, arg)
1719{
1720 return ksys_old_semctl(semid, semnum, cmd, arg);
1721}
1722#endif
1723
Al Viroc0ebccb2017-07-09 10:03:23 -04001724#ifdef CONFIG_COMPAT
1725
1726struct compat_semid_ds {
1727 struct compat_ipc_perm sem_perm;
Arnd Bergmann9afc5ee2018-07-13 12:52:28 +02001728 old_time32_t sem_otime;
1729 old_time32_t sem_ctime;
Al Viroc0ebccb2017-07-09 10:03:23 -04001730 compat_uptr_t sem_base;
1731 compat_uptr_t sem_pending;
1732 compat_uptr_t sem_pending_last;
1733 compat_uptr_t undo;
1734 unsigned short sem_nsems;
1735};
1736
1737static int copy_compat_semid_from_user(struct semid64_ds *out, void __user *buf,
1738 int version)
1739{
1740 memset(out, 0, sizeof(*out));
1741 if (version == IPC_64) {
Linus Torvalds6aa211e2017-09-25 18:37:28 -07001742 struct compat_semid64_ds __user *p = buf;
Al Viroc0ebccb2017-07-09 10:03:23 -04001743 return get_compat_ipc64_perm(&out->sem_perm, &p->sem_perm);
1744 } else {
Linus Torvalds6aa211e2017-09-25 18:37:28 -07001745 struct compat_semid_ds __user *p = buf;
Al Viroc0ebccb2017-07-09 10:03:23 -04001746 return get_compat_ipc_perm(&out->sem_perm, &p->sem_perm);
1747 }
1748}
1749
1750static int copy_compat_semid_to_user(void __user *buf, struct semid64_ds *in,
1751 int version)
1752{
1753 if (version == IPC_64) {
1754 struct compat_semid64_ds v;
1755 memset(&v, 0, sizeof(v));
1756 to_compat_ipc64_perm(&v.sem_perm, &in->sem_perm);
Arnd Bergmannc2ab9752015-04-28 21:39:50 +02001757 v.sem_otime = lower_32_bits(in->sem_otime);
1758 v.sem_otime_high = upper_32_bits(in->sem_otime);
1759 v.sem_ctime = lower_32_bits(in->sem_ctime);
1760 v.sem_ctime_high = upper_32_bits(in->sem_ctime);
Al Viroc0ebccb2017-07-09 10:03:23 -04001761 v.sem_nsems = in->sem_nsems;
1762 return copy_to_user(buf, &v, sizeof(v));
1763 } else {
1764 struct compat_semid_ds v;
1765 memset(&v, 0, sizeof(v));
1766 to_compat_ipc_perm(&v.sem_perm, &in->sem_perm);
1767 v.sem_otime = in->sem_otime;
1768 v.sem_ctime = in->sem_ctime;
1769 v.sem_nsems = in->sem_nsems;
1770 return copy_to_user(buf, &v, sizeof(v));
1771 }
1772}
1773
Arnd Bergmann275f2212018-12-31 22:22:40 +01001774static long compat_ksys_semctl(int semid, int semnum, int cmd, int arg, int version)
Al Viroc0ebccb2017-07-09 10:03:23 -04001775{
1776 void __user *p = compat_ptr(arg);
1777 struct ipc_namespace *ns;
1778 struct semid64_ds semid64;
Al Viroc0ebccb2017-07-09 10:03:23 -04001779 int err;
1780
1781 ns = current->nsproxy->ipc_ns;
1782
1783 if (semid < 0)
1784 return -EINVAL;
1785
1786 switch (cmd & (~IPC_64)) {
1787 case IPC_INFO:
1788 case SEM_INFO:
1789 return semctl_info(ns, semid, cmd, p);
1790 case IPC_STAT:
1791 case SEM_STAT:
Davidlohr Buesoa280d6d2018-04-10 16:35:26 -07001792 case SEM_STAT_ANY:
Al Viroc0ebccb2017-07-09 10:03:23 -04001793 err = semctl_stat(ns, semid, cmd, &semid64);
1794 if (err < 0)
1795 return err;
1796 if (copy_compat_semid_to_user(p, &semid64, version))
1797 err = -EFAULT;
1798 return err;
1799 case GETVAL:
1800 case GETPID:
1801 case GETNCNT:
1802 case GETZCNT:
1803 case GETALL:
1804 case SETALL:
1805 return semctl_main(ns, semid, semnum, cmd, p);
1806 case SETVAL:
1807 return semctl_setval(ns, semid, semnum, arg);
1808 case IPC_SET:
1809 if (copy_compat_semid_from_user(&semid64, p, version))
1810 return -EFAULT;
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05001811 fallthrough;
Al Viroc0ebccb2017-07-09 10:03:23 -04001812 case IPC_RMID:
1813 return semctl_down(ns, semid, cmd, &semid64);
1814 default:
1815 return -EINVAL;
1816 }
1817}
Dominik Brodowskid969c6f2018-03-20 20:00:39 +01001818
1819COMPAT_SYSCALL_DEFINE4(semctl, int, semid, int, semnum, int, cmd, int, arg)
1820{
Arnd Bergmann275f2212018-12-31 22:22:40 +01001821 return compat_ksys_semctl(semid, semnum, cmd, arg, IPC_64);
Dominik Brodowskid969c6f2018-03-20 20:00:39 +01001822}
Arnd Bergmann275f2212018-12-31 22:22:40 +01001823
1824#ifdef CONFIG_ARCH_WANT_COMPAT_IPC_PARSE_VERSION
1825long compat_ksys_old_semctl(int semid, int semnum, int cmd, int arg)
1826{
1827 int version = compat_ipc_parse_version(&cmd);
1828
1829 return compat_ksys_semctl(semid, semnum, cmd, arg, version);
1830}
1831
1832COMPAT_SYSCALL_DEFINE4(old_semctl, int, semid, int, semnum, int, cmd, int, arg)
1833{
1834 return compat_ksys_old_semctl(semid, semnum, cmd, arg);
1835}
1836#endif
Al Viroc0ebccb2017-07-09 10:03:23 -04001837#endif
1838
Linus Torvalds1da177e2005-04-16 15:20:36 -07001839/* If the task doesn't already have a undo_list, then allocate one
1840 * here. We guarantee there is only one thread using this undo list,
1841 * and current is THE ONE
1842 *
1843 * If this allocation and assignment succeeds, but later
1844 * portions of this code fail, there is no need to free the sem_undo_list.
1845 * Just let it stay associated with the task, and it'll be freed later
1846 * at exit time.
1847 *
1848 * This can block, so callers must hold no locks.
1849 */
1850static inline int get_undo_list(struct sem_undo_list **undo_listp)
1851{
1852 struct sem_undo_list *undo_list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001853
1854 undo_list = current->sysvsem.undo_list;
1855 if (!undo_list) {
Vasily Averin18319492021-09-02 14:55:31 -07001856 undo_list = kzalloc(sizeof(*undo_list), GFP_KERNEL_ACCOUNT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001857 if (undo_list == NULL)
1858 return -ENOMEM;
Ingo Molnar00a5dfd2005-08-05 23:05:27 +02001859 spin_lock_init(&undo_list->lock);
Elena Reshetovaf74370b2017-09-08 16:17:42 -07001860 refcount_set(&undo_list->refcnt, 1);
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001861 INIT_LIST_HEAD(&undo_list->list_proc);
1862
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863 current->sysvsem.undo_list = undo_list;
1864 }
1865 *undo_listp = undo_list;
1866 return 0;
1867}
1868
Nick Pigginbf17bb72009-12-15 16:47:28 -08001869static struct sem_undo *__lookup_undo(struct sem_undo_list *ulp, int semid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001870{
Nick Pigginbf17bb72009-12-15 16:47:28 -08001871 struct sem_undo *un;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001872
Joel Fernandes (Google)984035a2019-09-25 16:48:20 -07001873 list_for_each_entry_rcu(un, &ulp->list_proc, list_proc,
1874 spin_is_locked(&ulp->lock)) {
Nick Pigginbf17bb72009-12-15 16:47:28 -08001875 if (un->semid == semid)
1876 return un;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001877 }
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001878 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001879}
1880
Nick Pigginbf17bb72009-12-15 16:47:28 -08001881static struct sem_undo *lookup_undo(struct sem_undo_list *ulp, int semid)
1882{
1883 struct sem_undo *un;
1884
Manfred Spraul239521f2014-01-27 17:07:04 -08001885 assert_spin_locked(&ulp->lock);
Nick Pigginbf17bb72009-12-15 16:47:28 -08001886
1887 un = __lookup_undo(ulp, semid);
1888 if (un) {
1889 list_del_rcu(&un->list_proc);
1890 list_add_rcu(&un->list_proc, &ulp->list_proc);
1891 }
1892 return un;
1893}
1894
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001895/**
Davidlohr Bueso8001c852014-01-27 17:07:05 -08001896 * find_alloc_undo - lookup (and if not present create) undo array
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001897 * @ns: namespace
1898 * @semid: semaphore array id
1899 *
1900 * The function looks up (and if not present creates) the undo structure.
1901 * The size of the undo structure depends on the size of the semaphore
1902 * array, thus the alloc path is not that straightforward.
Manfred Spraul380af1b2008-07-25 01:48:06 -07001903 * Lifetime-rules: sem_undo is rcu-protected, on success, the function
1904 * performs a rcu_read_lock().
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001905 */
1906static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001907{
1908 struct sem_array *sma;
1909 struct sem_undo_list *ulp;
1910 struct sem_undo *un, *new;
Rik van Riel6062a8d2013-04-30 19:15:44 -07001911 int nsems, error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001912
1913 error = get_undo_list(&ulp);
1914 if (error)
1915 return ERR_PTR(error);
1916
Manfred Spraul380af1b2008-07-25 01:48:06 -07001917 rcu_read_lock();
Pierre Peifferc530c6a2007-10-18 23:40:55 -07001918 spin_lock(&ulp->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001919 un = lookup_undo(ulp, semid);
Pierre Peifferc530c6a2007-10-18 23:40:55 -07001920 spin_unlock(&ulp->lock);
Manfred Spraul239521f2014-01-27 17:07:04 -08001921 if (likely(un != NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001922 goto out;
1923
1924 /* no undo structure around - allocate one. */
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001925 /* step 1: figure out the size of the semaphore array */
Davidlohr Bueso16df3672013-04-30 19:15:29 -07001926 sma = sem_obtain_object_check(ns, semid);
1927 if (IS_ERR(sma)) {
1928 rcu_read_unlock();
Julia Lawall4de85cd2010-05-26 14:43:44 -07001929 return ERR_CAST(sma);
Davidlohr Bueso16df3672013-04-30 19:15:29 -07001930 }
Nadia Derbey023a5352007-10-18 23:40:51 -07001931
Linus Torvalds1da177e2005-04-16 15:20:36 -07001932 nsems = sma->sem_nsems;
Manfred Sprauldba4cdd2017-07-12 14:34:41 -07001933 if (!ipc_rcu_getref(&sma->sem_perm)) {
Rik van Riel6062a8d2013-04-30 19:15:44 -07001934 rcu_read_unlock();
1935 un = ERR_PTR(-EIDRM);
1936 goto out;
1937 }
Davidlohr Bueso16df3672013-04-30 19:15:29 -07001938 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001939
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001940 /* step 2: allocate new undo structure */
Vasily Averinfc37a3b2021-06-30 18:57:09 -07001941 new = kvzalloc(sizeof(struct sem_undo) + sizeof(short)*nsems,
Vasily Averin18319492021-09-02 14:55:31 -07001942 GFP_KERNEL_ACCOUNT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001943 if (!new) {
Manfred Sprauldba4cdd2017-07-12 14:34:41 -07001944 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001945 return ERR_PTR(-ENOMEM);
1946 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001947
Manfred Spraul380af1b2008-07-25 01:48:06 -07001948 /* step 3: Acquire the lock on semaphore array */
Linus Torvalds4091fd92013-05-04 10:13:40 -07001949 rcu_read_lock();
Pierre Peiffer6ff37972008-04-29 01:00:46 -07001950 sem_lock_and_putref(sma);
Rafael Aquini0f3d2b02014-01-27 17:07:01 -08001951 if (!ipc_valid_object(&sma->sem_perm)) {
Rik van Riel6062a8d2013-04-30 19:15:44 -07001952 sem_unlock(sma, -1);
Linus Torvalds6d49dab2013-05-03 15:04:40 -07001953 rcu_read_unlock();
Vasily Averinfc37a3b2021-06-30 18:57:09 -07001954 kvfree(new);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001955 un = ERR_PTR(-EIDRM);
1956 goto out;
1957 }
Manfred Spraul380af1b2008-07-25 01:48:06 -07001958 spin_lock(&ulp->lock);
1959
1960 /*
1961 * step 4: check for races: did someone else allocate the undo struct?
1962 */
1963 un = lookup_undo(ulp, semid);
1964 if (un) {
Minghao Chi520ba722022-02-03 20:49:33 -08001965 spin_unlock(&ulp->lock);
Vasily Averinfc37a3b2021-06-30 18:57:09 -07001966 kvfree(new);
Manfred Spraul380af1b2008-07-25 01:48:06 -07001967 goto success;
1968 }
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001969 /* step 5: initialize & link new undo structure */
1970 new->semadj = (short *) &new[1];
Manfred Spraul380af1b2008-07-25 01:48:06 -07001971 new->ulp = ulp;
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001972 new->semid = semid;
1973 assert_spin_locked(&ulp->lock);
Manfred Spraul380af1b2008-07-25 01:48:06 -07001974 list_add_rcu(&new->list_proc, &ulp->list_proc);
Davidlohr Buesocf9d5d72013-07-08 16:01:11 -07001975 ipc_assert_locked_object(&sma->sem_perm);
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001976 list_add(&new->list_id, &sma->list_id);
Manfred Spraul4daa28f2008-07-25 01:48:04 -07001977 un = new;
Manfred Spraul380af1b2008-07-25 01:48:06 -07001978 spin_unlock(&ulp->lock);
Minghao Chi520ba722022-02-03 20:49:33 -08001979success:
Rik van Riel6062a8d2013-04-30 19:15:44 -07001980 sem_unlock(sma, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001981out:
1982 return un;
1983}
1984
Arnd Bergmannbdec0142021-08-11 08:30:23 +01001985long __do_semtimedop(int semid, struct sembuf *sops,
1986 unsigned nsops, const struct timespec64 *timeout,
1987 struct ipc_namespace *ns)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001988{
1989 int error = -EINVAL;
1990 struct sem_array *sma;
Arnd Bergmannbdec0142021-08-11 08:30:23 +01001991 struct sembuf *sop;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001992 struct sem_undo *un;
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -08001993 int max, locknum;
1994 bool undos = false, alter = false, dupsop = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001995 struct sem_queue queue;
Prakash Sangappa49c9dd02022-05-09 18:29:20 -07001996 unsigned long dup = 0;
1997 ktime_t expires, *exp = NULL;
1998 bool timed_out = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001999
2000 if (nsops < 1 || semid < 0)
2001 return -EINVAL;
Kirill Korotaeve3893532006-10-02 02:18:22 -07002002 if (nsops > ns->sc_semopm)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002003 return -E2BIG;
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -08002004
Linus Torvalds1da177e2005-04-16 15:20:36 -07002005 if (timeout) {
Prakash Sangappa49c9dd02022-05-09 18:29:20 -07002006 if (!timespec64_valid(timeout))
2007 return -EINVAL;
2008 expires = ktime_add_safe(ktime_get(),
2009 timespec64_to_ktime(*timeout));
2010 exp = &expires;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002011 }
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -08002012
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002013
Linus Torvalds1da177e2005-04-16 15:20:36 -07002014 max = 0;
2015 for (sop = sops; sop < sops + nsops; sop++) {
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -08002016 unsigned long mask = 1ULL << ((sop->sem_num) % BITS_PER_LONG);
2017
Linus Torvalds1da177e2005-04-16 15:20:36 -07002018 if (sop->sem_num >= max)
2019 max = sop->sem_num;
2020 if (sop->sem_flg & SEM_UNDO)
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -08002021 undos = true;
2022 if (dup & mask) {
2023 /*
2024 * There was a previous alter access that appears
2025 * to have accessed the same semaphore, thus use
2026 * the dupsop logic. "appears", because the detection
2027 * can only check % BITS_PER_LONG.
2028 */
2029 dupsop = true;
2030 }
2031 if (sop->sem_op != 0) {
2032 alter = true;
2033 dup |= mask;
2034 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002035 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002036
Linus Torvalds1da177e2005-04-16 15:20:36 -07002037 if (undos) {
Rik van Riel6062a8d2013-04-30 19:15:44 -07002038 /* On success, find_alloc_undo takes the rcu_read_lock */
Manfred Spraul4daa28f2008-07-25 01:48:04 -07002039 un = find_alloc_undo(ns, semid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002040 if (IS_ERR(un)) {
2041 error = PTR_ERR(un);
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002042 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002043 }
Rik van Riel6062a8d2013-04-30 19:15:44 -07002044 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002045 un = NULL;
Rik van Riel6062a8d2013-04-30 19:15:44 -07002046 rcu_read_lock();
2047 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002048
Davidlohr Bueso16df3672013-04-30 19:15:29 -07002049 sma = sem_obtain_object_check(ns, semid);
Nadia Derbey023a5352007-10-18 23:40:51 -07002050 if (IS_ERR(sma)) {
Rik van Riel6062a8d2013-04-30 19:15:44 -07002051 rcu_read_unlock();
Nadia Derbey023a5352007-10-18 23:40:51 -07002052 error = PTR_ERR(sma);
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002053 goto out;
Nadia Derbey023a5352007-10-18 23:40:51 -07002054 }
2055
Davidlohr Bueso16df3672013-04-30 19:15:29 -07002056 error = -EFBIG;
Davidlohr Bueso248e7352016-12-14 15:06:31 -08002057 if (max >= sma->sem_nsems) {
2058 rcu_read_unlock();
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002059 goto out;
Davidlohr Bueso248e7352016-12-14 15:06:31 -08002060 }
Davidlohr Bueso16df3672013-04-30 19:15:29 -07002061
2062 error = -EACCES;
Davidlohr Bueso248e7352016-12-14 15:06:31 -08002063 if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO)) {
2064 rcu_read_unlock();
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002065 goto out;
Davidlohr Bueso248e7352016-12-14 15:06:31 -08002066 }
Davidlohr Bueso16df3672013-04-30 19:15:29 -07002067
Eric W. Biedermanaefad952018-03-22 20:52:43 -05002068 error = security_sem_semop(&sma->sem_perm, sops, nsops, alter);
Davidlohr Bueso248e7352016-12-14 15:06:31 -08002069 if (error) {
2070 rcu_read_unlock();
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002071 goto out;
Davidlohr Bueso248e7352016-12-14 15:06:31 -08002072 }
Davidlohr Bueso16df3672013-04-30 19:15:29 -07002073
Manfred Spraul6e224f92013-10-16 13:46:45 -07002074 error = -EIDRM;
2075 locknum = sem_lock(sma, sops, nsops);
Rafael Aquini0f3d2b02014-01-27 17:07:01 -08002076 /*
2077 * We eventually might perform the following check in a lockless
2078 * fashion, considering ipc_valid_object() locking constraints.
2079 * If nsops == 1 and there is no contention for sem_perm.lock, then
2080 * only a per-semaphore lock is held and it's OK to proceed with the
2081 * check below. More details on the fine grained locking scheme
2082 * entangled here and why it's RMID race safe on comments at sem_lock()
2083 */
2084 if (!ipc_valid_object(&sma->sem_perm))
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002085 goto out_unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002086 /*
Manfred Spraul4daa28f2008-07-25 01:48:04 -07002087 * semid identifiers are not unique - find_alloc_undo may have
Linus Torvalds1da177e2005-04-16 15:20:36 -07002088 * allocated an undo structure, it was invalidated by an RMID
Manfred Spraul4daa28f2008-07-25 01:48:04 -07002089 * and now a new array with received the same id. Check and fail.
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002090 * This case can be detected checking un->semid. The existence of
Manfred Spraul380af1b2008-07-25 01:48:06 -07002091 * "un" itself is guaranteed by rcu.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002092 */
Rik van Riel6062a8d2013-04-30 19:15:44 -07002093 if (un && un->semid == -1)
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002094 goto out_unlock;
Manfred Spraul4daa28f2008-07-25 01:48:04 -07002095
Manfred Sprauld198cd62014-06-06 14:37:49 -07002096 queue.sops = sops;
2097 queue.nsops = nsops;
2098 queue.undo = un;
Eric W. Biederman51d6f262018-03-23 01:11:29 -05002099 queue.pid = task_tgid(current);
Manfred Sprauld198cd62014-06-06 14:37:49 -07002100 queue.alter = alter;
Davidlohr Bueso4ce33ec2016-12-14 15:06:37 -08002101 queue.dupsop = dupsop;
Manfred Sprauld198cd62014-06-06 14:37:49 -07002102
2103 error = perform_atomic_semop(sma, &queue);
Bhaskar Chowdhuryb1989a32021-05-06 18:05:48 -07002104 if (error == 0) { /* non-blocking successful path */
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08002105 DEFINE_WAKE_Q(wake_q);
2106
2107 /*
2108 * If the operation was successful, then do
Manfred Spraul0e8c6652013-09-30 13:45:25 -07002109 * the required updates.
2110 */
2111 if (alter)
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08002112 do_smart_update(sma, sops, nsops, 1, &wake_q);
Manfred Spraul0e8c6652013-09-30 13:45:25 -07002113 else
2114 set_semotime(sma, sops);
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08002115
2116 sem_unlock(sma, locknum);
2117 rcu_read_unlock();
2118 wake_up_q(&wake_q);
2119
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002120 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002121 }
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08002122 if (error < 0) /* non-blocking error path */
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002123 goto out_unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002124
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08002125 /*
2126 * We need to sleep on this operation, so we put the current
Linus Torvalds1da177e2005-04-16 15:20:36 -07002127 * task into the pending queue and go to sleep.
2128 */
Manfred Spraulb97e8202009-12-15 16:47:32 -08002129 if (nsops == 1) {
2130 struct sem *curr;
Davidlohr Buesoec67aaa2018-06-14 15:27:51 -07002131 int idx = array_index_nospec(sops->sem_num, sma->sem_nsems);
2132 curr = &sma->sems[idx];
Manfred Spraulb97e8202009-12-15 16:47:32 -08002133
Manfred Spraulf269f402013-07-08 16:01:24 -07002134 if (alter) {
2135 if (sma->complex_count) {
2136 list_add_tail(&queue.list,
2137 &sma->pending_alter);
2138 } else {
2139
2140 list_add_tail(&queue.list,
2141 &curr->pending_alter);
2142 }
2143 } else {
Manfred Spraul1a82e9e2013-07-08 16:01:23 -07002144 list_add_tail(&queue.list, &curr->pending_const);
Manfred Spraulf269f402013-07-08 16:01:24 -07002145 }
Manfred Spraulb97e8202009-12-15 16:47:32 -08002146 } else {
Manfred Spraulf269f402013-07-08 16:01:24 -07002147 if (!sma->complex_count)
2148 merge_queues(sma);
2149
Rik van Riel9f1bc2c92013-04-30 19:15:39 -07002150 if (alter)
Manfred Spraul1a82e9e2013-07-08 16:01:23 -07002151 list_add_tail(&queue.list, &sma->pending_alter);
Rik van Riel9f1bc2c92013-04-30 19:15:39 -07002152 else
Manfred Spraul1a82e9e2013-07-08 16:01:23 -07002153 list_add_tail(&queue.list, &sma->pending_const);
2154
Manfred Spraulb97e8202009-12-15 16:47:32 -08002155 sma->complex_count++;
2156 }
2157
Davidlohr Buesob5fa01a2016-12-14 15:06:46 -08002158 do {
Manfred Spraul8116b542020-02-03 17:34:42 -08002159 /* memory ordering ensured by the lock in sem_lock() */
Davidlohr Buesof075faa2018-07-26 16:37:19 -07002160 WRITE_ONCE(queue.status, -EINTR);
Davidlohr Buesob5fa01a2016-12-14 15:06:46 -08002161 queue.sleeper = current;
Manfred Spraul0b0577f2011-11-02 13:38:52 -07002162
Manfred Spraul8116b542020-02-03 17:34:42 -08002163 /* memory ordering is ensured by the lock in sem_lock() */
Davidlohr Buesob5fa01a2016-12-14 15:06:46 -08002164 __set_current_state(TASK_INTERRUPTIBLE);
2165 sem_unlock(sma, locknum);
Linus Torvalds321310c2013-05-04 10:47:57 -07002166 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002167
Prakash Sangappa49c9dd02022-05-09 18:29:20 -07002168 timed_out = !schedule_hrtimeout_range(exp,
2169 current->timer_slack_ns, HRTIMER_MODE_ABS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002170
Davidlohr Buesob5fa01a2016-12-14 15:06:46 -08002171 /*
2172 * fastpath: the semop has completed, either successfully or
2173 * not, from the syscall pov, is quite irrelevant to us at this
2174 * point; we're done.
2175 *
2176 * We _do_ care, nonetheless, about being awoken by a signal or
2177 * spuriously. The queue.status is checked again in the
2178 * slowpath (aka after taking sem_lock), such that we can detect
2179 * scenarios where we were awakened externally, during the
2180 * window between wake_q_add() and wake_up_q().
2181 */
2182 error = READ_ONCE(queue.status);
2183 if (error != -EINTR) {
Manfred Spraul8116b542020-02-03 17:34:42 -08002184 /* see SEM_BARRIER_2 for purpose/pairing */
2185 smp_acquire__after_ctrl_dep();
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002186 goto out;
Davidlohr Buesob5fa01a2016-12-14 15:06:46 -08002187 }
Manfred Spraul0b0577f2011-11-02 13:38:52 -07002188
Davidlohr Buesob5fa01a2016-12-14 15:06:46 -08002189 rcu_read_lock();
Manfred Spraulc626bc42017-01-10 16:57:48 -08002190 locknum = sem_lock(sma, sops, nsops);
Davidlohr Buesob5fa01a2016-12-14 15:06:46 -08002191
Davidlohr Bueso370b2622016-12-14 15:06:49 -08002192 if (!ipc_valid_object(&sma->sem_perm))
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002193 goto out_unlock;
Davidlohr Bueso370b2622016-12-14 15:06:49 -08002194
Manfred Spraul8116b542020-02-03 17:34:42 -08002195 /*
2196 * No necessity for any barrier: We are protect by sem_lock()
2197 */
Davidlohr Bueso370b2622016-12-14 15:06:49 -08002198 error = READ_ONCE(queue.status);
Davidlohr Buesob5fa01a2016-12-14 15:06:46 -08002199
2200 /*
2201 * If queue.status != -EINTR we are woken up by another process.
2202 * Leave without unlink_queue(), but with sem_unlock().
2203 */
2204 if (error != -EINTR)
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002205 goto out_unlock;
Davidlohr Buesob5fa01a2016-12-14 15:06:46 -08002206
2207 /*
2208 * If an interrupt occurred we have to clean up the queue.
2209 */
Prakash Sangappa49c9dd02022-05-09 18:29:20 -07002210 if (timed_out)
Davidlohr Buesob5fa01a2016-12-14 15:06:46 -08002211 error = -EAGAIN;
2212 } while (error == -EINTR && !signal_pending(current)); /* spurious */
Manfred Spraul0b0577f2011-11-02 13:38:52 -07002213
Manfred Spraulb97e8202009-12-15 16:47:32 -08002214 unlink_queue(sma, &queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002215
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002216out_unlock:
Rik van Riel6062a8d2013-04-30 19:15:44 -07002217 sem_unlock(sma, locknum);
Linus Torvalds6d49dab2013-05-03 15:04:40 -07002218 rcu_read_unlock();
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002219out:
2220 return error;
2221}
2222
2223static long do_semtimedop(int semid, struct sembuf __user *tsops,
2224 unsigned nsops, const struct timespec64 *timeout)
2225{
2226 struct sembuf fast_sops[SEMOPM_FAST];
2227 struct sembuf *sops = fast_sops;
2228 struct ipc_namespace *ns;
2229 int ret;
2230
2231 ns = current->nsproxy->ipc_ns;
2232 if (nsops > ns->sc_semopm)
2233 return -E2BIG;
2234 if (nsops < 1)
2235 return -EINVAL;
2236
2237 if (nsops > SEMOPM_FAST) {
Vasily Averin6a4746b2021-09-11 10:40:08 +03002238 sops = kvmalloc_array(nsops, sizeof(*sops), GFP_KERNEL);
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002239 if (sops == NULL)
2240 return -ENOMEM;
2241 }
2242
2243 if (copy_from_user(sops, tsops, nsops * sizeof(*tsops))) {
2244 ret = -EFAULT;
2245 goto out_free;
2246 }
2247
2248 ret = __do_semtimedop(semid, sops, nsops, timeout, ns);
2249
Linus Torvalds1da177e2005-04-16 15:20:36 -07002250out_free:
Manfred Spraul239521f2014-01-27 17:07:04 -08002251 if (sops != fast_sops)
Davidlohr Buesoe4243b82017-09-08 16:17:52 -07002252 kvfree(sops);
Arnd Bergmannbdec0142021-08-11 08:30:23 +01002253
2254 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002255}
2256
Dominik Brodowski41f4f0e2018-03-20 19:48:14 +01002257long ksys_semtimedop(int semid, struct sembuf __user *tsops,
Arnd Bergmann21fc5382018-04-13 13:58:00 +02002258 unsigned int nsops, const struct __kernel_timespec __user *timeout)
Al Viro44ee4542017-07-09 10:50:14 -04002259{
2260 if (timeout) {
Deepa Dinamani3ef56dc2017-08-02 19:51:10 -07002261 struct timespec64 ts;
2262 if (get_timespec64(&ts, timeout))
Al Viro44ee4542017-07-09 10:50:14 -04002263 return -EFAULT;
2264 return do_semtimedop(semid, tsops, nsops, &ts);
2265 }
2266 return do_semtimedop(semid, tsops, nsops, NULL);
2267}
2268
Dominik Brodowski41f4f0e2018-03-20 19:48:14 +01002269SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
Arnd Bergmann21fc5382018-04-13 13:58:00 +02002270 unsigned int, nsops, const struct __kernel_timespec __user *, timeout)
Dominik Brodowski41f4f0e2018-03-20 19:48:14 +01002271{
2272 return ksys_semtimedop(semid, tsops, nsops, timeout);
2273}
2274
Arnd Bergmannb0d17572018-04-13 13:58:23 +02002275#ifdef CONFIG_COMPAT_32BIT_TIME
Dominik Brodowski41f4f0e2018-03-20 19:48:14 +01002276long compat_ksys_semtimedop(int semid, struct sembuf __user *tsems,
2277 unsigned int nsops,
Arnd Bergmann9afc5ee2018-07-13 12:52:28 +02002278 const struct old_timespec32 __user *timeout)
Al Viro44ee4542017-07-09 10:50:14 -04002279{
2280 if (timeout) {
Deepa Dinamani3ef56dc2017-08-02 19:51:10 -07002281 struct timespec64 ts;
Arnd Bergmann9afc5ee2018-07-13 12:52:28 +02002282 if (get_old_timespec32(&ts, timeout))
Al Viro44ee4542017-07-09 10:50:14 -04002283 return -EFAULT;
2284 return do_semtimedop(semid, tsems, nsops, &ts);
2285 }
2286 return do_semtimedop(semid, tsems, nsops, NULL);
2287}
Dominik Brodowski41f4f0e2018-03-20 19:48:14 +01002288
Arnd Bergmann8dabe722019-01-07 00:33:08 +01002289SYSCALL_DEFINE4(semtimedop_time32, int, semid, struct sembuf __user *, tsems,
Dominik Brodowski41f4f0e2018-03-20 19:48:14 +01002290 unsigned int, nsops,
Arnd Bergmann9afc5ee2018-07-13 12:52:28 +02002291 const struct old_timespec32 __user *, timeout)
Dominik Brodowski41f4f0e2018-03-20 19:48:14 +01002292{
2293 return compat_ksys_semtimedop(semid, tsems, nsops, timeout);
2294}
Al Viro44ee4542017-07-09 10:50:14 -04002295#endif
2296
Heiko Carstensd5460c92009-01-14 14:14:27 +01002297SYSCALL_DEFINE3(semop, int, semid, struct sembuf __user *, tsops,
2298 unsigned, nsops)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002299{
Al Viro44ee4542017-07-09 10:50:14 -04002300 return do_semtimedop(semid, tsops, nsops, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002301}
2302
2303/* If CLONE_SYSVSEM is set, establish sharing of SEM_UNDO state between
2304 * parent and child tasks.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002305 */
2306
2307int copy_semundo(unsigned long clone_flags, struct task_struct *tsk)
2308{
2309 struct sem_undo_list *undo_list;
2310 int error;
2311
2312 if (clone_flags & CLONE_SYSVSEM) {
2313 error = get_undo_list(&undo_list);
2314 if (error)
2315 return error;
Elena Reshetovaf74370b2017-09-08 16:17:42 -07002316 refcount_inc(&undo_list->refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002317 tsk->sysvsem.undo_list = undo_list;
Paul McQuade46c0a8c2014-06-06 14:37:37 -07002318 } else
Linus Torvalds1da177e2005-04-16 15:20:36 -07002319 tsk->sysvsem.undo_list = NULL;
2320
2321 return 0;
2322}
2323
2324/*
2325 * add semadj values to semaphores, free undo structures.
2326 * undo structures are not freed when semaphore arrays are destroyed
2327 * so some of them may be out of date.
2328 * IMPLEMENTATION NOTE: There is some confusion over whether the
2329 * set of adjustments that needs to be done should be done in an atomic
2330 * manner or not. That is, if we are attempting to decrement the semval
2331 * should we queue up and wait until we can do so legally?
2332 * The original implementation attempted to do this (queue and wait).
2333 * The current implementation does not do so. The POSIX standard
2334 * and SVID should be consulted to determine what behavior is mandated.
2335 */
2336void exit_sem(struct task_struct *tsk)
2337{
Manfred Spraul4daa28f2008-07-25 01:48:04 -07002338 struct sem_undo_list *ulp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002339
Manfred Spraul4daa28f2008-07-25 01:48:04 -07002340 ulp = tsk->sysvsem.undo_list;
2341 if (!ulp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002342 return;
Manfred Spraul9edff4a2008-04-29 01:00:57 -07002343 tsk->sysvsem.undo_list = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002344
Elena Reshetovaf74370b2017-09-08 16:17:42 -07002345 if (!refcount_dec_and_test(&ulp->refcnt))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002346 return;
2347
Manfred Spraul380af1b2008-07-25 01:48:06 -07002348 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002349 struct sem_array *sma;
Manfred Spraul380af1b2008-07-25 01:48:06 -07002350 struct sem_undo *un;
Rik van Riel6062a8d2013-04-30 19:15:44 -07002351 int semid, i;
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08002352 DEFINE_WAKE_Q(wake_q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002353
Nikolay Borisov2a1613a2016-10-11 13:55:05 -07002354 cond_resched();
2355
Manfred Spraul380af1b2008-07-25 01:48:06 -07002356 rcu_read_lock();
Jiri Pirko05725f72009-04-14 20:17:16 +02002357 un = list_entry_rcu(ulp->list_proc.next,
2358 struct sem_undo, list_proc);
Herton R. Krzesinski602b8592015-08-14 15:35:02 -07002359 if (&un->list_proc == &ulp->list_proc) {
2360 /*
2361 * We must wait for freeary() before freeing this ulp,
2362 * in case we raced with last sem_undo. There is a small
2363 * possibility where we exit while freeary() didn't
2364 * finish unlocking sem_undo_list.
2365 */
Paul E. McKenneye0892e02017-06-29 12:59:32 -07002366 spin_lock(&ulp->lock);
2367 spin_unlock(&ulp->lock);
Rik van Riel6062a8d2013-04-30 19:15:44 -07002368 rcu_read_unlock();
Manfred Spraul380af1b2008-07-25 01:48:06 -07002369 break;
Rik van Riel6062a8d2013-04-30 19:15:44 -07002370 }
Herton R. Krzesinski602b8592015-08-14 15:35:02 -07002371 spin_lock(&ulp->lock);
2372 semid = un->semid;
2373 spin_unlock(&ulp->lock);
Manfred Spraul380af1b2008-07-25 01:48:06 -07002374
Herton R. Krzesinski602b8592015-08-14 15:35:02 -07002375 /* exit_sem raced with IPC_RMID, nothing to do */
2376 if (semid == -1) {
2377 rcu_read_unlock();
2378 continue;
2379 }
2380
2381 sma = sem_obtain_object_check(tsk->nsproxy->ipc_ns, semid);
Manfred Spraul380af1b2008-07-25 01:48:06 -07002382 /* exit_sem raced with IPC_RMID, nothing to do */
Rik van Riel6062a8d2013-04-30 19:15:44 -07002383 if (IS_ERR(sma)) {
2384 rcu_read_unlock();
Manfred Spraul380af1b2008-07-25 01:48:06 -07002385 continue;
Rik van Riel6062a8d2013-04-30 19:15:44 -07002386 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002387
Rik van Riel6062a8d2013-04-30 19:15:44 -07002388 sem_lock(sma, NULL, -1);
Manfred Spraul6e224f92013-10-16 13:46:45 -07002389 /* exit_sem raced with IPC_RMID, nothing to do */
Rafael Aquini0f3d2b02014-01-27 17:07:01 -08002390 if (!ipc_valid_object(&sma->sem_perm)) {
Manfred Spraul6e224f92013-10-16 13:46:45 -07002391 sem_unlock(sma, -1);
2392 rcu_read_unlock();
2393 continue;
2394 }
Nick Pigginbf17bb72009-12-15 16:47:28 -08002395 un = __lookup_undo(ulp, semid);
Manfred Spraul380af1b2008-07-25 01:48:06 -07002396 if (un == NULL) {
2397 /* exit_sem raced with IPC_RMID+semget() that created
2398 * exactly the same semid. Nothing to do.
2399 */
Rik van Riel6062a8d2013-04-30 19:15:44 -07002400 sem_unlock(sma, -1);
Linus Torvalds6d49dab2013-05-03 15:04:40 -07002401 rcu_read_unlock();
Manfred Spraul380af1b2008-07-25 01:48:06 -07002402 continue;
2403 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002404
Manfred Spraul380af1b2008-07-25 01:48:06 -07002405 /* remove un from the linked lists */
Davidlohr Buesocf9d5d72013-07-08 16:01:11 -07002406 ipc_assert_locked_object(&sma->sem_perm);
Manfred Spraul4daa28f2008-07-25 01:48:04 -07002407 list_del(&un->list_id);
2408
Ioanna Alifierakiedf28f42020-02-20 20:04:00 -08002409 spin_lock(&ulp->lock);
Manfred Spraul380af1b2008-07-25 01:48:06 -07002410 list_del_rcu(&un->list_proc);
Ioanna Alifierakiedf28f42020-02-20 20:04:00 -08002411 spin_unlock(&ulp->lock);
Manfred Spraul380af1b2008-07-25 01:48:06 -07002412
Manfred Spraul4daa28f2008-07-25 01:48:04 -07002413 /* perform adjustments registered in un */
2414 for (i = 0; i < sma->sem_nsems; i++) {
Manfred Spraul1a233952017-07-12 14:34:38 -07002415 struct sem *semaphore = &sma->sems[i];
Manfred Spraul4daa28f2008-07-25 01:48:04 -07002416 if (un->semadj[i]) {
2417 semaphore->semval += un->semadj[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002418 /*
2419 * Range checks of the new semaphore value,
2420 * not defined by sus:
2421 * - Some unices ignore the undo entirely
2422 * (e.g. HP UX 11i 11.22, Tru64 V5.1)
2423 * - some cap the value (e.g. FreeBSD caps
2424 * at 0, but doesn't enforce SEMVMX)
2425 *
2426 * Linux caps the semaphore value, both at 0
2427 * and at SEMVMX.
2428 *
Manfred Spraul239521f2014-01-27 17:07:04 -08002429 * Manfred <manfred@colorfullife.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -07002430 */
Ingo Molnar5f921ae2006-03-26 01:37:17 -08002431 if (semaphore->semval < 0)
2432 semaphore->semval = 0;
2433 if (semaphore->semval > SEMVMX)
2434 semaphore->semval = SEMVMX;
Eric W. Biederman51d6f262018-03-23 01:11:29 -05002435 ipc_update_pid(&semaphore->sempid, task_tgid(current));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002436 }
2437 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002438 /* maybe some queued-up processes were waiting for this */
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08002439 do_smart_update(sma, NULL, 0, 1, &wake_q);
Rik van Riel6062a8d2013-04-30 19:15:44 -07002440 sem_unlock(sma, -1);
Linus Torvalds6d49dab2013-05-03 15:04:40 -07002441 rcu_read_unlock();
Davidlohr Bueso9ae949f2016-12-14 15:06:34 -08002442 wake_up_q(&wake_q);
Manfred Spraul380af1b2008-07-25 01:48:06 -07002443
Vasily Averinfc37a3b2021-06-30 18:57:09 -07002444 kvfree_rcu(un, rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002445 }
Manfred Spraul4daa28f2008-07-25 01:48:04 -07002446 kfree(ulp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002447}
2448
2449#ifdef CONFIG_PROC_FS
Mike Waychison19b49462005-09-06 15:17:10 -07002450static int sysvipc_sem_proc_show(struct seq_file *s, void *it)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002451{
Eric W. Biederman1efdb692012-02-07 16:54:11 -08002452 struct user_namespace *user_ns = seq_user_ns(s);
Kees Cookade9f912017-08-02 13:32:21 -07002453 struct kern_ipc_perm *ipcp = it;
2454 struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm);
Deepa Dinamanie54d02b2017-08-02 19:51:13 -07002455 time64_t sem_otime;
Manfred Sprauld12e1e52013-07-08 16:01:25 -07002456
Manfred Sprauld8c63372013-09-30 13:45:07 -07002457 /*
2458 * The proc interface isn't aware of sem_lock(), it calls
Manfred Spraul17d056e2021-06-30 18:57:15 -07002459 * ipc_lock_object(), i.e. spin_lock(&sma->sem_perm.lock).
2460 * (in sysvipc_find_ipc)
Manfred Spraul5864a2f2016-10-11 13:54:50 -07002461 * In order to stay compatible with sem_lock(), we must
2462 * enter / leave complex_mode.
Manfred Sprauld8c63372013-09-30 13:45:07 -07002463 */
Manfred Spraul5864a2f2016-10-11 13:54:50 -07002464 complexmode_enter(sma);
Manfred Sprauld8c63372013-09-30 13:45:07 -07002465
Manfred Sprauld12e1e52013-07-08 16:01:25 -07002466 sem_otime = get_semotime(sma);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002467
Joe Perches7f032d62015-04-15 16:17:54 -07002468 seq_printf(s,
Deepa Dinamanie54d02b2017-08-02 19:51:13 -07002469 "%10d %10d %4o %10u %5u %5u %5u %5u %10llu %10llu\n",
Joe Perches7f032d62015-04-15 16:17:54 -07002470 sma->sem_perm.key,
2471 sma->sem_perm.id,
2472 sma->sem_perm.mode,
2473 sma->sem_nsems,
2474 from_kuid_munged(user_ns, sma->sem_perm.uid),
2475 from_kgid_munged(user_ns, sma->sem_perm.gid),
2476 from_kuid_munged(user_ns, sma->sem_perm.cuid),
2477 from_kgid_munged(user_ns, sma->sem_perm.cgid),
2478 sem_otime,
2479 sma->sem_ctime);
2480
Manfred Spraul5864a2f2016-10-11 13:54:50 -07002481 complexmode_tryleave(sma);
2482
Joe Perches7f032d62015-04-15 16:17:54 -07002483 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002484}
2485#endif