Line data Source code
1 : /*-
2 : * Copyright (c) 2009-2010 Brad Penoff
3 : * Copyright (c) 2009-2010 Humaira Kamal
4 : * Copyright (c) 2011-2012 Irene Ruengeler
5 : * Copyright (c) 2011-2012 Michael Tuexen
6 : *
7 : * All rights reserved.
8 : *
9 : * Redistribution and use in source and binary forms, with or without
10 : * modification, are permitted provided that the following conditions
11 : * are met:
12 : * 1. Redistributions of source code must retain the above copyright
13 : * notice, this list of conditions and the following disclaimer.
14 : * 2. Redistributions in binary form must reproduce the above copyright
15 : * notice, this list of conditions and the following disclaimer in the
16 : * documentation and/or other materials provided with the distribution.
17 : *
18 : * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 : * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 : * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 : * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 : * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 : * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 : * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 : * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 : * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 : * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 : * SUCH DAMAGE.
29 : */
30 :
31 : #ifndef _USER_ATOMIC_H_
32 : #define _USER_ATOMIC_H_
33 :
34 : /* __Userspace__ version of sys/i386/include/atomic.h goes here */
35 :
36 : /* TODO In the future, might want to not use i386 specific assembly.
37 : * The options include:
38 : * - implement them generically (but maybe not truly atomic?) in userspace
39 : * - have ifdef's for __Userspace_arch_ perhaps (OS isn't enough...)
40 : */
41 :
42 : #include <stdio.h>
43 : #include <sys/types.h>
44 :
45 : #if defined(__Userspace_os_Darwin) || defined (__Userspace_os_Windows)
46 : #if defined (__Userspace_os_Windows)
47 : #define atomic_add_int(addr, val) InterlockedExchangeAdd((LPLONG)addr, (LONG)val)
48 : #define atomic_fetchadd_int(addr, val) InterlockedExchangeAdd((LPLONG)addr, (LONG)val)
49 : #define atomic_subtract_int(addr, val) InterlockedExchangeAdd((LPLONG)addr,-((LONG)val))
50 : #define atomic_cmpset_int(dst, exp, src) InterlockedCompareExchange((LPLONG)dst, src, exp)
51 : #define SCTP_DECREMENT_AND_CHECK_REFCOUNT(addr) (InterlockedExchangeAdd((LPLONG)addr, (-1L)) == 1)
52 : #else
53 : #include <libkern/OSAtomic.h>
54 : #define atomic_add_int(addr, val) OSAtomicAdd32Barrier(val, (int32_t *)addr)
55 : #define atomic_fetchadd_int(addr, val) OSAtomicAdd32Barrier(val, (int32_t *)addr)
56 : #define atomic_subtract_int(addr, val) OSAtomicAdd32Barrier(-val, (int32_t *)addr)
57 : #define atomic_cmpset_int(dst, exp, src) OSAtomicCompareAndSwapIntBarrier(exp, src, (int *)dst)
58 : #define SCTP_DECREMENT_AND_CHECK_REFCOUNT(addr) (atomic_fetchadd_int(addr, -1) == 0)
59 : #endif
60 :
61 : #if defined(INVARIANTS)
62 : #define SCTP_SAVE_ATOMIC_DECREMENT(addr, val) \
63 : { \
64 : int32_t newval; \
65 : newval = atomic_fetchadd_int(addr, -val); \
66 : if (newval < 0) { \
67 : panic("Counter goes negative"); \
68 : } \
69 : }
70 : #else
71 : #define SCTP_SAVE_ATOMIC_DECREMENT(addr, val) \
72 : { \
73 : int32_t newval; \
74 : newval = atomic_fetchadd_int(addr, -val); \
75 : if (newval < 0) { \
76 : *addr = 0; \
77 : } \
78 : }
79 : #if defined(__Userspace_os_Windows)
80 : static void atomic_init() {} /* empty when we are not using atomic_mtx */
81 : #else
82 : static inline void atomic_init() {} /* empty when we are not using atomic_mtx */
83 : #endif
84 : #endif
85 :
86 : #else
87 : /* Using gcc built-in functions for atomic memory operations
88 : Reference: http://gcc.gnu.org/onlinedocs/gcc-4.1.0/gcc/Atomic-Builtins.html
89 : Requires gcc version 4.1.0
90 : compile with -march=i486
91 : */
92 :
93 : /*Atomically add V to *P.*/
94 : #define atomic_add_int(P, V) (void) __sync_fetch_and_add(P, V)
95 :
96 : /*Atomically subtrace V from *P.*/
97 : #define atomic_subtract_int(P, V) (void) __sync_fetch_and_sub(P, V)
98 :
99 : /*
100 : * Atomically add the value of v to the integer pointed to by p and return
101 : * the previous value of *p.
102 : */
103 : #define atomic_fetchadd_int(p, v) __sync_fetch_and_add(p, v)
104 :
105 : /* Following explanation from src/sys/i386/include/atomic.h,
106 : * for atomic compare and set
107 : *
108 : * if (*dst == exp) *dst = src (all 32 bit words)
109 : *
110 : * Returns 0 on failure, non-zero on success
111 : */
112 :
113 : #define atomic_cmpset_int(dst, exp, src) __sync_bool_compare_and_swap(dst, exp, src)
114 :
115 : #define SCTP_DECREMENT_AND_CHECK_REFCOUNT(addr) (atomic_fetchadd_int(addr, -1) == 1)
116 : #if defined(INVARIANTS)
117 : #define SCTP_SAVE_ATOMIC_DECREMENT(addr, val) \
118 : { \
119 : int32_t oldval; \
120 : oldval = atomic_fetchadd_int(addr, -val); \
121 : if (oldval < val) { \
122 : panic("Counter goes negative"); \
123 : } \
124 : }
125 : #else
126 : #define SCTP_SAVE_ATOMIC_DECREMENT(addr, val) \
127 : { \
128 : int32_t oldval; \
129 : oldval = atomic_fetchadd_int(addr, -val); \
130 : if (oldval < val) { \
131 : *addr = 0; \
132 : } \
133 : }
134 : #endif
135 0 : static inline void atomic_init() {} /* empty when we are not using atomic_mtx */
136 : #endif
137 :
138 : #if 0 /* using libatomic_ops */
139 : #include "user_include/atomic_ops.h"
140 :
141 : /*Atomically add incr to *P, and return the original value of *P.*/
142 : #define atomic_add_int(P, V) AO_fetch_and_add((AO_t*)P, V)
143 :
144 : #define atomic_subtract_int(P, V) AO_fetch_and_add((AO_t*)P, -(V))
145 :
146 : /*
147 : * Atomically add the value of v to the integer pointed to by p and return
148 : * the previous value of *p.
149 : */
150 : #define atomic_fetchadd_int(p, v) AO_fetch_and_add((AO_t*)p, v)
151 :
152 : /* Atomically compare *addr to old_val, and replace *addr by new_val
153 : if the first comparison succeeds. Returns nonzero if the comparison
154 : succeeded and *addr was updated.
155 : */
156 : /* Following Explanation from src/sys/i386/include/atomic.h, which
157 : matches that of AO_compare_and_swap above.
158 : * Atomic compare and set, used by the mutex functions
159 : *
160 : * if (*dst == exp) *dst = src (all 32 bit words)
161 : *
162 : * Returns 0 on failure, non-zero on success
163 : */
164 :
165 : #define atomic_cmpset_int(dst, exp, src) AO_compare_and_swap((AO_t*)dst, exp, src)
166 :
167 : static inline void atomic_init() {} /* empty when we are not using atomic_mtx */
168 : #endif /* closing #if for libatomic */
169 :
170 : #if 0 /* using atomic_mtx */
171 :
172 : #include <pthread.h>
173 :
174 : extern userland_mutex_t atomic_mtx;
175 :
176 : #if defined (__Userspace_os_Windows)
177 : static inline void atomic_init() {
178 : InitializeCriticalSection(&atomic_mtx);
179 : }
180 : static inline void atomic_destroy() {
181 : DeleteCriticalSection(&atomic_mtx);
182 : }
183 : static inline void atomic_lock() {
184 : EnterCriticalSection(&atomic_mtx);
185 : }
186 : static inline void atomic_unlock() {
187 : LeaveCriticalSection(&atomic_mtx);
188 : }
189 : #else
190 : static inline void atomic_init() {
191 : (void)pthread_mutex_init(&atomic_mtx, NULL);
192 : }
193 : static inline void atomic_destroy() {
194 : (void)pthread_mutex_destroy(&atomic_mtx);
195 : }
196 : static inline void atomic_lock() {
197 : (void)pthread_mutex_lock(&atomic_mtx);
198 : }
199 : static inline void atomic_unlock() {
200 : (void)pthread_mutex_unlock(&atomic_mtx);
201 : }
202 : #endif
203 : /*
204 : * For userland, always use lock prefixes so that the binaries will run
205 : * on both SMP and !SMP systems.
206 : */
207 :
208 : #define MPLOCKED "lock ; "
209 :
210 : /*
211 : * Atomically add the value of v to the integer pointed to by p and return
212 : * the previous value of *p.
213 : */
214 : static __inline u_int
215 : atomic_fetchadd_int(volatile void *n, u_int v)
216 : {
217 : int *p = (int *) n;
218 : atomic_lock();
219 : __asm __volatile(
220 : " " MPLOCKED " "
221 : " xaddl %0, %1 ; "
222 : "# atomic_fetchadd_int"
223 : : "+r" (v), /* 0 (result) */
224 : "=m" (*p) /* 1 */
225 : : "m" (*p)); /* 2 */
226 : atomic_unlock();
227 :
228 : return (v);
229 : }
230 :
231 :
232 : #ifdef CPU_DISABLE_CMPXCHG
233 :
234 : static __inline int
235 : atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src)
236 : {
237 : u_char res;
238 :
239 : atomic_lock();
240 : __asm __volatile(
241 : " pushfl ; "
242 : " cli ; "
243 : " cmpl %3,%4 ; "
244 : " jne 1f ; "
245 : " movl %2,%1 ; "
246 : "1: "
247 : " sete %0 ; "
248 : " popfl ; "
249 : "# atomic_cmpset_int"
250 : : "=q" (res), /* 0 */
251 : "=m" (*dst) /* 1 */
252 : : "r" (src), /* 2 */
253 : "r" (exp), /* 3 */
254 : "m" (*dst) /* 4 */
255 : : "memory");
256 : atomic_unlock();
257 :
258 : return (res);
259 : }
260 :
261 : #else /* !CPU_DISABLE_CMPXCHG */
262 :
263 : static __inline int
264 : atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src)
265 : {
266 : atomic_lock();
267 : u_char res;
268 :
269 : __asm __volatile(
270 : " " MPLOCKED " "
271 : " cmpxchgl %2,%1 ; "
272 : " sete %0 ; "
273 : "1: "
274 : "# atomic_cmpset_int"
275 : : "=a" (res), /* 0 */
276 : "=m" (*dst) /* 1 */
277 : : "r" (src), /* 2 */
278 : "a" (exp), /* 3 */
279 : "m" (*dst) /* 4 */
280 : : "memory");
281 : atomic_unlock();
282 :
283 : return (res);
284 : }
285 :
286 : #endif /* CPU_DISABLE_CMPXCHG */
287 :
288 : #define atomic_add_int(P, V) do { \
289 : atomic_lock(); \
290 : (*(u_int *)(P) += (V)); \
291 : atomic_unlock(); \
292 : } while(0)
293 : #define atomic_subtract_int(P, V) do { \
294 : atomic_lock(); \
295 : (*(u_int *)(P) -= (V)); \
296 : atomic_unlock(); \
297 : } while(0)
298 :
299 : #endif
300 : #endif
|
