Line data Source code
1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 : // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
4 : // Use of this source code is governed by a BSD-style license that can be
5 : // found in the LICENSE file.
6 :
7 : // This file is an internal atomic implementation, use base/atomicops.h instead.
8 :
9 : #ifndef BASE_ATOMICOPS_INTERNALS_X86_GCC_H_
10 : #define BASE_ATOMICOPS_INTERNALS_X86_GCC_H_
11 :
12 : // This struct is not part of the public API of this module; clients may not
13 : // use it.
14 : // Features of this x86. Values may not be correct before main() is run,
15 : // but are set conservatively.
16 : struct AtomicOps_x86CPUFeatureStruct {
17 : bool has_amd_lock_mb_bug; // Processor has AMD memory-barrier bug; do lfence
18 : // after acquire compare-and-swap.
19 : bool has_sse2; // Processor has SSE2.
20 : };
21 : extern struct AtomicOps_x86CPUFeatureStruct AtomicOps_Internalx86CPUFeatures;
22 :
23 : #define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory")
24 :
25 : namespace base {
26 : namespace subtle {
27 :
28 : // 32-bit low-level operations on any platform.
29 :
30 : inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr,
31 : Atomic32 old_value,
32 : Atomic32 new_value) {
33 : Atomic32 prev;
34 : __asm__ __volatile__("lock; cmpxchgl %1,%2"
35 : : "=a" (prev)
36 : : "q" (new_value), "m" (*ptr), "0" (old_value)
37 : : "memory");
38 : return prev;
39 : }
40 :
41 : inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr,
42 : Atomic32 new_value) {
43 : __asm__ __volatile__("xchgl %1,%0" // The lock prefix is implicit for xchg.
44 : : "=r" (new_value)
45 : : "m" (*ptr), "0" (new_value)
46 : : "memory");
47 : return new_value; // Now it's the previous value.
48 : }
49 :
50 15 : inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr,
51 : Atomic32 increment) {
52 15 : Atomic32 temp = increment;
53 : __asm__ __volatile__("lock; xaddl %0,%1"
54 : : "+r" (temp), "+m" (*ptr)
55 15 : : : "memory");
56 : // temp now holds the old value of *ptr
57 15 : return temp + increment;
58 : }
59 :
60 : inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr,
61 : Atomic32 increment) {
62 : Atomic32 temp = increment;
63 : __asm__ __volatile__("lock; xaddl %0,%1"
64 : : "+r" (temp), "+m" (*ptr)
65 : : : "memory");
66 : // temp now holds the old value of *ptr
67 : if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) {
68 : __asm__ __volatile__("lfence" : : : "memory");
69 : }
70 : return temp + increment;
71 : }
72 :
73 : inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr,
74 : Atomic32 old_value,
75 : Atomic32 new_value) {
76 : Atomic32 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value);
77 : if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) {
78 : __asm__ __volatile__("lfence" : : : "memory");
79 : }
80 : return x;
81 : }
82 :
83 : inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr,
84 : Atomic32 old_value,
85 : Atomic32 new_value) {
86 : return NoBarrier_CompareAndSwap(ptr, old_value, new_value);
87 : }
88 :
89 : inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) {
90 : *ptr = value;
91 : }
92 :
93 : #if defined(__x86_64__)
94 :
95 : // 64-bit implementations of memory barrier can be simpler, because it
96 : // "mfence" is guaranteed to exist.
97 : inline void MemoryBarrier() {
98 : __asm__ __volatile__("mfence" : : : "memory");
99 : }
100 :
101 : inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) {
102 : *ptr = value;
103 : MemoryBarrier();
104 : }
105 :
106 : #else
107 :
108 : inline void MemoryBarrier() {
109 : if (AtomicOps_Internalx86CPUFeatures.has_sse2) {
110 : __asm__ __volatile__("mfence" : : : "memory");
111 : } else { // mfence is faster but not present on PIII
112 : Atomic32 x = 0;
113 : NoBarrier_AtomicExchange(&x, 0); // acts as a barrier on PIII
114 : }
115 : }
116 :
117 : inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) {
118 : if (AtomicOps_Internalx86CPUFeatures.has_sse2) {
119 : *ptr = value;
120 : __asm__ __volatile__("mfence" : : : "memory");
121 : } else {
122 : NoBarrier_AtomicExchange(ptr, value);
123 : // acts as a barrier on PIII
124 : }
125 : }
126 : #endif
127 :
128 : inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) {
129 : ATOMICOPS_COMPILER_BARRIER();
130 : *ptr = value; // An x86 store acts as a release barrier.
131 : // See comments in Atomic64 version of Release_Store(), below.
132 : }
133 :
134 : inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) {
135 : return *ptr;
136 : }
137 :
138 : inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) {
139 : Atomic32 value = *ptr; // An x86 load acts as a acquire barrier.
140 : // See comments in Atomic64 version of Release_Store(), below.
141 : ATOMICOPS_COMPILER_BARRIER();
142 : return value;
143 : }
144 :
145 : inline Atomic32 Release_Load(volatile const Atomic32* ptr) {
146 : MemoryBarrier();
147 : return *ptr;
148 : }
149 :
150 : #if defined(__x86_64__)
151 :
152 : // 64-bit low-level operations on 64-bit platform.
153 :
154 3 : inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr,
155 : Atomic64 old_value,
156 : Atomic64 new_value) {
157 : Atomic64 prev;
158 : __asm__ __volatile__("lock; cmpxchgq %1,%2"
159 : : "=a" (prev)
160 : : "q" (new_value), "m" (*ptr), "0" (old_value)
161 3 : : "memory");
162 3 : return prev;
163 : }
164 :
165 0 : inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr,
166 : Atomic64 new_value) {
167 : __asm__ __volatile__("xchgq %1,%0" // The lock prefix is implicit for xchg.
168 : : "=r" (new_value)
169 : : "m" (*ptr), "0" (new_value)
170 0 : : "memory");
171 0 : return new_value; // Now it's the previous value.
172 : }
173 :
174 : inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr,
175 : Atomic64 increment) {
176 : Atomic64 temp = increment;
177 : __asm__ __volatile__("lock; xaddq %0,%1"
178 : : "+r" (temp), "+m" (*ptr)
179 : : : "memory");
180 : // temp now contains the previous value of *ptr
181 : return temp + increment;
182 : }
183 :
184 : inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr,
185 : Atomic64 increment) {
186 : Atomic64 temp = increment;
187 : __asm__ __volatile__("lock; xaddq %0,%1"
188 : : "+r" (temp), "+m" (*ptr)
189 : : : "memory");
190 : // temp now contains the previous value of *ptr
191 : if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) {
192 : __asm__ __volatile__("lfence" : : : "memory");
193 : }
194 : return temp + increment;
195 : }
196 :
197 : inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) {
198 : *ptr = value;
199 : }
200 :
201 3 : inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr,
202 : Atomic64 old_value,
203 : Atomic64 new_value) {
204 3 : Atomic64 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value);
205 : /* XXX/cjones: no idea if this is necessary... */
206 3 : if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) {
207 0 : __asm__ __volatile__("lfence" : : : "memory");
208 : }
209 3 : return x;
210 : }
211 :
212 : inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) {
213 : *ptr = value;
214 : MemoryBarrier();
215 : }
216 :
217 3 : inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) {
218 3 : ATOMICOPS_COMPILER_BARRIER();
219 :
220 3 : *ptr = value; // An x86 store acts as a release barrier
221 : // for current AMD/Intel chips as of Jan 2008.
222 : // See also Acquire_Load(), below.
223 :
224 : // When new chips come out, check:
225 : // IA-32 Intel Architecture Software Developer's Manual, Volume 3:
226 : // System Programming Guide, Chatper 7: Multiple-processor management,
227 : // Section 7.2, Memory Ordering.
228 : // Last seen at:
229 : // http://developer.intel.com/design/pentium4/manuals/index_new.htm
230 : //
231 : // x86 stores/loads fail to act as barriers for a few instructions (clflush
232 : // maskmovdqu maskmovq movntdq movnti movntpd movntps movntq) but these are
233 : // not generated by the compiler, and are rare. Users of these instructions
234 : // need to know about cache behaviour in any case since all of these involve
235 : // either flushing cache lines or non-temporal cache hints.
236 3 : }
237 :
238 32 : inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) {
239 32 : return *ptr;
240 : }
241 :
242 : inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) {
243 : Atomic64 value = *ptr; // An x86 load acts as a acquire barrier,
244 : // for current AMD/Intel chips as of Jan 2008.
245 : // See also Release_Store(), above.
246 : ATOMICOPS_COMPILER_BARRIER();
247 : return value;
248 : }
249 :
250 : inline Atomic64 Release_Load(volatile const Atomic64* ptr) {
251 : MemoryBarrier();
252 : return *ptr;
253 : }
254 : #endif // defined(__x86_64__)
255 :
256 : } // namespace base::subtle
257 : } // namespace base
258 :
259 : #undef ATOMICOPS_COMPILER_BARRIER
260 :
261 : #endif // BASE_ATOMICOPS_INTERNALS_X86_GCC_H_
|
