Line data Source code
1 : /* -----------------------------------------------------------------------
2 : ffi64.c - Copyright (c) 2013 The Written Word, Inc.
3 : Copyright (c) 2011 Anthony Green
4 : Copyright (c) 2008, 2010 Red Hat, Inc.
5 : Copyright (c) 2002, 2007 Bo Thorsen <bo@suse.de>
6 :
7 : x86-64 Foreign Function Interface
8 :
9 : Permission is hereby granted, free of charge, to any person obtaining
10 : a copy of this software and associated documentation files (the
11 : ``Software''), to deal in the Software without restriction, including
12 : without limitation the rights to use, copy, modify, merge, publish,
13 : distribute, sublicense, and/or sell copies of the Software, and to
14 : permit persons to whom the Software is furnished to do so, subject to
15 : the following conditions:
16 :
17 : The above copyright notice and this permission notice shall be included
18 : in all copies or substantial portions of the Software.
19 :
20 : THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
21 : EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 : MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 : NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 : HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 : WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 : OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 : DEALINGS IN THE SOFTWARE.
28 : ----------------------------------------------------------------------- */
29 :
30 : #include <ffi.h>
31 : #include <ffi_common.h>
32 :
33 : #include <stdlib.h>
34 : #include <stdarg.h>
35 :
36 : #ifdef __x86_64__
37 :
38 : #define MAX_GPR_REGS 6
39 : #define MAX_SSE_REGS 8
40 :
41 : #if defined(__INTEL_COMPILER)
42 : #include "xmmintrin.h"
43 : #define UINT128 __m128
44 : #else
45 : #if defined(__SUNPRO_C)
46 : #include <sunmedia_types.h>
47 : #define UINT128 __m128i
48 : #else
49 : #define UINT128 __int128_t
50 : #endif
51 : #endif
52 :
53 : union big_int_union
54 : {
55 : UINT32 i32;
56 : UINT64 i64;
57 : UINT128 i128;
58 : };
59 :
60 : struct register_args
61 : {
62 : /* Registers for argument passing. */
63 : UINT64 gpr[MAX_GPR_REGS];
64 : union big_int_union sse[MAX_SSE_REGS];
65 : };
66 :
67 : extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
68 : void *raddr, void (*fnaddr)(void), unsigned ssecount);
69 :
70 : /* All reference to register classes here is identical to the code in
71 : gcc/config/i386/i386.c. Do *not* change one without the other. */
72 :
73 : /* Register class used for passing given 64bit part of the argument.
74 : These represent classes as documented by the PS ABI, with the
75 : exception of SSESF, SSEDF classes, that are basically SSE class,
76 : just gcc will use SF or DFmode move instead of DImode to avoid
77 : reformatting penalties.
78 :
79 : Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
80 : whenever possible (upper half does contain padding). */
81 : enum x86_64_reg_class
82 : {
83 : X86_64_NO_CLASS,
84 : X86_64_INTEGER_CLASS,
85 : X86_64_INTEGERSI_CLASS,
86 : X86_64_SSE_CLASS,
87 : X86_64_SSESF_CLASS,
88 : X86_64_SSEDF_CLASS,
89 : X86_64_SSEUP_CLASS,
90 : X86_64_X87_CLASS,
91 : X86_64_X87UP_CLASS,
92 : X86_64_COMPLEX_X87_CLASS,
93 : X86_64_MEMORY_CLASS
94 : };
95 :
96 : #define MAX_CLASSES 4
97 :
98 : #define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
99 :
100 : /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
101 : of this code is to classify each 8bytes of incoming argument by the register
102 : class and assign registers accordingly. */
103 :
104 : /* Return the union class of CLASS1 and CLASS2.
105 : See the x86-64 PS ABI for details. */
106 :
107 : static enum x86_64_reg_class
108 0 : merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
109 : {
110 : /* Rule #1: If both classes are equal, this is the resulting class. */
111 0 : if (class1 == class2)
112 0 : return class1;
113 :
114 : /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
115 : the other class. */
116 0 : if (class1 == X86_64_NO_CLASS)
117 0 : return class2;
118 0 : if (class2 == X86_64_NO_CLASS)
119 0 : return class1;
120 :
121 : /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
122 0 : if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
123 0 : return X86_64_MEMORY_CLASS;
124 :
125 : /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
126 0 : if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
127 0 : || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
128 0 : return X86_64_INTEGERSI_CLASS;
129 0 : if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
130 0 : || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
131 0 : return X86_64_INTEGER_CLASS;
132 :
133 : /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
134 : MEMORY is used. */
135 0 : if (class1 == X86_64_X87_CLASS
136 0 : || class1 == X86_64_X87UP_CLASS
137 0 : || class1 == X86_64_COMPLEX_X87_CLASS
138 0 : || class2 == X86_64_X87_CLASS
139 0 : || class2 == X86_64_X87UP_CLASS
140 0 : || class2 == X86_64_COMPLEX_X87_CLASS)
141 0 : return X86_64_MEMORY_CLASS;
142 :
143 : /* Rule #6: Otherwise class SSE is used. */
144 0 : return X86_64_SSE_CLASS;
145 : }
146 :
147 : /* Classify the argument of type TYPE and mode MODE.
148 : CLASSES will be filled by the register class used to pass each word
149 : of the operand. The number of words is returned. In case the parameter
150 : should be passed in memory, 0 is returned. As a special case for zero
151 : sized containers, classes[0] will be NO_CLASS and 1 is returned.
152 :
153 : See the x86-64 PS ABI for details.
154 : */
155 : static size_t
156 0 : classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
157 : size_t byte_offset)
158 : {
159 0 : switch (type->type)
160 : {
161 : case FFI_TYPE_UINT8:
162 : case FFI_TYPE_SINT8:
163 : case FFI_TYPE_UINT16:
164 : case FFI_TYPE_SINT16:
165 : case FFI_TYPE_UINT32:
166 : case FFI_TYPE_SINT32:
167 : case FFI_TYPE_UINT64:
168 : case FFI_TYPE_SINT64:
169 : case FFI_TYPE_POINTER:
170 : {
171 0 : size_t size = byte_offset + type->size;
172 :
173 0 : if (size <= 4)
174 : {
175 0 : classes[0] = X86_64_INTEGERSI_CLASS;
176 0 : return 1;
177 : }
178 0 : else if (size <= 8)
179 : {
180 0 : classes[0] = X86_64_INTEGER_CLASS;
181 0 : return 1;
182 : }
183 0 : else if (size <= 12)
184 : {
185 0 : classes[0] = X86_64_INTEGER_CLASS;
186 0 : classes[1] = X86_64_INTEGERSI_CLASS;
187 0 : return 2;
188 : }
189 0 : else if (size <= 16)
190 : {
191 0 : classes[0] = classes[1] = X86_64_INTEGERSI_CLASS;
192 0 : return 2;
193 : }
194 : else
195 0 : FFI_ASSERT (0);
196 : }
197 : case FFI_TYPE_FLOAT:
198 0 : if (!(byte_offset % 8))
199 0 : classes[0] = X86_64_SSESF_CLASS;
200 : else
201 0 : classes[0] = X86_64_SSE_CLASS;
202 0 : return 1;
203 : case FFI_TYPE_DOUBLE:
204 0 : classes[0] = X86_64_SSEDF_CLASS;
205 0 : return 1;
206 : #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
207 : case FFI_TYPE_LONGDOUBLE:
208 : classes[0] = X86_64_X87_CLASS;
209 : classes[1] = X86_64_X87UP_CLASS;
210 : return 2;
211 : #endif
212 : case FFI_TYPE_STRUCT:
213 : {
214 0 : const size_t UNITS_PER_WORD = 8;
215 0 : size_t words = (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
216 : ffi_type **ptr;
217 : int i;
218 : enum x86_64_reg_class subclasses[MAX_CLASSES];
219 :
220 : /* If the struct is larger than 32 bytes, pass it on the stack. */
221 0 : if (type->size > 32)
222 0 : return 0;
223 :
224 0 : for (i = 0; i < words; i++)
225 0 : classes[i] = X86_64_NO_CLASS;
226 :
227 : /* Zero sized arrays or structures are NO_CLASS. We return 0 to
228 : signalize memory class, so handle it as special case. */
229 0 : if (!words)
230 : {
231 0 : classes[0] = X86_64_NO_CLASS;
232 0 : return 1;
233 : }
234 :
235 : /* Merge the fields of structure. */
236 0 : for (ptr = type->elements; *ptr != NULL; ptr++)
237 : {
238 : size_t num;
239 :
240 0 : byte_offset = ALIGN (byte_offset, (*ptr)->alignment);
241 :
242 0 : num = classify_argument (*ptr, subclasses, byte_offset % 8);
243 0 : if (num == 0)
244 0 : return 0;
245 0 : for (i = 0; i < num; i++)
246 : {
247 0 : size_t pos = byte_offset / 8;
248 0 : classes[i + pos] =
249 0 : merge_classes (subclasses[i], classes[i + pos]);
250 : }
251 :
252 0 : byte_offset += (*ptr)->size;
253 : }
254 :
255 0 : if (words > 2)
256 : {
257 : /* When size > 16 bytes, if the first one isn't
258 : X86_64_SSE_CLASS or any other ones aren't
259 : X86_64_SSEUP_CLASS, everything should be passed in
260 : memory. */
261 0 : if (classes[0] != X86_64_SSE_CLASS)
262 0 : return 0;
263 :
264 0 : for (i = 1; i < words; i++)
265 0 : if (classes[i] != X86_64_SSEUP_CLASS)
266 0 : return 0;
267 : }
268 :
269 : /* Final merger cleanup. */
270 0 : for (i = 0; i < words; i++)
271 : {
272 : /* If one class is MEMORY, everything should be passed in
273 : memory. */
274 0 : if (classes[i] == X86_64_MEMORY_CLASS)
275 0 : return 0;
276 :
277 : /* The X86_64_SSEUP_CLASS should be always preceded by
278 : X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
279 0 : if (classes[i] == X86_64_SSEUP_CLASS
280 0 : && classes[i - 1] != X86_64_SSE_CLASS
281 0 : && classes[i - 1] != X86_64_SSEUP_CLASS)
282 : {
283 : /* The first one should never be X86_64_SSEUP_CLASS. */
284 0 : FFI_ASSERT (i != 0);
285 0 : classes[i] = X86_64_SSE_CLASS;
286 : }
287 :
288 : /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
289 : everything should be passed in memory. */
290 0 : if (classes[i] == X86_64_X87UP_CLASS
291 0 : && (classes[i - 1] != X86_64_X87_CLASS))
292 : {
293 : /* The first one should never be X86_64_X87UP_CLASS. */
294 0 : FFI_ASSERT (i != 0);
295 0 : return 0;
296 : }
297 : }
298 0 : return words;
299 : }
300 :
301 : default:
302 0 : FFI_ASSERT(0);
303 : }
304 0 : return 0; /* Never reached. */
305 : }
306 :
307 : /* Examine the argument and return set number of register required in each
308 : class. Return zero iff parameter should be passed in memory, otherwise
309 : the number of registers. */
310 :
311 : static size_t
312 0 : examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
313 : _Bool in_return, int *pngpr, int *pnsse)
314 : {
315 : size_t n;
316 : int i, ngpr, nsse;
317 :
318 0 : n = classify_argument (type, classes, 0);
319 0 : if (n == 0)
320 0 : return 0;
321 :
322 0 : ngpr = nsse = 0;
323 0 : for (i = 0; i < n; ++i)
324 0 : switch (classes[i])
325 : {
326 : case X86_64_INTEGER_CLASS:
327 : case X86_64_INTEGERSI_CLASS:
328 0 : ngpr++;
329 0 : break;
330 : case X86_64_SSE_CLASS:
331 : case X86_64_SSESF_CLASS:
332 : case X86_64_SSEDF_CLASS:
333 0 : nsse++;
334 0 : break;
335 : case X86_64_NO_CLASS:
336 : case X86_64_SSEUP_CLASS:
337 0 : break;
338 : case X86_64_X87_CLASS:
339 : case X86_64_X87UP_CLASS:
340 : case X86_64_COMPLEX_X87_CLASS:
341 0 : return in_return != 0;
342 : default:
343 0 : abort ();
344 : }
345 :
346 0 : *pngpr = ngpr;
347 0 : *pnsse = nsse;
348 :
349 0 : return n;
350 : }
351 :
352 : /* Perform machine dependent cif processing. */
353 :
354 : ffi_status
355 0 : ffi_prep_cif_machdep (ffi_cif *cif)
356 : {
357 : int gprcount, ssecount, i, avn, ngpr, nsse, flags;
358 : enum x86_64_reg_class classes[MAX_CLASSES];
359 : size_t bytes, n;
360 :
361 0 : gprcount = ssecount = 0;
362 :
363 0 : flags = cif->rtype->type;
364 0 : if (flags != FFI_TYPE_VOID)
365 : {
366 0 : n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
367 0 : if (n == 0)
368 : {
369 : /* The return value is passed in memory. A pointer to that
370 : memory is the first argument. Allocate a register for it. */
371 0 : gprcount++;
372 : /* We don't have to do anything in asm for the return. */
373 0 : flags = FFI_TYPE_VOID;
374 : }
375 0 : else if (flags == FFI_TYPE_STRUCT)
376 : {
377 : /* Mark which registers the result appears in. */
378 0 : _Bool sse0 = SSE_CLASS_P (classes[0]);
379 0 : _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
380 0 : if (sse0 && !sse1)
381 0 : flags |= 1 << 8;
382 0 : else if (!sse0 && sse1)
383 0 : flags |= 1 << 9;
384 0 : else if (sse0 && sse1)
385 0 : flags |= 1 << 10;
386 : /* Mark the true size of the structure. */
387 0 : flags |= cif->rtype->size << 12;
388 : }
389 : }
390 :
391 : /* Go over all arguments and determine the way they should be passed.
392 : If it's in a register and there is space for it, let that be so. If
393 : not, add it's size to the stack byte count. */
394 0 : for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
395 : {
396 0 : if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
397 0 : || gprcount + ngpr > MAX_GPR_REGS
398 0 : || ssecount + nsse > MAX_SSE_REGS)
399 0 : {
400 0 : long align = cif->arg_types[i]->alignment;
401 :
402 0 : if (align < 8)
403 0 : align = 8;
404 :
405 0 : bytes = ALIGN (bytes, align);
406 0 : bytes += cif->arg_types[i]->size;
407 : }
408 : else
409 : {
410 0 : gprcount += ngpr;
411 0 : ssecount += nsse;
412 : }
413 : }
414 0 : if (ssecount)
415 0 : flags |= 1 << 11;
416 0 : cif->flags = flags;
417 0 : cif->bytes = (unsigned)ALIGN (bytes, 8);
418 :
419 0 : return FFI_OK;
420 : }
421 :
422 : #ifndef __SANITIZE_ADDRESS__
423 : # ifdef __clang__
424 : # if __has_feature(address_sanitizer)
425 : # define __SANITIZE_ADDRESS__
426 : # endif
427 : # endif
428 : #endif
429 : #ifdef __SANITIZE_ADDRESS__
430 : __attribute__((noinline,no_sanitize_address))
431 : #endif
432 : void
433 0 : ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
434 : {
435 : enum x86_64_reg_class classes[MAX_CLASSES];
436 : char *stack, *argp;
437 : ffi_type **arg_types;
438 : int gprcount, ssecount, ngpr, nsse, i, avn;
439 : _Bool ret_in_memory;
440 : struct register_args *reg_args;
441 :
442 : /* Can't call 32-bit mode from 64-bit mode. */
443 0 : FFI_ASSERT (cif->abi == FFI_UNIX64);
444 :
445 : /* If the return value is a struct and we don't have a return value
446 : address then we need to make one. Note the setting of flags to
447 : VOID above in ffi_prep_cif_machdep. */
448 0 : ret_in_memory = (cif->rtype->type == FFI_TYPE_STRUCT
449 0 : && (cif->flags & 0xff) == FFI_TYPE_VOID);
450 0 : if (rvalue == NULL && ret_in_memory)
451 0 : rvalue = alloca (cif->rtype->size);
452 :
453 : /* Allocate the space for the arguments, plus 4 words of temp space. */
454 0 : stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
455 0 : reg_args = (struct register_args *) stack;
456 0 : argp = stack + sizeof (struct register_args);
457 :
458 0 : gprcount = ssecount = 0;
459 :
460 : /* If the return value is passed in memory, add the pointer as the
461 : first integer argument. */
462 0 : if (ret_in_memory)
463 0 : reg_args->gpr[gprcount++] = (unsigned long) rvalue;
464 :
465 0 : avn = cif->nargs;
466 0 : arg_types = cif->arg_types;
467 :
468 0 : for (i = 0; i < avn; ++i)
469 : {
470 0 : size_t n, size = arg_types[i]->size;
471 :
472 0 : n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
473 0 : if (n == 0
474 0 : || gprcount + ngpr > MAX_GPR_REGS
475 0 : || ssecount + nsse > MAX_SSE_REGS)
476 0 : {
477 0 : long align = arg_types[i]->alignment;
478 :
479 : /* Stack arguments are *always* at least 8 byte aligned. */
480 0 : if (align < 8)
481 0 : align = 8;
482 :
483 : /* Pass this argument in memory. */
484 0 : argp = (void *) ALIGN (argp, align);
485 0 : memcpy (argp, avalue[i], size);
486 0 : argp += size;
487 : }
488 : else
489 : {
490 : /* The argument is passed entirely in registers. */
491 0 : char *a = (char *) avalue[i];
492 : int j;
493 :
494 0 : for (j = 0; j < n; j++, a += 8, size -= 8)
495 : {
496 0 : switch (classes[j])
497 : {
498 : case X86_64_INTEGER_CLASS:
499 : case X86_64_INTEGERSI_CLASS:
500 : /* Sign-extend integer arguments passed in general
501 : purpose registers, to cope with the fact that
502 : LLVM incorrectly assumes that this will be done
503 : (the x86-64 PS ABI does not specify this). */
504 0 : switch (arg_types[i]->type)
505 : {
506 : case FFI_TYPE_SINT8:
507 0 : *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT8 *) a);
508 0 : break;
509 : case FFI_TYPE_SINT16:
510 0 : *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT16 *) a);
511 0 : break;
512 : case FFI_TYPE_SINT32:
513 0 : *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT32 *) a);
514 0 : break;
515 : default:
516 0 : reg_args->gpr[gprcount] = 0;
517 0 : memcpy (®_args->gpr[gprcount], a, size < 8 ? size : 8);
518 : }
519 0 : gprcount++;
520 0 : break;
521 : case X86_64_SSE_CLASS:
522 : case X86_64_SSEDF_CLASS:
523 0 : reg_args->sse[ssecount++].i64 = *(UINT64 *) a;
524 0 : break;
525 : case X86_64_SSESF_CLASS:
526 0 : reg_args->sse[ssecount++].i32 = *(UINT32 *) a;
527 0 : break;
528 : default:
529 0 : abort();
530 : }
531 : }
532 : }
533 : }
534 :
535 0 : ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
536 : cif->flags, rvalue, fn, ssecount);
537 0 : }
538 :
539 :
540 : extern void ffi_closure_unix64(void);
541 :
542 : ffi_status
543 0 : ffi_prep_closure_loc (ffi_closure* closure,
544 : ffi_cif* cif,
545 : void (*fun)(ffi_cif*, void*, void**, void*),
546 : void *user_data,
547 : void *codeloc)
548 : {
549 : volatile unsigned short *tramp;
550 :
551 : /* Sanity check on the cif ABI. */
552 : {
553 0 : int abi = cif->abi;
554 0 : if (UNLIKELY (! (abi > FFI_FIRST_ABI && abi < FFI_LAST_ABI)))
555 0 : return FFI_BAD_ABI;
556 : }
557 :
558 0 : tramp = (volatile unsigned short *) &closure->tramp[0];
559 :
560 0 : tramp[0] = 0xbb49; /* mov <code>, %r11 */
561 0 : *((unsigned long long * volatile) &tramp[1])
562 0 : = (unsigned long) ffi_closure_unix64;
563 0 : tramp[5] = 0xba49; /* mov <data>, %r10 */
564 0 : *((unsigned long long * volatile) &tramp[6])
565 0 : = (unsigned long) codeloc;
566 :
567 : /* Set the carry bit iff the function uses any sse registers.
568 : This is clc or stc, together with the first byte of the jmp. */
569 0 : tramp[10] = cif->flags & (1 << 11) ? 0x49f9 : 0x49f8;
570 :
571 0 : tramp[11] = 0xe3ff; /* jmp *%r11 */
572 :
573 0 : closure->cif = cif;
574 0 : closure->fun = fun;
575 0 : closure->user_data = user_data;
576 :
577 0 : return FFI_OK;
578 : }
579 :
580 : int
581 0 : ffi_closure_unix64_inner(ffi_closure *closure, void *rvalue,
582 : struct register_args *reg_args, char *argp)
583 : {
584 : ffi_cif *cif;
585 : void **avalue;
586 : ffi_type **arg_types;
587 : long i, avn;
588 : int gprcount, ssecount, ngpr, nsse;
589 : int ret;
590 :
591 0 : cif = closure->cif;
592 0 : avalue = alloca(cif->nargs * sizeof(void *));
593 0 : gprcount = ssecount = 0;
594 :
595 0 : ret = cif->rtype->type;
596 0 : if (ret != FFI_TYPE_VOID)
597 : {
598 : enum x86_64_reg_class classes[MAX_CLASSES];
599 0 : size_t n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
600 0 : if (n == 0)
601 : {
602 : /* The return value goes in memory. Arrange for the closure
603 : return value to go directly back to the original caller. */
604 0 : rvalue = (void *) (unsigned long) reg_args->gpr[gprcount++];
605 : /* We don't have to do anything in asm for the return. */
606 0 : ret = FFI_TYPE_VOID;
607 : }
608 0 : else if (ret == FFI_TYPE_STRUCT && n == 2)
609 : {
610 : /* Mark which register the second word of the structure goes in. */
611 0 : _Bool sse0 = SSE_CLASS_P (classes[0]);
612 0 : _Bool sse1 = SSE_CLASS_P (classes[1]);
613 0 : if (!sse0 && sse1)
614 0 : ret |= 1 << 8;
615 0 : else if (sse0 && !sse1)
616 0 : ret |= 1 << 9;
617 : }
618 : }
619 :
620 0 : avn = cif->nargs;
621 0 : arg_types = cif->arg_types;
622 :
623 0 : for (i = 0; i < avn; ++i)
624 : {
625 : enum x86_64_reg_class classes[MAX_CLASSES];
626 : size_t n;
627 :
628 0 : n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
629 0 : if (n == 0
630 0 : || gprcount + ngpr > MAX_GPR_REGS
631 0 : || ssecount + nsse > MAX_SSE_REGS)
632 0 : {
633 0 : long align = arg_types[i]->alignment;
634 :
635 : /* Stack arguments are *always* at least 8 byte aligned. */
636 0 : if (align < 8)
637 0 : align = 8;
638 :
639 : /* Pass this argument in memory. */
640 0 : argp = (void *) ALIGN (argp, align);
641 0 : avalue[i] = argp;
642 0 : argp += arg_types[i]->size;
643 : }
644 : /* If the argument is in a single register, or two consecutive
645 : integer registers, then we can use that address directly. */
646 0 : else if (n == 1
647 0 : || (n == 2 && !(SSE_CLASS_P (classes[0])
648 0 : || SSE_CLASS_P (classes[1]))))
649 : {
650 : /* The argument is in a single register. */
651 0 : if (SSE_CLASS_P (classes[0]))
652 : {
653 0 : avalue[i] = ®_args->sse[ssecount];
654 0 : ssecount += n;
655 : }
656 : else
657 : {
658 0 : avalue[i] = ®_args->gpr[gprcount];
659 0 : gprcount += n;
660 : }
661 : }
662 : /* Otherwise, allocate space to make them consecutive. */
663 : else
664 : {
665 0 : char *a = alloca (16);
666 : int j;
667 :
668 0 : avalue[i] = a;
669 0 : for (j = 0; j < n; j++, a += 8)
670 : {
671 0 : if (SSE_CLASS_P (classes[j]))
672 0 : memcpy (a, ®_args->sse[ssecount++], 8);
673 : else
674 0 : memcpy (a, ®_args->gpr[gprcount++], 8);
675 : }
676 : }
677 : }
678 :
679 : /* Invoke the closure. */
680 0 : closure->fun (cif, rvalue, avalue, closure->user_data);
681 :
682 : /* Tell assembly how to perform return type promotions. */
683 0 : return ret;
684 : }
685 :
686 : #endif /* __x86_64__ */
|
