1 /* -----------------------------------------------------------------------
2 ffi64.c - Copyright (c) 2013 The Written Word, Inc.
3 Copyright (c) 2011 Anthony Green
4 Copyright (c) 2008, 2010 Red Hat, Inc.
5 Copyright (c) 2002, 2007 Bo Thorsen <bo@suse.de>
6
7 x86-64 Foreign Function Interface
8
9 Permission is hereby granted, free of charge, to any person obtaining
10 a copy of this software and associated documentation files (the
11 ``Software''), to deal in the Software without restriction, including
12 without limitation the rights to use, copy, modify, merge, publish,
13 distribute, sublicense, and/or sell copies of the Software, and to
14 permit persons to whom the Software is furnished to do so, subject to
15 the following conditions:
16
17 The above copyright notice and this permission notice shall be included
18 in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
29
30 #include <ffi.h>
31 #include <ffi_common.h>
32
33 #include <stdlib.h>
34 #include <stdarg.h>
35
36 #ifdef __x86_64__
37
38 #define MAX_GPR_REGS 6
39 #define MAX_SSE_REGS 8
40
41 #if defined(__INTEL_COMPILER)
42 #include "xmmintrin.h"
43 #define UINT128 __m128
44 #else
45 #if defined(__SUNPRO_C)
46 #include <sunmedia_types.h>
47 #define UINT128 __m128i
48 #else
49 #define UINT128 __int128_t
50 #endif
51 #endif
52
53 union big_int_union
54 {
55 UINT32 i32;
56 UINT64 i64;
57 UINT128 i128;
58 };
59
60 struct register_args
61 {
62 /* Registers for argument passing. */
63 UINT64 gpr[MAX_GPR_REGS];
64 union big_int_union sse[MAX_SSE_REGS];
65 };
66
67 extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
68 void *raddr, void (*fnaddr)(void), unsigned ssecount);
69
70 /* All reference to register classes here is identical to the code in
71 gcc/config/i386/i386.c. Do *not* change one without the other. */
72
73 /* Register class used for passing given 64bit part of the argument.
74 These represent classes as documented by the PS ABI, with the
75 exception of SSESF, SSEDF classes, that are basically SSE class,
76 just gcc will use SF or DFmode move instead of DImode to avoid
77 reformatting penalties.
78
79 Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
80 whenever possible (upper half does contain padding). */
81 enum x86_64_reg_class
82 {
83 X86_64_NO_CLASS,
84 X86_64_INTEGER_CLASS,
85 X86_64_INTEGERSI_CLASS,
86 X86_64_SSE_CLASS,
87 X86_64_SSESF_CLASS,
88 X86_64_SSEDF_CLASS,
150 should be passed in memory, 0 is returned. As a special case for zero
151 sized containers, classes[0] will be NO_CLASS and 1 is returned.
152
153 See the x86-64 PS ABI for details.
154 */
155 static size_t
156 classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
157 size_t byte_offset)
158 {
159 switch (type->type)
160 {
161 case FFI_TYPE_UINT8:
162 case FFI_TYPE_SINT8:
163 case FFI_TYPE_UINT16:
164 case FFI_TYPE_SINT16:
165 case FFI_TYPE_UINT32:
166 case FFI_TYPE_SINT32:
167 case FFI_TYPE_UINT64:
168 case FFI_TYPE_SINT64:
169 case FFI_TYPE_POINTER:
170 {
171 size_t size = byte_offset + type->size;
172
173 if (size <= 4)
174 {
175 classes[0] = X86_64_INTEGERSI_CLASS;
176 return 1;
177 }
178 else if (size <= 8)
179 {
180 classes[0] = X86_64_INTEGER_CLASS;
181 return 1;
182 }
183 else if (size <= 12)
184 {
185 classes[0] = X86_64_INTEGER_CLASS;
186 classes[1] = X86_64_INTEGERSI_CLASS;
187 return 2;
188 }
189 else if (size <= 16)
190 {
191 classes[0] = classes[1] = X86_64_INTEGERSI_CLASS;
192 return 2;
193 }
194 else
195 FFI_ASSERT (0);
196 }
197 case FFI_TYPE_FLOAT:
198 if (!(byte_offset % 8))
199 classes[0] = X86_64_SSESF_CLASS;
200 else
201 classes[0] = X86_64_SSE_CLASS;
202 return 1;
203 case FFI_TYPE_DOUBLE:
204 classes[0] = X86_64_SSEDF_CLASS;
205 return 1;
206 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
207 case FFI_TYPE_LONGDOUBLE:
208 classes[0] = X86_64_X87_CLASS;
209 classes[1] = X86_64_X87UP_CLASS;
210 return 2;
211 #endif
212 case FFI_TYPE_STRUCT:
213 {
214 const size_t UNITS_PER_WORD = 8;
215 size_t words = (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
216 ffi_type **ptr;
217 int i;
218 enum x86_64_reg_class subclasses[MAX_CLASSES];
219
220 /* If the struct is larger than 32 bytes, pass it on the stack. */
221 if (type->size > 32)
222 return 0;
223
224 for (i = 0; i < words; i++)
225 classes[i] = X86_64_NO_CLASS;
226
227 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
228 signalize memory class, so handle it as special case. */
229 if (!words)
230 {
231 classes[0] = X86_64_NO_CLASS;
232 return 1;
233 }
234
235 /* Merge the fields of structure. */
236 for (ptr = type->elements; *ptr != NULL; ptr++)
237 {
238 size_t num;
239
240 byte_offset = ALIGN (byte_offset, (*ptr)->alignment);
241
242 num = classify_argument (*ptr, subclasses, byte_offset % 8);
243 if (num == 0)
244 return 0;
245 for (i = 0; i < num; i++)
246 {
247 size_t pos = byte_offset / 8;
248 classes[i + pos] =
249 merge_classes (subclasses[i], classes[i + pos]);
250 }
251
252 byte_offset += (*ptr)->size;
253 }
254
255 if (words > 2)
256 {
257 /* When size > 16 bytes, if the first one isn't
258 X86_64_SSE_CLASS or any other ones aren't
259 X86_64_SSEUP_CLASS, everything should be passed in
260 memory. */
261 if (classes[0] != X86_64_SSE_CLASS)
262 return 0;
263
264 for (i = 1; i < words; i++)
265 if (classes[i] != X86_64_SSEUP_CLASS)
266 return 0;
267 }
268
269 /* Final merger cleanup. */
270 for (i = 0; i < words; i++)
271 {
272 /* If one class is MEMORY, everything should be passed in
273 memory. */
274 if (classes[i] == X86_64_MEMORY_CLASS)
275 return 0;
276
277 /* The X86_64_SSEUP_CLASS should be always preceded by
278 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
279 if (classes[i] == X86_64_SSEUP_CLASS
280 && classes[i - 1] != X86_64_SSE_CLASS
281 && classes[i - 1] != X86_64_SSEUP_CLASS)
282 {
283 /* The first one should never be X86_64_SSEUP_CLASS. */
284 FFI_ASSERT (i != 0);
285 classes[i] = X86_64_SSE_CLASS;
286 }
287
288 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
289 everything should be passed in memory. */
290 if (classes[i] == X86_64_X87UP_CLASS
291 && (classes[i - 1] != X86_64_X87_CLASS))
292 {
293 /* The first one should never be X86_64_X87UP_CLASS. */
294 FFI_ASSERT (i != 0);
295 return 0;
296 }
297 }
298 return words;
299 }
300
301 default:
302 FFI_ASSERT(0);
303 }
304 return 0; /* Never reached. */
305 }
306
307 /* Examine the argument and return set number of register required in each
308 class. Return zero iff parameter should be passed in memory, otherwise
309 the number of registers. */
310
311 static size_t
312 examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
313 _Bool in_return, int *pngpr, int *pnsse)
314 {
315 size_t n;
316 int i, ngpr, nsse;
317
318 n = classify_argument (type, classes, 0);
319 if (n == 0)
320 return 0;
321
322 ngpr = nsse = 0;
323 for (i = 0; i < n; ++i)
324 switch (classes[i])
325 {
326 case X86_64_INTEGER_CLASS:
327 case X86_64_INTEGERSI_CLASS:
328 ngpr++;
329 break;
330 case X86_64_SSE_CLASS:
331 case X86_64_SSESF_CLASS:
332 case X86_64_SSEDF_CLASS:
333 nsse++;
334 break;
335 case X86_64_NO_CLASS:
336 case X86_64_SSEUP_CLASS:
337 break;
338 case X86_64_X87_CLASS:
339 case X86_64_X87UP_CLASS:
340 case X86_64_COMPLEX_X87_CLASS:
341 return in_return != 0;
342 default:
343 abort ();
344 }
345
346 *pngpr = ngpr;
347 *pnsse = nsse;
348
349 return n;
350 }
351
352 /* Perform machine dependent cif processing. */
353
354 ffi_status
355 ffi_prep_cif_machdep (ffi_cif *cif)
356 {
357 int gprcount, ssecount, i, avn, ngpr, nsse, flags;
358 enum x86_64_reg_class classes[MAX_CLASSES];
359 size_t bytes, n;
360
361 gprcount = ssecount = 0;
362
363 flags = cif->rtype->type;
364 if (flags != FFI_TYPE_VOID)
365 {
366 n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
367 if (n == 0)
368 {
369 /* The return value is passed in memory. A pointer to that
370 memory is the first argument. Allocate a register for it. */
371 gprcount++;
372 /* We don't have to do anything in asm for the return. */
373 flags = FFI_TYPE_VOID;
374 }
375 else if (flags == FFI_TYPE_STRUCT)
376 {
377 /* Mark which registers the result appears in. */
378 _Bool sse0 = SSE_CLASS_P (classes[0]);
379 _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
380 if (sse0 && !sse1)
381 flags |= 1 << 8;
382 else if (!sse0 && sse1)
383 flags |= 1 << 9;
384 else if (sse0 && sse1)
385 flags |= 1 << 10;
386 /* Mark the true size of the structure. */
387 flags |= cif->rtype->size << 12;
388 }
389 }
390
391 /* Go over all arguments and determine the way they should be passed.
392 If it's in a register and there is space for it, let that be so. If
393 not, add it's size to the stack byte count. */
394 for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
395 {
396 if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
397 || gprcount + ngpr > MAX_GPR_REGS
398 || ssecount + nsse > MAX_SSE_REGS)
399 {
400 long align = cif->arg_types[i]->alignment;
401
402 if (align < 8)
403 align = 8;
404
405 bytes = ALIGN (bytes, align);
406 bytes += cif->arg_types[i]->size;
407 }
408 else
409 {
410 gprcount += ngpr;
411 ssecount += nsse;
412 }
413 }
414 if (ssecount)
415 flags |= 1 << 11;
416 cif->flags = flags;
417 cif->bytes = (unsigned)ALIGN (bytes, 8);
418
419 return FFI_OK;
420 }
421
422 void
423 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
424 {
425 enum x86_64_reg_class classes[MAX_CLASSES];
426 char *stack, *argp;
427 ffi_type **arg_types;
428 int gprcount, ssecount, ngpr, nsse, i, avn;
429 _Bool ret_in_memory;
430 struct register_args *reg_args;
431
432 /* Can't call 32-bit mode from 64-bit mode. */
433 FFI_ASSERT (cif->abi == FFI_UNIX64);
434
435 /* If the return value is a struct and we don't have a return value
436 address then we need to make one. Note the setting of flags to
437 VOID above in ffi_prep_cif_machdep. */
438 ret_in_memory = (cif->rtype->type == FFI_TYPE_STRUCT
439 && (cif->flags & 0xff) == FFI_TYPE_VOID);
440 if (rvalue == NULL && ret_in_memory)
441 rvalue = alloca (cif->rtype->size);
442
443 /* Allocate the space for the arguments, plus 4 words of temp space. */
444 stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
445 reg_args = (struct register_args *) stack;
446 argp = stack + sizeof (struct register_args);
447
448 gprcount = ssecount = 0;
449
450 /* If the return value is passed in memory, add the pointer as the
451 first integer argument. */
452 if (ret_in_memory)
453 reg_args->gpr[gprcount++] = (unsigned long) rvalue;
454
455 avn = cif->nargs;
456 arg_types = cif->arg_types;
457
458 for (i = 0; i < avn; ++i)
459 {
460 size_t n, size = arg_types[i]->size;
461
462 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
463 if (n == 0
464 || gprcount + ngpr > MAX_GPR_REGS
465 || ssecount + nsse > MAX_SSE_REGS)
466 {
467 long align = arg_types[i]->alignment;
468
469 /* Stack arguments are *always* at least 8 byte aligned. */
470 if (align < 8)
471 align = 8;
472
473 /* Pass this argument in memory. */
474 argp = (void *) ALIGN (argp, align);
475 memcpy (argp, avalue[i], size);
476 argp += size;
477 }
478 else
479 {
480 /* The argument is passed entirely in registers. */
481 char *a = (char *) avalue[i];
482 int j;
483
484 for (j = 0; j < n; j++, a += 8, size -= 8)
485 {
486 switch (classes[j])
487 {
488 case X86_64_INTEGER_CLASS:
489 case X86_64_INTEGERSI_CLASS:
490 /* Sign-extend integer arguments passed in general
491 purpose registers, to cope with the fact that
492 LLVM incorrectly assumes that this will be done
493 (the x86-64 PS ABI does not specify this). */
494 switch (arg_types[i]->type)
495 {
496 case FFI_TYPE_SINT8:
497 *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT8 *) a);
498 break;
499 case FFI_TYPE_SINT16:
500 *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT16 *) a);
501 break;
502 case FFI_TYPE_SINT32:
503 *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT32 *) a);
504 break;
505 default:
506 reg_args->gpr[gprcount] = 0;
507 memcpy (®_args->gpr[gprcount], a, size < 8 ? size : 8);
508 }
509 gprcount++;
510 break;
511 case X86_64_SSE_CLASS:
512 case X86_64_SSEDF_CLASS:
513 reg_args->sse[ssecount++].i64 = *(UINT64 *) a;
514 break;
515 case X86_64_SSESF_CLASS:
516 reg_args->sse[ssecount++].i32 = *(UINT32 *) a;
517 break;
518 default:
519 abort();
520 }
521 }
522 }
523 }
524
525 ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
526 cif->flags, rvalue, fn, ssecount);
527 }
528
529
530 extern void ffi_closure_unix64(void);
531
532 ffi_status
533 ffi_prep_closure_loc (ffi_closure* closure,
534 ffi_cif* cif,
535 void (*fun)(ffi_cif*, void*, void**, void*),
536 void *user_data,
537 void *codeloc)
538 {
539 volatile unsigned short *tramp;
540
541 /* Sanity check on the cif ABI. */
542 {
543 int abi = cif->abi;
544 if (UNLIKELY (! (abi > FFI_FIRST_ABI && abi < FFI_LAST_ABI)))
545 return FFI_BAD_ABI;
546 }
547
548 tramp = (volatile unsigned short *) &closure->tramp[0];
549
550 tramp[0] = 0xbb49; /* mov <code>, %r11 */
551 *((unsigned long long * volatile) &tramp[1])
552 = (unsigned long) ffi_closure_unix64;
553 tramp[5] = 0xba49; /* mov <data>, %r10 */
554 *((unsigned long long * volatile) &tramp[6])
555 = (unsigned long) codeloc;
556
557 /* Set the carry bit iff the function uses any sse registers.
558 This is clc or stc, together with the first byte of the jmp. */
559 tramp[10] = cif->flags & (1 << 11) ? 0x49f9 : 0x49f8;
560
561 tramp[11] = 0xe3ff; /* jmp *%r11 */
562
563 closure->cif = cif;
564 closure->fun = fun;
565 closure->user_data = user_data;
566
567 return FFI_OK;
568 }
569
570 int
571 ffi_closure_unix64_inner(ffi_closure *closure, void *rvalue,
572 struct register_args *reg_args, char *argp)
573 {
574 ffi_cif *cif;
575 void **avalue;
576 ffi_type **arg_types;
577 long i, avn;
578 int gprcount, ssecount, ngpr, nsse;
579 int ret;
580
581 cif = closure->cif;
582 avalue = alloca(cif->nargs * sizeof(void *));
583 gprcount = ssecount = 0;
584
585 ret = cif->rtype->type;
586 if (ret != FFI_TYPE_VOID)
587 {
588 enum x86_64_reg_class classes[MAX_CLASSES];
589 size_t n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
590 if (n == 0)
591 {
592 /* The return value goes in memory. Arrange for the closure
593 return value to go directly back to the original caller. */
594 rvalue = (void *) (unsigned long) reg_args->gpr[gprcount++];
595 /* We don't have to do anything in asm for the return. */
596 ret = FFI_TYPE_VOID;
597 }
598 else if (ret == FFI_TYPE_STRUCT && n == 2)
599 {
600 /* Mark which register the second word of the structure goes in. */
601 _Bool sse0 = SSE_CLASS_P (classes[0]);
602 _Bool sse1 = SSE_CLASS_P (classes[1]);
603 if (!sse0 && sse1)
604 ret |= 1 << 8;
605 else if (sse0 && !sse1)
606 ret |= 1 << 9;
607 }
608 }
609
610 avn = cif->nargs;
611 arg_types = cif->arg_types;
612
613 for (i = 0; i < avn; ++i)
614 {
615 enum x86_64_reg_class classes[MAX_CLASSES];
616 size_t n;
617
618 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
619 if (n == 0
620 || gprcount + ngpr > MAX_GPR_REGS
621 || ssecount + nsse > MAX_SSE_REGS)
622 {
623 long align = arg_types[i]->alignment;
624
625 /* Stack arguments are *always* at least 8 byte aligned. */
626 if (align < 8)
627 align = 8;
628
629 /* Pass this argument in memory. */
630 argp = (void *) ALIGN (argp, align);
631 avalue[i] = argp;
632 argp += arg_types[i]->size;
633 }
634 /* If the argument is in a single register, or two consecutive
635 integer registers, then we can use that address directly. */
636 else if (n == 1
637 || (n == 2 && !(SSE_CLASS_P (classes[0])
638 || SSE_CLASS_P (classes[1]))))
639 {
640 /* The argument is in a single register. */
641 if (SSE_CLASS_P (classes[0]))
642 {
643 avalue[i] = ®_args->sse[ssecount];
644 ssecount += n;
645 }
646 else
647 {
648 avalue[i] = ®_args->gpr[gprcount];
649 gprcount += n;
650 }
651 }
652 /* Otherwise, allocate space to make them consecutive. */
653 else
654 {
655 char *a = alloca (16);
656 int j;
657
658 avalue[i] = a;
659 for (j = 0; j < n; j++, a += 8)
660 {
661 if (SSE_CLASS_P (classes[j]))
662 memcpy (a, ®_args->sse[ssecount++], 8);
663 else
664 memcpy (a, ®_args->gpr[gprcount++], 8);
665 }
666 }
667 }
668
669 /* Invoke the closure. */
670 closure->fun (cif, rvalue, avalue, closure->user_data);
671
672 /* Tell assembly how to perform return type promotions. */
673 return ret;
674 }
675
676 #endif /* __x86_64__ */
|
1 /* -----------------------------------------------------------------------
2 ffi64.c - Copyright (c) 2011, 2018 Anthony Green
3 Copyright (c) 2013 The Written Word, Inc.
4 Copyright (c) 2008, 2010 Red Hat, Inc.
5 Copyright (c) 2002, 2007 Bo Thorsen <bo@suse.de>
6
7 x86-64 Foreign Function Interface
8
9 Permission is hereby granted, free of charge, to any person obtaining
10 a copy of this software and associated documentation files (the
11 ``Software''), to deal in the Software without restriction, including
12 without limitation the rights to use, copy, modify, merge, publish,
13 distribute, sublicense, and/or sell copies of the Software, and to
14 permit persons to whom the Software is furnished to do so, subject to
15 the following conditions:
16
17 The above copyright notice and this permission notice shall be included
18 in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
29
30 #include <ffi.h>
31 #include <ffi_common.h>
32
33 #include <stdlib.h>
34 #include <stdarg.h>
35 #include <stdint.h>
36 #include "internal64.h"
37
38 #ifdef __x86_64__
39
40 #define MAX_GPR_REGS 6
41 #define MAX_SSE_REGS 8
42
43 #if defined(__INTEL_COMPILER)
44 #include "xmmintrin.h"
45 #define UINT128 __m128
46 #else
47 #if defined(__SUNPRO_C)
48 #include <sunmedia_types.h>
49 #define UINT128 __m128i
50 #else
51 #define UINT128 __int128_t
52 #endif
53 #endif
54
55 union big_int_union
56 {
57 UINT32 i32;
58 UINT64 i64;
59 UINT128 i128;
60 };
61
62 struct register_args
63 {
64 /* Registers for argument passing. */
65 UINT64 gpr[MAX_GPR_REGS];
66 union big_int_union sse[MAX_SSE_REGS];
67 UINT64 rax; /* ssecount */
68 UINT64 r10; /* static chain */
69 };
70
71 extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
72 void *raddr, void (*fnaddr)(void)) FFI_HIDDEN;
73
74 /* All reference to register classes here is identical to the code in
75 gcc/config/i386/i386.c. Do *not* change one without the other. */
76
77 /* Register class used for passing given 64bit part of the argument.
78 These represent classes as documented by the PS ABI, with the
79 exception of SSESF, SSEDF classes, that are basically SSE class,
80 just gcc will use SF or DFmode move instead of DImode to avoid
81 reformatting penalties.
82
83 Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
84 whenever possible (upper half does contain padding). */
85 enum x86_64_reg_class
86 {
87 X86_64_NO_CLASS,
88 X86_64_INTEGER_CLASS,
89 X86_64_INTEGERSI_CLASS,
90 X86_64_SSE_CLASS,
91 X86_64_SSESF_CLASS,
92 X86_64_SSEDF_CLASS,
154 should be passed in memory, 0 is returned. As a special case for zero
155 sized containers, classes[0] will be NO_CLASS and 1 is returned.
156
157 See the x86-64 PS ABI for details.
158 */
159 static size_t
160 classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
161 size_t byte_offset)
162 {
163 switch (type->type)
164 {
165 case FFI_TYPE_UINT8:
166 case FFI_TYPE_SINT8:
167 case FFI_TYPE_UINT16:
168 case FFI_TYPE_SINT16:
169 case FFI_TYPE_UINT32:
170 case FFI_TYPE_SINT32:
171 case FFI_TYPE_UINT64:
172 case FFI_TYPE_SINT64:
173 case FFI_TYPE_POINTER:
174 do_integer:
175 {
176 size_t size = byte_offset + type->size;
177
178 if (size <= 4)
179 {
180 classes[0] = X86_64_INTEGERSI_CLASS;
181 return 1;
182 }
183 else if (size <= 8)
184 {
185 classes[0] = X86_64_INTEGER_CLASS;
186 return 1;
187 }
188 else if (size <= 12)
189 {
190 classes[0] = X86_64_INTEGER_CLASS;
191 classes[1] = X86_64_INTEGERSI_CLASS;
192 return 2;
193 }
194 else if (size <= 16)
195 {
196 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
197 return 2;
198 }
199 else
200 FFI_ASSERT (0);
201 }
202 case FFI_TYPE_FLOAT:
203 if (!(byte_offset % 8))
204 classes[0] = X86_64_SSESF_CLASS;
205 else
206 classes[0] = X86_64_SSE_CLASS;
207 return 1;
208 case FFI_TYPE_DOUBLE:
209 classes[0] = X86_64_SSEDF_CLASS;
210 return 1;
211 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
212 case FFI_TYPE_LONGDOUBLE:
213 classes[0] = X86_64_X87_CLASS;
214 classes[1] = X86_64_X87UP_CLASS;
215 return 2;
216 #endif
217 case FFI_TYPE_STRUCT:
218 {
219 const size_t UNITS_PER_WORD = 8;
220 size_t words = (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
221 ffi_type **ptr;
222 unsigned int i;
223 enum x86_64_reg_class subclasses[MAX_CLASSES];
224
225 /* If the struct is larger than 32 bytes, pass it on the stack. */
226 if (type->size > 32)
227 return 0;
228
229 for (i = 0; i < words; i++)
230 classes[i] = X86_64_NO_CLASS;
231
232 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
233 signalize memory class, so handle it as special case. */
234 if (!words)
235 {
236 case FFI_TYPE_VOID:
237 classes[0] = X86_64_NO_CLASS;
238 return 1;
239 }
240
241 /* Merge the fields of structure. */
242 for (ptr = type->elements; *ptr != NULL; ptr++)
243 {
244 size_t num;
245
246 byte_offset = FFI_ALIGN (byte_offset, (*ptr)->alignment);
247
248 num = classify_argument (*ptr, subclasses, byte_offset % 8);
249 if (num == 0)
250 return 0;
251 for (i = 0; i < num; i++)
252 {
253 size_t pos = byte_offset / 8;
254 classes[i + pos] =
255 merge_classes (subclasses[i], classes[i + pos]);
256 }
257
258 byte_offset += (*ptr)->size;
259 }
260
261 if (words > 2)
262 {
263 /* When size > 16 bytes, if the first one isn't
264 X86_64_SSE_CLASS or any other ones aren't
265 X86_64_SSEUP_CLASS, everything should be passed in
266 memory. */
267 if (classes[0] != X86_64_SSE_CLASS)
268 return 0;
269
270 for (i = 1; i < words; i++)
271 if (classes[i] != X86_64_SSEUP_CLASS)
272 return 0;
273 }
274
275 /* Final merger cleanup. */
276 for (i = 0; i < words; i++)
277 {
278 /* If one class is MEMORY, everything should be passed in
279 memory. */
280 if (classes[i] == X86_64_MEMORY_CLASS)
281 return 0;
282
283 /* The X86_64_SSEUP_CLASS should be always preceded by
284 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
285 if (i > 1 && classes[i] == X86_64_SSEUP_CLASS
286 && classes[i - 1] != X86_64_SSE_CLASS
287 && classes[i - 1] != X86_64_SSEUP_CLASS)
288 {
289 /* The first one should never be X86_64_SSEUP_CLASS. */
290 FFI_ASSERT (i != 0);
291 classes[i] = X86_64_SSE_CLASS;
292 }
293
294 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
295 everything should be passed in memory. */
296 if (i > 1 && classes[i] == X86_64_X87UP_CLASS
297 && (classes[i - 1] != X86_64_X87_CLASS))
298 {
299 /* The first one should never be X86_64_X87UP_CLASS. */
300 FFI_ASSERT (i != 0);
301 return 0;
302 }
303 }
304 return words;
305 }
306 case FFI_TYPE_COMPLEX:
307 {
308 ffi_type *inner = type->elements[0];
309 switch (inner->type)
310 {
311 case FFI_TYPE_INT:
312 case FFI_TYPE_UINT8:
313 case FFI_TYPE_SINT8:
314 case FFI_TYPE_UINT16:
315 case FFI_TYPE_SINT16:
316 case FFI_TYPE_UINT32:
317 case FFI_TYPE_SINT32:
318 case FFI_TYPE_UINT64:
319 case FFI_TYPE_SINT64:
320 goto do_integer;
321
322 case FFI_TYPE_FLOAT:
323 classes[0] = X86_64_SSE_CLASS;
324 if (byte_offset % 8)
325 {
326 classes[1] = X86_64_SSESF_CLASS;
327 return 2;
328 }
329 return 1;
330 case FFI_TYPE_DOUBLE:
331 classes[0] = classes[1] = X86_64_SSEDF_CLASS;
332 return 2;
333 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
334 case FFI_TYPE_LONGDOUBLE:
335 classes[0] = X86_64_COMPLEX_X87_CLASS;
336 return 1;
337 #endif
338 }
339 }
340 }
341 abort();
342 }
343
344 /* Examine the argument and return set number of register required in each
345 class. Return zero iff parameter should be passed in memory, otherwise
346 the number of registers. */
347
348 static size_t
349 examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
350 _Bool in_return, int *pngpr, int *pnsse)
351 {
352 size_t n;
353 unsigned int i;
354 int ngpr, nsse;
355
356 n = classify_argument (type, classes, 0);
357 if (n == 0)
358 return 0;
359
360 ngpr = nsse = 0;
361 for (i = 0; i < n; ++i)
362 switch (classes[i])
363 {
364 case X86_64_INTEGER_CLASS:
365 case X86_64_INTEGERSI_CLASS:
366 ngpr++;
367 break;
368 case X86_64_SSE_CLASS:
369 case X86_64_SSESF_CLASS:
370 case X86_64_SSEDF_CLASS:
371 nsse++;
372 break;
373 case X86_64_NO_CLASS:
374 case X86_64_SSEUP_CLASS:
375 break;
376 case X86_64_X87_CLASS:
377 case X86_64_X87UP_CLASS:
378 case X86_64_COMPLEX_X87_CLASS:
379 return in_return != 0;
380 default:
381 abort ();
382 }
383
384 *pngpr = ngpr;
385 *pnsse = nsse;
386
387 return n;
388 }
389
390 /* Perform machine dependent cif processing. */
391
392 #ifndef __ILP32__
393 extern ffi_status
394 ffi_prep_cif_machdep_efi64(ffi_cif *cif);
395 #endif
396
397 ffi_status FFI_HIDDEN
398 ffi_prep_cif_machdep (ffi_cif *cif)
399 {
400 int gprcount, ssecount, i, avn, ngpr, nsse;
401 unsigned flags;
402 enum x86_64_reg_class classes[MAX_CLASSES];
403 size_t bytes, n, rtype_size;
404 ffi_type *rtype;
405
406 #ifndef __ILP32__
407 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
408 return ffi_prep_cif_machdep_efi64(cif);
409 #endif
410 if (cif->abi != FFI_UNIX64)
411 return FFI_BAD_ABI;
412
413 gprcount = ssecount = 0;
414
415 rtype = cif->rtype;
416 rtype_size = rtype->size;
417 switch (rtype->type)
418 {
419 case FFI_TYPE_VOID:
420 flags = UNIX64_RET_VOID;
421 break;
422 case FFI_TYPE_UINT8:
423 flags = UNIX64_RET_UINT8;
424 break;
425 case FFI_TYPE_SINT8:
426 flags = UNIX64_RET_SINT8;
427 break;
428 case FFI_TYPE_UINT16:
429 flags = UNIX64_RET_UINT16;
430 break;
431 case FFI_TYPE_SINT16:
432 flags = UNIX64_RET_SINT16;
433 break;
434 case FFI_TYPE_UINT32:
435 flags = UNIX64_RET_UINT32;
436 break;
437 case FFI_TYPE_INT:
438 case FFI_TYPE_SINT32:
439 flags = UNIX64_RET_SINT32;
440 break;
441 case FFI_TYPE_UINT64:
442 case FFI_TYPE_SINT64:
443 flags = UNIX64_RET_INT64;
444 break;
445 case FFI_TYPE_POINTER:
446 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
447 break;
448 case FFI_TYPE_FLOAT:
449 flags = UNIX64_RET_XMM32;
450 break;
451 case FFI_TYPE_DOUBLE:
452 flags = UNIX64_RET_XMM64;
453 break;
454 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
455 case FFI_TYPE_LONGDOUBLE:
456 flags = UNIX64_RET_X87;
457 break;
458 #endif
459 case FFI_TYPE_STRUCT:
460 n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
461 if (n == 0)
462 {
463 /* The return value is passed in memory. A pointer to that
464 memory is the first argument. Allocate a register for it. */
465 gprcount++;
466 /* We don't have to do anything in asm for the return. */
467 flags = UNIX64_RET_VOID | UNIX64_FLAG_RET_IN_MEM;
468 }
469 else
470 {
471 _Bool sse0 = SSE_CLASS_P (classes[0]);
472
473 if (rtype_size == 4 && sse0)
474 flags = UNIX64_RET_XMM32;
475 else if (rtype_size == 8)
476 flags = sse0 ? UNIX64_RET_XMM64 : UNIX64_RET_INT64;
477 else
478 {
479 _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
480 if (sse0 && sse1)
481 flags = UNIX64_RET_ST_XMM0_XMM1;
482 else if (sse0)
483 flags = UNIX64_RET_ST_XMM0_RAX;
484 else if (sse1)
485 flags = UNIX64_RET_ST_RAX_XMM0;
486 else
487 flags = UNIX64_RET_ST_RAX_RDX;
488 flags |= rtype_size << UNIX64_SIZE_SHIFT;
489 }
490 }
491 break;
492 case FFI_TYPE_COMPLEX:
493 switch (rtype->elements[0]->type)
494 {
495 case FFI_TYPE_UINT8:
496 case FFI_TYPE_SINT8:
497 case FFI_TYPE_UINT16:
498 case FFI_TYPE_SINT16:
499 case FFI_TYPE_INT:
500 case FFI_TYPE_UINT32:
501 case FFI_TYPE_SINT32:
502 case FFI_TYPE_UINT64:
503 case FFI_TYPE_SINT64:
504 flags = UNIX64_RET_ST_RAX_RDX | ((unsigned) rtype_size << UNIX64_SIZE_SHIFT);
505 break;
506 case FFI_TYPE_FLOAT:
507 flags = UNIX64_RET_XMM64;
508 break;
509 case FFI_TYPE_DOUBLE:
510 flags = UNIX64_RET_ST_XMM0_XMM1 | (16 << UNIX64_SIZE_SHIFT);
511 break;
512 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
513 case FFI_TYPE_LONGDOUBLE:
514 flags = UNIX64_RET_X87_2;
515 break;
516 #endif
517 default:
518 return FFI_BAD_TYPEDEF;
519 }
520 break;
521 default:
522 return FFI_BAD_TYPEDEF;
523 }
524
525 /* Go over all arguments and determine the way they should be passed.
526 If it's in a register and there is space for it, let that be so. If
527 not, add it's size to the stack byte count. */
528 for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
529 {
530 if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
531 || gprcount + ngpr > MAX_GPR_REGS
532 || ssecount + nsse > MAX_SSE_REGS)
533 {
534 long align = cif->arg_types[i]->alignment;
535
536 if (align < 8)
537 align = 8;
538
539 bytes = FFI_ALIGN (bytes, align);
540 bytes += cif->arg_types[i]->size;
541 }
542 else
543 {
544 gprcount += ngpr;
545 ssecount += nsse;
546 }
547 }
548 if (ssecount)
549 flags |= UNIX64_FLAG_XMM_ARGS;
550
551 cif->flags = flags;
552 cif->bytes = (unsigned) FFI_ALIGN (bytes, 8);
553
554 return FFI_OK;
555 }
556
557 static void
558 ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue,
559 void **avalue, void *closure)
560 {
561 enum x86_64_reg_class classes[MAX_CLASSES];
562 char *stack, *argp;
563 ffi_type **arg_types;
564 int gprcount, ssecount, ngpr, nsse, i, avn, flags;
565 struct register_args *reg_args;
566
567 /* Can't call 32-bit mode from 64-bit mode. */
568 FFI_ASSERT (cif->abi == FFI_UNIX64);
569
570 /* If the return value is a struct and we don't have a return value
571 address then we need to make one. Otherwise we can ignore it. */
572 flags = cif->flags;
573 if (rvalue == NULL)
574 {
575 if (flags & UNIX64_FLAG_RET_IN_MEM)
576 rvalue = alloca (cif->rtype->size);
577 else
578 flags = UNIX64_RET_VOID;
579 }
580
581 /* Allocate the space for the arguments, plus 4 words of temp space. */
582 stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
583 reg_args = (struct register_args *) stack;
584 argp = stack + sizeof (struct register_args);
585
586 reg_args->r10 = (uintptr_t) closure;
587
588 gprcount = ssecount = 0;
589
590 /* If the return value is passed in memory, add the pointer as the
591 first integer argument. */
592 if (flags & UNIX64_FLAG_RET_IN_MEM)
593 reg_args->gpr[gprcount++] = (unsigned long) rvalue;
594
595 avn = cif->nargs;
596 arg_types = cif->arg_types;
597
598 for (i = 0; i < avn; ++i)
599 {
600 size_t n, size = arg_types[i]->size;
601
602 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
603 if (n == 0
604 || gprcount + ngpr > MAX_GPR_REGS
605 || ssecount + nsse > MAX_SSE_REGS)
606 {
607 long align = arg_types[i]->alignment;
608
609 /* Stack arguments are *always* at least 8 byte aligned. */
610 if (align < 8)
611 align = 8;
612
613 /* Pass this argument in memory. */
614 argp = (void *) FFI_ALIGN (argp, align);
615 memcpy (argp, avalue[i], size);
616 argp += size;
617 }
618 else
619 {
620 /* The argument is passed entirely in registers. */
621 char *a = (char *) avalue[i];
622 unsigned int j;
623
624 for (j = 0; j < n; j++, a += 8, size -= 8)
625 {
626 switch (classes[j])
627 {
628 case X86_64_NO_CLASS:
629 case X86_64_SSEUP_CLASS:
630 break;
631 case X86_64_INTEGER_CLASS:
632 case X86_64_INTEGERSI_CLASS:
633 /* Sign-extend integer arguments passed in general
634 purpose registers, to cope with the fact that
635 LLVM incorrectly assumes that this will be done
636 (the x86-64 PS ABI does not specify this). */
637 switch (arg_types[i]->type)
638 {
639 case FFI_TYPE_SINT8:
640 reg_args->gpr[gprcount] = (SINT64) *((SINT8 *) a);
641 break;
642 case FFI_TYPE_SINT16:
643 reg_args->gpr[gprcount] = (SINT64) *((SINT16 *) a);
644 break;
645 case FFI_TYPE_SINT32:
646 reg_args->gpr[gprcount] = (SINT64) *((SINT32 *) a);
647 break;
648 default:
649 reg_args->gpr[gprcount] = 0;
650 memcpy (®_args->gpr[gprcount], a, size);
651 }
652 gprcount++;
653 break;
654 case X86_64_SSE_CLASS:
655 case X86_64_SSEDF_CLASS:
656 memcpy (®_args->sse[ssecount++].i64, a, sizeof(UINT64));
657 break;
658 case X86_64_SSESF_CLASS:
659 memcpy (®_args->sse[ssecount++].i32, a, sizeof(UINT32));
660 break;
661 default:
662 abort();
663 }
664 }
665 }
666 }
667 reg_args->rax = ssecount;
668
669 ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
670 flags, rvalue, fn);
671 }
672
673 #ifndef __ILP32__
674 extern void
675 ffi_call_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue);
676 #endif
677
678 void
679 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
680 {
681 #ifndef __ILP32__
682 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
683 {
684 ffi_call_efi64(cif, fn, rvalue, avalue);
685 return;
686 }
687 #endif
688 ffi_call_int (cif, fn, rvalue, avalue, NULL);
689 }
690
691 #ifndef __ILP32__
692 extern void
693 ffi_call_go_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue,
694 void **avalue, void *closure);
695 #endif
696
697 void
698 ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue,
699 void **avalue, void *closure)
700 {
701 #ifndef __ILP32__
702 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
703 {
704 ffi_call_go_efi64(cif, fn, rvalue, avalue, closure);
705 return;
706 }
707 #endif
708 ffi_call_int (cif, fn, rvalue, avalue, closure);
709 }
710
711
712 extern void ffi_closure_unix64(void) FFI_HIDDEN;
713 extern void ffi_closure_unix64_sse(void) FFI_HIDDEN;
714
715 #ifndef __ILP32__
716 extern ffi_status
717 ffi_prep_closure_loc_efi64(ffi_closure* closure,
718 ffi_cif* cif,
719 void (*fun)(ffi_cif*, void*, void**, void*),
720 void *user_data,
721 void *codeloc);
722 #endif
723
724 ffi_status
725 ffi_prep_closure_loc (ffi_closure* closure,
726 ffi_cif* cif,
727 void (*fun)(ffi_cif*, void*, void**, void*),
728 void *user_data,
729 void *codeloc)
730 {
731 static const unsigned char trampoline[16] = {
732 /* leaq -0x7(%rip),%r10 # 0x0 */
733 0x4c, 0x8d, 0x15, 0xf9, 0xff, 0xff, 0xff,
734 /* jmpq *0x3(%rip) # 0x10 */
735 0xff, 0x25, 0x03, 0x00, 0x00, 0x00,
736 /* nopl (%rax) */
737 0x0f, 0x1f, 0x00
738 };
739 void (*dest)(void);
740 char *tramp = closure->tramp;
741
742 #ifndef __ILP32__
743 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
744 return ffi_prep_closure_loc_efi64(closure, cif, fun, user_data, codeloc);
745 #endif
746 if (cif->abi != FFI_UNIX64)
747 return FFI_BAD_ABI;
748
749 if (cif->flags & UNIX64_FLAG_XMM_ARGS)
750 dest = ffi_closure_unix64_sse;
751 else
752 dest = ffi_closure_unix64;
753
754 memcpy (tramp, trampoline, sizeof(trampoline));
755 *(UINT64 *)(tramp + 16) = (uintptr_t)dest;
756
757 closure->cif = cif;
758 closure->fun = fun;
759 closure->user_data = user_data;
760
761 return FFI_OK;
762 }
763
764 int FFI_HIDDEN
765 ffi_closure_unix64_inner(ffi_cif *cif,
766 void (*fun)(ffi_cif*, void*, void**, void*),
767 void *user_data,
768 void *rvalue,
769 struct register_args *reg_args,
770 char *argp)
771 {
772 void **avalue;
773 ffi_type **arg_types;
774 long i, avn;
775 int gprcount, ssecount, ngpr, nsse;
776 int flags;
777
778 avn = cif->nargs;
779 flags = cif->flags;
780 avalue = alloca(avn * sizeof(void *));
781 gprcount = ssecount = 0;
782
783 if (flags & UNIX64_FLAG_RET_IN_MEM)
784 {
785 /* On return, %rax will contain the address that was passed
786 by the caller in %rdi. */
787 void *r = (void *)(uintptr_t)reg_args->gpr[gprcount++];
788 *(void **)rvalue = r;
789 rvalue = r;
790 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
791 }
792
793 arg_types = cif->arg_types;
794 for (i = 0; i < avn; ++i)
795 {
796 enum x86_64_reg_class classes[MAX_CLASSES];
797 size_t n;
798
799 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
800 if (n == 0
801 || gprcount + ngpr > MAX_GPR_REGS
802 || ssecount + nsse > MAX_SSE_REGS)
803 {
804 long align = arg_types[i]->alignment;
805
806 /* Stack arguments are *always* at least 8 byte aligned. */
807 if (align < 8)
808 align = 8;
809
810 /* Pass this argument in memory. */
811 argp = (void *) FFI_ALIGN (argp, align);
812 avalue[i] = argp;
813 argp += arg_types[i]->size;
814 }
815 /* If the argument is in a single register, or two consecutive
816 integer registers, then we can use that address directly. */
817 else if (n == 1
818 || (n == 2 && !(SSE_CLASS_P (classes[0])
819 || SSE_CLASS_P (classes[1]))))
820 {
821 /* The argument is in a single register. */
822 if (SSE_CLASS_P (classes[0]))
823 {
824 avalue[i] = ®_args->sse[ssecount];
825 ssecount += n;
826 }
827 else
828 {
829 avalue[i] = ®_args->gpr[gprcount];
830 gprcount += n;
831 }
832 }
833 /* Otherwise, allocate space to make them consecutive. */
834 else
835 {
836 char *a = alloca (16);
837 unsigned int j;
838
839 avalue[i] = a;
840 for (j = 0; j < n; j++, a += 8)
841 {
842 if (SSE_CLASS_P (classes[j]))
843 memcpy (a, ®_args->sse[ssecount++], 8);
844 else
845 memcpy (a, ®_args->gpr[gprcount++], 8);
846 }
847 }
848 }
849
850 /* Invoke the closure. */
851 fun (cif, rvalue, avalue, user_data);
852
853 /* Tell assembly how to perform return type promotions. */
854 return flags;
855 }
856
857 extern void ffi_go_closure_unix64(void) FFI_HIDDEN;
858 extern void ffi_go_closure_unix64_sse(void) FFI_HIDDEN;
859
860 #ifndef __ILP32__
861 extern ffi_status
862 ffi_prep_go_closure_efi64(ffi_go_closure* closure, ffi_cif* cif,
863 void (*fun)(ffi_cif*, void*, void**, void*));
864 #endif
865
866 ffi_status
867 ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif,
868 void (*fun)(ffi_cif*, void*, void**, void*))
869 {
870 #ifndef __ILP32__
871 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
872 return ffi_prep_go_closure_efi64(closure, cif, fun);
873 #endif
874 if (cif->abi != FFI_UNIX64)
875 return FFI_BAD_ABI;
876
877 closure->tramp = (cif->flags & UNIX64_FLAG_XMM_ARGS
878 ? ffi_go_closure_unix64_sse
879 : ffi_go_closure_unix64);
880 closure->cif = cif;
881 closure->fun = fun;
882
883 return FFI_OK;
884 }
885
886 #endif /* __x86_64__ */
|