40 NUM_GPR_ARG_REGISTERS64 = 8,
41 NUM_FPR_ARG_REGISTERS64 = 13
43 enum { ASM_NEEDS_REGISTERS64 = 4 };
46 #if HAVE_LONG_DOUBLE_VARIANT && FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
51 if ((abi & (FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128)) == FFI_LINUX)
53 ffi_type_longdouble.size = 8;
54 ffi_type_longdouble.alignment = 8;
58 ffi_type_longdouble.size = 16;
59 ffi_type_longdouble.alignment = 16;
67 discover_homogeneous_aggregate (
const ffi_type *t,
unsigned int *elnum)
76 case FFI_TYPE_STRUCT:;
78 unsigned int base_elt = 0, total_elnum = 0;
79 ffi_type **el = t->elements;
82 unsigned int el_elt, el_elnum = 0;
83 el_elt = discover_homogeneous_aggregate (*el, &el_elnum);
85 || (base_elt && base_elt != el_elt))
88 total_elnum += el_elnum;
106 ffi_prep_cif_linux64_core (ffi_cif *cif)
110 unsigned i, fparg_count = 0, intarg_count = 0;
111 unsigned flags = cif->flags;
113 unsigned int elt, elnum;
116 #if FFI_TYPE_LONGDOUBLE == FFI_TYPE_DOUBLE
118 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
126 bytes = (4 + ASM_NEEDS_REGISTERS64) *
sizeof (
long);
129 bytes += NUM_GPR_ARG_REGISTERS64 *
sizeof (
long);
133 bytes = (6 + ASM_NEEDS_REGISTERS64) *
sizeof (
long);
136 bytes += 2 * NUM_GPR_ARG_REGISTERS64 *
sizeof (
long);
140 switch (cif->rtype->type)
142 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
144 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
148 case FFI_TYPE_DOUBLE:
158 case FFI_TYPE_UINT64:
159 case FFI_TYPE_SINT64:
163 case FFI_TYPE_STRUCT:
165 elt = discover_homogeneous_aggregate (cif->rtype, &elnum);
168 if (elt == FFI_TYPE_DOUBLE)
173 if (cif->rtype->size <= 16)
191 for (
ptr = cif->arg_types,
i = cif->nargs;
i > 0;
i--,
ptr++)
195 switch ((*ptr)->type)
197 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
199 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
206 case FFI_TYPE_DOUBLE:
210 if (fparg_count > NUM_FPR_ARG_REGISTERS64)
214 case FFI_TYPE_STRUCT:
215 if ((cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
217 align = (*ptr)->alignment;
222 intarg_count =
ALIGN (intarg_count, align);
224 intarg_count += ((*ptr)->size + 7) / 8;
226 elt = discover_homogeneous_aggregate (*
ptr, &elnum);
229 fparg_count += elnum;
230 if (fparg_count > NUM_FPR_ARG_REGISTERS64)
236 if (intarg_count > NUM_GPR_ARG_REGISTERS64)
241 case FFI_TYPE_POINTER:
242 case FFI_TYPE_UINT64:
243 case FFI_TYPE_SINT64:
245 case FFI_TYPE_UINT32:
246 case FFI_TYPE_SINT32:
247 case FFI_TYPE_UINT16:
248 case FFI_TYPE_SINT16:
254 if (intarg_count > NUM_GPR_ARG_REGISTERS64)
262 if (fparg_count != 0)
264 if (intarg_count > 4)
268 if (fparg_count != 0)
269 bytes += NUM_FPR_ARG_REGISTERS64 *
sizeof (
double);
274 bytes += intarg_count *
sizeof (
long);
276 if (intarg_count > NUM_GPR_ARG_REGISTERS64)
277 bytes += (intarg_count - NUM_GPR_ARG_REGISTERS64) *
sizeof (
long);
281 bytes = (bytes + 15) & ~0xF;
292 if ((cif->abi & FFI_LINUX) != 0)
293 cif->nfixedargs = cif->nargs;
301 cif->abi = FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128;
306 return ffi_prep_cif_linux64_core (cif);
311 unsigned int nfixedargs,
314 if ((cif->abi & FFI_LINUX) != 0)
315 cif->nfixedargs = nfixedargs;
323 cif->abi = FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128;
331 return ffi_prep_cif_linux64_core (cif);
372 const unsigned long bytes = ecif->
cif->bytes;
373 const unsigned long flags = ecif->
cif->flags;
397 unsigned int fparg_count;
399 unsigned int i, words, nargs, nfixedargs;
416 unsigned long gprvalue;
419 stacktop.c = (
char *) stack + bytes;
420 gpr_base.ul = stacktop.ul - ASM_NEEDS_REGISTERS64 - NUM_GPR_ARG_REGISTERS64;
421 gpr_end.ul = gpr_base.ul + NUM_GPR_ARG_REGISTERS64;
423 rest.ul = stack + 4 + NUM_GPR_ARG_REGISTERS64;
425 rest.ul = stack + 6 + NUM_GPR_ARG_REGISTERS64;
427 fpr_base.d = gpr_base.d - NUM_FPR_ARG_REGISTERS64;
429 next_arg.ul = gpr_base.ul;
432 FFI_ASSERT (((
unsigned long) (
char *) stack & 0xF) == 0);
433 FFI_ASSERT (((
unsigned long) stacktop.c & 0xF) == 0);
438 *next_arg.ul++ = (
unsigned long) (
char *) ecif->
rvalue;
442 nargs = ecif->
cif->nargs;
447 nfixedargs = ecif->
cif->nfixedargs;
448 for (
ptr = ecif->
cif->arg_types,
i = 0;
450 i++,
ptr++, p_argv.v++)
453 unsigned int elt, elnum;
456 switch ((*ptr)->type)
458 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
460 if ((ecif->
cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
462 double_tmp = (*p_argv.d)[0];
463 if (fparg_count < NUM_FPR_ARG_REGISTERS64 &&
i < nfixedargs)
465 *fpr_base.d++ = double_tmp;
468 *next_arg.d = double_tmp;
472 *next_arg.d = double_tmp;
473 if (++next_arg.ul == gpr_end.ul)
474 next_arg.ul = rest.ul;
476 double_tmp = (*p_argv.d)[1];
477 if (fparg_count < NUM_FPR_ARG_REGISTERS64 &&
i < nfixedargs)
479 *fpr_base.d++ = double_tmp;
482 *next_arg.d = double_tmp;
486 *next_arg.d = double_tmp;
487 if (++next_arg.ul == gpr_end.ul)
488 next_arg.ul = rest.ul;
496 case FFI_TYPE_DOUBLE:
497 double_tmp = **p_argv.d;
498 if (fparg_count < NUM_FPR_ARG_REGISTERS64 &&
i < nfixedargs)
500 *fpr_base.d++ = double_tmp;
503 *next_arg.d = double_tmp;
507 *next_arg.d = double_tmp;
508 if (++next_arg.ul == gpr_end.ul)
509 next_arg.ul = rest.ul;
515 double_tmp = **p_argv.f;
516 if (fparg_count < NUM_FPR_ARG_REGISTERS64 &&
i < nfixedargs)
518 *fpr_base.d++ = double_tmp;
521 *next_arg.f = (float) double_tmp;
525 *next_arg.f = (float) double_tmp;
526 if (++next_arg.ul == gpr_end.ul)
527 next_arg.ul = rest.ul;
532 case FFI_TYPE_STRUCT:
533 if ((ecif->
cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
535 align = (*ptr)->alignment;
539 next_arg.p =
ALIGN (next_arg.p, align);
542 elt = discover_homogeneous_aggregate (*
ptr, &elnum);
552 if (elt == FFI_TYPE_FLOAT)
556 double_tmp = *
arg.f++;
557 if (fparg_count < NUM_FPR_ARG_REGISTERS64
559 *fpr_base.d++ = double_tmp;
561 *next_arg.f = (float) double_tmp;
562 if (++next_arg.f == gpr_end.f)
566 while (--elnum != 0);
567 if ((next_arg.p & 3) != 0)
569 if (++next_arg.f == gpr_end.f)
576 double_tmp = *
arg.d++;
577 if (fparg_count < NUM_FPR_ARG_REGISTERS64 &&
i < nfixedargs)
578 *fpr_base.d++ = double_tmp;
580 *next_arg.d = double_tmp;
581 if (++next_arg.d == gpr_end.d)
585 while (--elnum != 0);
590 words = ((*ptr)->size + 7) / 8;
591 if (next_arg.ul >= gpr_base.ul && next_arg.ul + words > gpr_end.ul)
593 size_t first = gpr_end.c - next_arg.c;
594 memcpy (next_arg.c, *p_argv.c, first);
595 memcpy (rest.c, *p_argv.c + first, (*ptr)->size - first);
596 next_arg.c = rest.c + words * 8 - first;
600 char *where = next_arg.c;
602 #ifndef __LITTLE_ENDIAN__
605 if ((*ptr)->size < 8)
606 where += 8 - (*ptr)->size;
608 memcpy (where, *p_argv.c, (*ptr)->size);
609 next_arg.ul += words;
610 if (next_arg.ul == gpr_end.ul)
611 next_arg.ul = rest.ul;
617 gprvalue = **p_argv.uc;
620 gprvalue = **p_argv.sc;
622 case FFI_TYPE_UINT16:
623 gprvalue = **p_argv.us;
625 case FFI_TYPE_SINT16:
626 gprvalue = **p_argv.ss;
628 case FFI_TYPE_UINT32:
629 gprvalue = **p_argv.ui;
632 case FFI_TYPE_SINT32:
633 gprvalue = **p_argv.si;
636 case FFI_TYPE_UINT64:
637 case FFI_TYPE_SINT64:
638 case FFI_TYPE_POINTER:
639 gprvalue = **p_argv.ul;
641 *next_arg.ul++ = gprvalue;
642 if (next_arg.ul == gpr_end.ul)
643 next_arg.ul = rest.ul;
649 || (next_arg.ul >= gpr_base.ul
650 && next_arg.ul <= gpr_base.ul + 4));
655 #define MIN_CACHE_LINE_SIZE 8
658 flush_icache (
char *wraddr,
char *xaddr,
int size)
662 __asm__ volatile (
"icbi 0,%0;" "dcbf 0,%1;"
663 : :
"r" (xaddr +
i),
"r" (wraddr +
i) :
"memory");
664 __asm__ volatile (
"icbi 0,%0;" "dcbf 0,%1;" "sync;" "isync;"
665 : :
"r"(xaddr +
size - 1),
"r"(wraddr +
size - 1)
671 ffi_prep_closure_loc_linux64 (ffi_closure *closure,
673 void (*fun) (ffi_cif *,
void *,
void **,
void *),
678 unsigned int *tramp = (
unsigned int *) &closure->tramp[0];
683 tramp[0] = 0xe96c0018;
684 tramp[1] = 0xe98c0010;
685 tramp[2] = 0x7d8903a6;
686 tramp[3] = 0x4e800420;
690 *(
void **) &tramp[6] = codeloc;
693 void **tramp = (
void **) &closure->tramp[0];
706 closure->user_data = user_data;
722 ffi_type **arg_types;
723 unsigned long i, avn, nfixedargs;
725 ffi_dblfl *end_pfr = pfr + NUM_FPR_ARG_REGISTERS64;
729 avalue =
alloca (cif->nargs * sizeof (
void *));
733 if (cif->rtype->type == FFI_TYPE_STRUCT
736 rvalue = (
void *) *pst;
746 nfixedargs = cif->nfixedargs;
747 arg_types = cif->arg_types;
752 unsigned int elt, elnum;
754 switch (arg_types[
i]->
type)
758 #ifndef __LITTLE_ENDIAN__
759 avalue[
i] = (
char *) pst + 7;
764 case FFI_TYPE_SINT16:
765 case FFI_TYPE_UINT16:
766 #ifndef __LITTLE_ENDIAN__
767 avalue[
i] = (
char *) pst + 6;
772 case FFI_TYPE_SINT32:
773 case FFI_TYPE_UINT32:
774 #ifndef __LITTLE_ENDIAN__
775 avalue[
i] = (
char *) pst + 4;
780 case FFI_TYPE_SINT64:
781 case FFI_TYPE_UINT64:
782 case FFI_TYPE_POINTER:
787 case FFI_TYPE_STRUCT:
788 if ((cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
790 align = arg_types[
i]->alignment;
794 pst = (
unsigned long *)
ALIGN ((
size_t) pst, align);
798 elt = discover_homogeneous_aggregate (arg_types[
i], &elnum);
814 if (pfr + elnum <= end_pfr)
821 if (elt == FFI_TYPE_FLOAT)
825 if (pfr < end_pfr &&
i < nfixedargs)
827 *to.
f = (float) pfr->
d;
835 while (--elnum != 0);
841 if (pfr < end_pfr &&
i < nfixedargs)
851 while (--elnum != 0);
856 #ifndef __LITTLE_ENDIAN__
859 if (arg_types[
i]->
size < 8)
860 avalue[
i] = (
char *) pst + 8 - arg_types[
i]->
size;
865 pst += (arg_types[
i]->size + 7) / 8;
868 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
870 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
872 if (pfr + 1 < end_pfr &&
i + 1 < nfixedargs)
879 if (pfr < end_pfr &&
i < nfixedargs)
883 *pst = *(
unsigned long *) pfr;
893 case FFI_TYPE_DOUBLE:
897 if (pfr < end_pfr &&
i < nfixedargs)
908 if (pfr < end_pfr &&
i < nfixedargs)
912 pfr->
f = (float) pfr->
d;
929 (closure->fun) (cif, rvalue, avalue, closure->user_data);
941 return cif->rtype->type;