address generate_unsafe_copy(const char *name,
address byte_copy_entry, address short_copy_entry,
address int_copy_entry, address long_copy_entry) {
Label L_long_aligned, L_int_aligned, L_short_aligned;
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
const Register size = c_rarg2; // byte count (size_t)
const Register bits = rax; // test copy of low bits
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", name);
address start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
__ mov(bits, from);
__ orptr(bits, to);
__ orptr(bits, size);
__ testb(bits, BytesPerLong-1);
__ jccb(Assembler::zero, L_long_aligned);
__ testb(bits, BytesPerInt-1);
__ jccb(Assembler::zero, L_int_aligned);
__ testb(bits, BytesPerShort-1);
__ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
__ BIND(L_short_aligned);
__ shrptr(size, LogBytesPerShort); // size => short_count
__ jump(RuntimeAddress(short_copy_entry));
__ BIND(L_int_aligned);
__ shrptr(size, LogBytesPerInt); // size => int_count
__ jump(RuntimeAddress(int_copy_entry));
__ BIND(L_long_aligned);
__ shrptr(size, LogBytesPerLong); // size => qword_count
__ jump(RuntimeAddress(long_copy_entry));
return start;
}
void arraycopy_range_checks(Register src, // source array oop (c_rarg0)
Register src_pos, // source position (c_rarg1)
Register dst, // destination array oo (c_rarg2)
Register dst_pos, // destination position (c_rarg3)
Register length,
Register temp,
Label& L_failed) {
BLOCK_COMMENT("arraycopy_range_checks:");
__ movl(temp, length);
__ addl(temp, src_pos); // src_pos + length
__ cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes()));
__ jcc(Assembler::above, L_failed);
__ movl(temp, length);
__ addl(temp, dst_pos); // dst_pos + length
__ cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes()));
__ jcc(Assembler::above, L_failed);
__ movslq(src_pos, src_pos);
__ movslq(dst_pos, dst_pos);
BLOCK_COMMENT("arraycopy_range_checks done");
}
address generate_generic_copy(const char *name,
address byte_copy_entry, address short_copy_entry,
address int_copy_entry, address oop_copy_entry,
address long_copy_entry, address checkcast_copy_entry) {
Label L_failed, L_failed_0, L_objArray;
Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
const Register src = c_rarg0; // source array oop
const Register src_pos = c_rarg1; // source position
const Register dst = c_rarg2; // destination array oop
const Register dst_pos = c_rarg3; // destination position
#ifndef _WIN64
const Register length = c_rarg4;
#else
const Address length(rsp, 6 * wordSize); // elements count is on stack on Win64
#endif
{ int modulus = CodeEntryAlignment;
int target = modulus - 5; // 5 = sizeof jmp(L_failed)
int advance = target - (__ offset() % modulus);
if (advance < 0) advance += modulus;
if (advance > 0) __ nop(advance);
}
StubCodeMark mark(this, "StubRoutines", name);
__ BIND(L_failed_0);
__ jmp(L_failed);
assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
__ align(CodeEntryAlignment);
address start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
__ testptr(src, src); // src oop
size_t j1off = __ offset();
__ jccb(Assembler::zero, L_failed_0);
__ testl(src_pos, src_pos); // src_pos (32-bits)
__ jccb(Assembler::negative, L_failed_0);
__ testptr(dst, dst); // dst oop
__ jccb(Assembler::zero, L_failed_0);
__ testl(dst_pos, dst_pos); // dst_pos (32-bits)
size_t j4off = __ offset();
__ jccb(Assembler::negative, L_failed_0);
guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps");
const Register r11_length = r11; // elements count to copy
const Register r10_src_klass = r10; // array klass
__ movl(r11_length, length); // length (elements count, 32-bits value)
__ testl(r11_length, r11_length);
__ jccb(Assembler::negative, L_failed_0);
__ load_klass(r10_src_klass, src);
#ifdef ASSERT
{
BLOCK_COMMENT("assert klasses not null {");
Label L1, L2;
__ testptr(r10_src_klass, r10_src_klass);
__ jcc(Assembler::notZero, L2); // it is broken if klass is NULL
__ bind(L1);
__ stop("broken null klass");
__ bind(L2);
__ load_klass(rax, dst);
__ cmpq(rax, 0);
__ jcc(Assembler::equal, L1); // this would be broken also
BLOCK_COMMENT("} assert klasses not null done");
}
#endif
const int lh_offset = in_bytes(Klass::layout_helper_offset());
const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
__ cmpl(Address(r10_src_klass, lh_offset), objArray_lh);
__ jcc(Assembler::equal, L_objArray);
__ load_klass(rax, dst);
__ cmpq(r10_src_klass, rax);
__ jcc(Assembler::notEqual, L_failed);
const Register rax_lh = rax; // layout helper
__ movl(rax_lh, Address(r10_src_klass, lh_offset));
__ cmpl(rax_lh, Klass::_lh_neutral_value);
__ jcc(Assembler::greaterEqual, L_failed);
#ifdef ASSERT
{
BLOCK_COMMENT("assert primitive array {");
Label L;
__ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
__ jcc(Assembler::greaterEqual, L);
__ stop("must be a primitive array");
__ bind(L);
BLOCK_COMMENT("} assert primitive array done");
}
#endif
arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
r10, L_failed);
const Register r10_offset = r10; // array offset
const Register rax_elsize = rax_lh; // element size
__ movl(r10_offset, rax_lh);
__ shrl(r10_offset, Klass::_lh_header_size_shift);
__ andptr(r10_offset, Klass::_lh_header_size_mask); // array_offset
__ addptr(src, r10_offset); // src array offset
__ addptr(dst, r10_offset); // dst array offset
BLOCK_COMMENT("choose copy loop based on element size");
__ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
const Register count = c_rarg2; // elements count
__ BIND(L_copy_bytes);
__ cmpl(rax_elsize, 0);
__ jccb(Assembler::notEqual, L_copy_shorts);
__ lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr
__ lea(to, Address(dst, dst_pos, Address::times_1, 0));// dst_addr
__ movl2ptr(count, r11_length); // length
__ jump(RuntimeAddress(byte_copy_entry));
__ BIND(L_copy_shorts);
__ cmpl(rax_elsize, LogBytesPerShort);
__ jccb(Assembler::notEqual, L_copy_ints);
__ lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr
__ lea(to, Address(dst, dst_pos, Address::times_2, 0));// dst_addr
__ movl2ptr(count, r11_length); // length
__ jump(RuntimeAddress(short_copy_entry));
__ BIND(L_copy_ints);
__ cmpl(rax_elsize, LogBytesPerInt);
__ jccb(Assembler::notEqual, L_copy_longs);
__ lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr
__ lea(to, Address(dst, dst_pos, Address::times_4, 0));// dst_addr
__ movl2ptr(count, r11_length); // length
__ jump(RuntimeAddress(int_copy_entry));
__ BIND(L_copy_longs);
#ifdef ASSERT
{
BLOCK_COMMENT("assert long copy {");
Label L;
__ cmpl(rax_elsize, LogBytesPerLong);
__ jcc(Assembler::equal, L);
__ stop("must be long copy, but elsize is wrong");
__ bind(L);
BLOCK_COMMENT("} assert long copy done");
}
#endif
__ lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr
__ lea(to, Address(dst, dst_pos, Address::times_8, 0));// dst_addr
__ movl2ptr(count, r11_length); // length
__ jump(RuntimeAddress(long_copy_entry));
__ BIND(L_objArray);
Label L_plain_copy, L_checkcast_copy;
__ load_klass(rax, dst);
__ cmpq(r10_src_klass, rax); // usual case is exact equality
__ jcc(Assembler::notEqual, L_checkcast_copy);
arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
r10, L_failed);
__ lea(from, Address(src, src_pos, TIMES_OOP,
arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
__ lea(to, Address(dst, dst_pos, TIMES_OOP,
arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
__ movl2ptr(count, r11_length); // length
__ BIND(L_plain_copy);
__ jump(RuntimeAddress(oop_copy_entry));
__ BIND(L_checkcast_copy);
{
__ cmpl(Address(rax, lh_offset), objArray_lh);
__ jcc(Assembler::notEqual, L_failed);
arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
rax, L_failed);
const Register r11_dst_klass = r11;
__ load_klass(r11_dst_klass, dst); // reload
__ lea(from, Address(src, src_pos, TIMES_OOP,
arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
__ lea(to, Address(dst, dst_pos, TIMES_OOP,
arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
__ movl(count, length); // length (reloaded)
Register sco_temp = c_rarg3; // this register is free now
assert_different_registers(from, to, count, sco_temp,
r11_dst_klass, r10_src_klass);
assert_clean_int(count, sco_temp);
const int sco_offset = in_bytes(Klass::super_check_offset_offset());
__ movl(sco_temp, Address(r11_dst_klass, sco_offset));
assert_clean_int(sco_temp, rax);
generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy);
int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
__ movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset));
__ movl( sco_temp, Address(r11_dst_klass, sco_offset));
assert_clean_int(sco_temp, rax);
assert(c_rarg3 == sco_temp, "#3 already in place");
setup_arg_regs(4);
__ movptr(r8, r11_dst_klass); // dst.klass.element_klass, r8 is c_rarg4 on Linux/Solaris
__ jump(RuntimeAddress(checkcast_copy_entry));
}
__ BIND(L_failed);
__ xorptr(rax, rax);
__ notptr(rax); // return -1
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
return start;
}
void generate_arraycopy_stubs() {
address entry;
address entry_jbyte_arraycopy;
address entry_jshort_arraycopy;
address entry_jint_arraycopy;
address entry_oop_arraycopy;
address entry_jlong_arraycopy;
address entry_checkcast_arraycopy;
StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, &entry,
"jbyte_disjoint_arraycopy");
StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, entry, &entry_jbyte_arraycopy,
"jbyte_arraycopy");
StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry,
"jshort_disjoint_arraycopy");
StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, entry, &entry_jshort_arraycopy,
"jshort_arraycopy");
StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, false, &entry,
"jint_disjoint_arraycopy");
StubRoutines::_jint_arraycopy = generate_conjoint_int_oop_copy(false, false, entry,
&entry_jint_arraycopy, "jint_arraycopy");
StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, false, &entry,
"jlong_disjoint_arraycopy");
StubRoutines::_jlong_arraycopy = generate_conjoint_long_oop_copy(false, false, entry,
&entry_jlong_arraycopy, "jlong_arraycopy");
if (UseCompressedOops) {
StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, true, &entry,
"oop_disjoint_arraycopy");
StubRoutines::_oop_arraycopy = generate_conjoint_int_oop_copy(false, true, entry,
&entry_oop_arraycopy, "oop_arraycopy");
StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_int_oop_copy(false, true, &entry,
"oop_disjoint_arraycopy_uninit",
StubRoutines::_oop_arraycopy_uninit = generate_conjoint_int_oop_copy(false, true, entry,
NULL, "oop_arraycopy_uninit",
} else {
StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, true, &entry,
"oop_disjoint_arraycopy");
StubRoutines::_oop_arraycopy = generate_conjoint_long_oop_copy(false, true, entry,
&entry_oop_arraycopy, "oop_arraycopy");
StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_long_oop_copy(false, true, &entry,
"oop_disjoint_arraycopy_uninit",
StubRoutines::_oop_arraycopy_uninit = generate_conjoint_long_oop_copy(false, true, entry,
NULL, "oop_arraycopy_uninit",
}
StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL,
StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy",
entry_jbyte_arraycopy,
entry_jshort_arraycopy,
entry_jint_arraycopy,
entry_jlong_arraycopy);
StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy",
entry_jbyte_arraycopy,
entry_jshort_arraycopy,
entry_jint_arraycopy,
entry_oop_arraycopy,
entry_jlong_arraycopy,
entry_checkcast_arraycopy);
StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
StubRoutines::_arrayof_jbyte_disjoint_arraycopy = StubRoutines::_jbyte_disjoint_arraycopy;
StubRoutines::_arrayof_jbyte_arraycopy = StubRoutines::_jbyte_arraycopy;
StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy;
StubRoutines::_arrayof_jshort_arraycopy = StubRoutines::_jshort_arraycopy;
StubRoutines::_arrayof_jint_disjoint_arraycopy = StubRoutines::_jint_disjoint_arraycopy;
StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy;
StubRoutines::_arrayof_jlong_disjoint_arraycopy = StubRoutines::_jlong_disjoint_arraycopy;
StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy;
StubRoutines::_arrayof_oop_disjoint_arraycopy = StubRoutines::_oop_disjoint_arraycopy;
StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy;
StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit;
StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit;
}
void generate_math_stubs() {
{
StubCodeMark mark(this, "StubRoutines", "log");
StubRoutines::_intrinsic_log = (double (*)(double)) __ pc();
__ subq(rsp, 8);
__ movdbl(Address(rsp, 0), xmm0);
__ fld_d(Address(rsp, 0));
__ flog();
__ fstp_d(Address(rsp, 0));
__ movdbl(xmm0, Address(rsp, 0));
__ addq(rsp, 8);
__ ret(0);
}
{
StubCodeMark mark(this, "StubRoutines", "log10");
StubRoutines::_intrinsic_log10 = (double (*)(double)) __ pc();
__ subq(rsp, 8);
__ movdbl(Address(rsp, 0), xmm0);
__ fld_d(Address(rsp, 0));
__ flog10();
__ fstp_d(Address(rsp, 0));
__ movdbl(xmm0, Address(rsp, 0));
__ addq(rsp, 8);
__ ret(0);
}
{
StubCodeMark mark(this, "StubRoutines", "sin");
StubRoutines::_intrinsic_sin = (double (*)(double)) __ pc();
__ subq(rsp, 8);
__ movdbl(Address(rsp, 0), xmm0);
__ fld_d(Address(rsp, 0));
__ trigfunc('s');
__ fstp_d(Address(rsp, 0));
__ movdbl(xmm0, Address(rsp, 0));
__ addq(rsp, 8);
__ ret(0);
}
{
StubCodeMark mark(this, "StubRoutines", "cos");
StubRoutines::_intrinsic_cos = (double (*)(double)) __ pc();
__ subq(rsp, 8);
__ movdbl(Address(rsp, 0), xmm0);
__ fld_d(Address(rsp, 0));
__ trigfunc('c');
__ fstp_d(Address(rsp, 0));
__ movdbl(xmm0, Address(rsp, 0));
__ addq(rsp, 8);
__ ret(0);
}
{
StubCodeMark mark(this, "StubRoutines", "tan");
StubRoutines::_intrinsic_tan = (double (*)(double)) __ pc();
__ subq(rsp, 8);
__ movdbl(Address(rsp, 0), xmm0);
__ fld_d(Address(rsp, 0));
__ trigfunc('t');
__ fstp_d(Address(rsp, 0));
__ movdbl(xmm0, Address(rsp, 0));
__ addq(rsp, 8);
__ ret(0);
}
{
StubCodeMark mark(this, "StubRoutines", "exp");
StubRoutines::_intrinsic_exp = (double (*)(double)) __ pc();
__ subq(rsp, 8);
__ movdbl(Address(rsp, 0), xmm0);
__ fld_d(Address(rsp, 0));
__ exp_with_fallback(0);
__ fstp_d(Address(rsp, 0));
__ movdbl(xmm0, Address(rsp, 0));
__ addq(rsp, 8);
__ ret(0);
}
{
StubCodeMark mark(this, "StubRoutines", "pow");
StubRoutines::_intrinsic_pow = (double (*)(double,double)) __ pc();
__ subq(rsp, 8);
__ movdbl(Address(rsp, 0), xmm1);
__ fld_d(Address(rsp, 0));
__ movdbl(Address(rsp, 0), xmm0);
__ fld_d(Address(rsp, 0));
__ pow_with_fallback(0);
__ fstp_d(Address(rsp, 0));
__ movdbl(xmm0, Address(rsp, 0));
__ addq(rsp, 8);
__ ret(0);
}
}
enum {AESBlockSize = 16};
address generate_key_shuffle_mask() {
__ align(16);
StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
address start = __ pc();
__ emit_data64( 0x0405060700010203, relocInfo::none );
__ emit_data64( 0x0c0d0e0f08090a0b, relocInfo::none );
return start;
}
void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
__ movdqu(xmmdst, Address(key, offset));
if (xmm_shuf_mask != NULL) {
__ pshufb(xmmdst, xmm_shuf_mask);
} else {
__ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
}
}
address generate_aescrypt_encryptBlock() {
assert(UseAES, "need AES instructions and misaligned SSE support");
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
Label L_doLast;
address start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
const Register key = c_rarg2; // key array address
const Register keylen = rax;
const XMMRegister xmm_result = xmm0;
const XMMRegister xmm_key_shuf_mask = xmm1;
const XMMRegister xmm_temp1 = xmm2;
const XMMRegister xmm_temp2 = xmm3;
const XMMRegister xmm_temp3 = xmm4;
const XMMRegister xmm_temp4 = xmm5;
__ enter(); // required for proper stackwalking of RuntimeStub frame
__ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
__ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
__ movdqu(xmm_result, Address(from, 0)); // get 16 bytes of input
load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
__ pxor(xmm_result, xmm_temp1);
load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
__ aesenc(xmm_result, xmm_temp1);
__ aesenc(xmm_result, xmm_temp2);
__ aesenc(xmm_result, xmm_temp3);
__ aesenc(xmm_result, xmm_temp4);
load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
__ aesenc(xmm_result, xmm_temp1);
__ aesenc(xmm_result, xmm_temp2);
__ aesenc(xmm_result, xmm_temp3);
__ aesenc(xmm_result, xmm_temp4);
load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
__ cmpl(keylen, 44);
__ jccb(Assembler::equal, L_doLast);
__ aesenc(xmm_result, xmm_temp1);
__ aesenc(xmm_result, xmm_temp2);
load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
__ cmpl(keylen, 52);
__ jccb(Assembler::equal, L_doLast);
__ aesenc(xmm_result, xmm_temp1);
__ aesenc(xmm_result, xmm_temp2);
load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
__ BIND(L_doLast);
__ aesenc(xmm_result, xmm_temp1);
__ aesenclast(xmm_result, xmm_temp2);
__ movdqu(Address(to, 0), xmm_result); // store the result
__ xorptr(rax, rax); // return 0
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
return start;
}
address generate_aescrypt_decryptBlock() {
assert(UseAES, "need AES instructions and misaligned SSE support");
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
Label L_doLast;
address start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
const Register key = c_rarg2; // key array address
const Register keylen = rax;
const XMMRegister xmm_result = xmm0;
const XMMRegister xmm_key_shuf_mask = xmm1;
const XMMRegister xmm_temp1 = xmm2;
const XMMRegister xmm_temp2 = xmm3;
const XMMRegister xmm_temp3 = xmm4;
const XMMRegister xmm_temp4 = xmm5;
__ enter(); // required for proper stackwalking of RuntimeStub frame
__ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
__ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
__ movdqu(xmm_result, Address(from, 0));
load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
__ pxor (xmm_result, xmm_temp1);
__ aesdec(xmm_result, xmm_temp2);
__ aesdec(xmm_result, xmm_temp3);
__ aesdec(xmm_result, xmm_temp4);
load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
__ aesdec(xmm_result, xmm_temp1);
__ aesdec(xmm_result, xmm_temp2);
__ aesdec(xmm_result, xmm_temp3);
__ aesdec(xmm_result, xmm_temp4);
load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
__ cmpl(keylen, 44);
__ jccb(Assembler::equal, L_doLast);
__ aesdec(xmm_result, xmm_temp1);
__ aesdec(xmm_result, xmm_temp2);
load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
__ cmpl(keylen, 52);
__ jccb(Assembler::equal, L_doLast);
__ aesdec(xmm_result, xmm_temp1);
__ aesdec(xmm_result, xmm_temp2);
load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
__ BIND(L_doLast);
__ aesdec(xmm_result, xmm_temp1);
__ aesdec(xmm_result, xmm_temp2);
__ aesdeclast(xmm_result, xmm_temp3);
__ movdqu(Address(to, 0), xmm_result); // store the result
__ xorptr(rax, rax); // return 0
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
return start;
}
address generate_cipherBlockChaining_encryptAESCrypt() {
assert(UseAES, "need AES instructions and misaligned SSE support");
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
address start = __ pc();
Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
const Register key = c_rarg2; // key array address
const Register rvec = c_rarg3; // r byte array initialized from initvector array address
#ifndef _WIN64
const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
#else
const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64
const Register len_reg = r10; // pick the first volatile windows register
#endif
const Register pos = rax;
const XMMRegister xmm_result = xmm0;
const XMMRegister xmm_temp = xmm1;
const int XMM_REG_NUM_KEY_FIRST = 2;
const int XMM_REG_NUM_KEY_LAST = 15;
const XMMRegister xmm_key0 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
const XMMRegister xmm_key10 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+10);
const XMMRegister xmm_key11 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+11);
const XMMRegister xmm_key12 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+12);
const XMMRegister xmm_key13 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+13);
__ enter(); // required for proper stackwalking of RuntimeStub frame
#ifdef _WIN64
__ movl(len_reg, len_mem);
__ subptr(rsp, -rsp_after_call_off * wordSize);
for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
__ movdqu(xmm_save(i), as_XMMRegister(i));
}
#else
__ push(len_reg); // Save
#endif
const XMMRegister xmm_key_shuf_mask = xmm_temp; // used temporarily to swap key bytes up front
__ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum <= XMM_REG_NUM_KEY_FIRST+10; rnum++) {
load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
offset += 0x10;
}
__ movdqu(xmm_result, Address(rvec, 0x00)); // initialize xmm_result with r vec
__ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
__ cmpl(rax, 44);
__ jcc(Assembler::notEqual, L_key_192_256);
__ movptr(pos, 0);
__ align(OptoLoopAlignment);
__ BIND(L_loopTop_128);
__ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input
__ pxor (xmm_result, xmm_temp); // xor with the current r vector
__ pxor (xmm_result, xmm_key0); // do the aes rounds
for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 9; rnum++) {
__ aesenc(xmm_result, as_XMMRegister(rnum));
}
__ aesenclast(xmm_result, xmm_key10);
__ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
__ addptr(pos, AESBlockSize);
__ subptr(len_reg, AESBlockSize);
__ jcc(Assembler::notEqual, L_loopTop_128);
__ BIND(L_exit);
__ movdqu(Address(rvec, 0), xmm_result); // final value of r stored in rvec of CipherBlockChaining object
#ifdef _WIN64
for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
__ movdqu(as_XMMRegister(i), xmm_save(i));
}
__ movl(rax, len_mem);
#else
__ pop(rax); // return length
#endif
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
__ BIND(L_key_192_256);
load_key(xmm_key11, key, 0xb0, xmm_key_shuf_mask);
load_key(xmm_key12, key, 0xc0, xmm_key_shuf_mask);
__ cmpl(rax, 52);
__ jcc(Assembler::notEqual, L_key_256);
__ movptr(pos, 0);
__ align(OptoLoopAlignment);
__ BIND(L_loopTop_192);
__ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input
__ pxor (xmm_result, xmm_temp); // xor with the current r vector
__ pxor (xmm_result, xmm_key0); // do the aes rounds
for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 11; rnum++) {
__ aesenc(xmm_result, as_XMMRegister(rnum));
}
__ aesenclast(xmm_result, xmm_key12);
__ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
__ addptr(pos, AESBlockSize);
__ subptr(len_reg, AESBlockSize);
__ jcc(Assembler::notEqual, L_loopTop_192);
__ jmp(L_exit);
__ BIND(L_key_256);
load_key(xmm_key13, key, 0xd0, xmm_key_shuf_mask);
__ movptr(pos, 0);
__ align(OptoLoopAlignment);
__ BIND(L_loopTop_256);
__ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input
__ pxor (xmm_result, xmm_temp); // xor with the current r vector
__ pxor (xmm_result, xmm_key0); // do the aes rounds
for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 13; rnum++) {
__ aesenc(xmm_result, as_XMMRegister(rnum));
}
load_key(xmm_temp, key, 0xe0);
__ aesenclast(xmm_result, xmm_temp);
__ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
__ addptr(pos, AESBlockSize);
__ subptr(len_reg, AESBlockSize);
__ jcc(Assembler::notEqual, L_loopTop_256);
__ jmp(L_exit);
return start;
}
void generate_safefetch(const char* name, int size, address* entry,
address* fault_pc, address* continuation_pc) {
StubCodeMark mark(this, "StubRoutines", name);
switch (size) {
case 4:
__ movl(c_rarg1, Address(c_rarg0, 0));
break;
case 8:
__ movq(c_rarg1, Address(c_rarg0, 0));
break;
default:
ShouldNotReachHere();
}
__ movq(rax, c_rarg1);
__ ret(0);
}
address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
assert(UseAES, "need AES instructions and misaligned SSE support");
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
address start = __ pc();
Label L_exit, L_key_192_256, L_key_256;
Label L_singleBlock_loopTop_128, L_multiBlock_loopTop_128;
Label L_singleBlock_loopTop_192, L_singleBlock_loopTop_256;
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
const Register key = c_rarg2; // key array address
const Register rvec = c_rarg3; // r byte array initialized from initvector array address
#ifndef _WIN64
const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
#else
const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64
const Register len_reg = r10; // pick the first volatile windows register
#endif
const Register pos = rax;
const int XMM_REG_NUM_KEY_FIRST = 5;
const int XMM_REG_NUM_KEY_LAST = 15;
const XMMRegister xmm_key_first = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
const XMMRegister xmm_key_last = as_XMMRegister(XMM_REG_NUM_KEY_LAST);
__ enter(); // required for proper stackwalking of RuntimeStub frame
#ifdef _WIN64
__ movl(len_reg, len_mem);
__ subptr(rsp, -rsp_after_call_off * wordSize);
for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
__ movdqu(xmm_save(i), as_XMMRegister(i));
}
#else
__ push(len_reg); // Save
#endif
const XMMRegister xmm_key_shuf_mask = xmm1; // used temporarily to swap key bytes up front
__ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x10; rnum < XMM_REG_NUM_KEY_LAST; rnum++) {
load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
offset += 0x10;
}
load_key(xmm_key_last, key, 0x00, xmm_key_shuf_mask);
const XMMRegister xmm_prev_block_cipher = xmm1; // holds cipher of previous block
const XMMRegister xmm_result0 = xmm0;
const XMMRegister xmm_result1 = xmm2;
const XMMRegister xmm_result2 = xmm3;
const XMMRegister xmm_result3 = xmm4;
__ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // initialize with initial rvec
__ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
__ cmpl(rax, 44);
__ jcc(Assembler::notEqual, L_key_192_256);
__ movptr(pos, 0);
__ align(OptoLoopAlignment);
__ BIND(L_multiBlock_loopTop_128);
__ cmpptr(len_reg, 4*AESBlockSize); // see if at least 4 blocks left
__ jcc(Assembler::less, L_singleBlock_loopTop_128);
__ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0*AESBlockSize)); // get next 4 blocks into xmmresult registers
__ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1*AESBlockSize));
__ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2*AESBlockSize));
__ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3*AESBlockSize));
#define DoFour(opc, src_reg) \
__ opc(xmm_result0, src_reg); \
__ opc(xmm_result1, src_reg); \
__ opc(xmm_result2, src_reg); \
__ opc(xmm_result3, src_reg);
DoFour(pxor, xmm_key_first);
for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
DoFour(aesdec, as_XMMRegister(rnum));
}
DoFour(aesdeclast, xmm_key_last);
__ pxor(xmm_result0, xmm_prev_block_cipher);
__ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0*AESBlockSize));
__ pxor(xmm_result1, xmm_prev_block_cipher);
__ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1*AESBlockSize));
__ pxor(xmm_result2, xmm_prev_block_cipher);
__ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2*AESBlockSize));
__ pxor(xmm_result3, xmm_prev_block_cipher);
__ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3*AESBlockSize)); // this will carry over to next set of blocks
__ movdqu(Address(to, pos, Address::times_1, 0*AESBlockSize), xmm_result0); // store 4 results into the next 64 bytes of output
__ movdqu(Address(to, pos, Address::times_1, 1*AESBlockSize), xmm_result1);
__ movdqu(Address(to, pos, Address::times_1, 2*AESBlockSize), xmm_result2);
__ movdqu(Address(to, pos, Address::times_1, 3*AESBlockSize), xmm_result3);
__ addptr(pos, 4*AESBlockSize);
__ subptr(len_reg, 4*AESBlockSize);
__ jmp(L_multiBlock_loopTop_128);
const XMMRegister xmm_result = xmm0;
const XMMRegister xmm_prev_block_cipher_save = xmm2;
const XMMRegister xmm_key11 = xmm3;
const XMMRegister xmm_key12 = xmm4;
const XMMRegister xmm_temp = xmm4;
__ align(OptoLoopAlignment);
__ BIND(L_singleBlock_loopTop_128);
__ cmpptr(len_reg, 0); // any blocks left??
__ jcc(Assembler::equal, L_exit);
__ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
__ movdqa(xmm_prev_block_cipher_save, xmm_result); // save for next r vector
__ pxor (xmm_result, xmm_key_first); // do the aes dec rounds
for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
__ aesdec(xmm_result, as_XMMRegister(rnum));
}
__ aesdeclast(xmm_result, xmm_key_last);
__ pxor (xmm_result, xmm_prev_block_cipher); // xor with the current r vector
__ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
__ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save); // set up next r vector with cipher input from this block
__ addptr(pos, AESBlockSize);
__ subptr(len_reg, AESBlockSize);
__ jmp(L_singleBlock_loopTop_128);
__ BIND(L_exit);
__ movdqu(Address(rvec, 0), xmm_prev_block_cipher); // final value of r stored in rvec of CipherBlockChaining object
#ifdef _WIN64
for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
__ movdqu(as_XMMRegister(i), xmm_save(i));
}
__ movl(rax, len_mem);
#else
__ pop(rax); // return length
#endif
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
__ BIND(L_key_192_256);
load_key(xmm_key11, key, 0xb0);
__ cmpl(rax, 52);
__ jcc(Assembler::notEqual, L_key_256);
load_key(xmm_key12, key, 0xc0); // 192-bit key goes up to c0
__ movptr(pos, 0);
__ align(OptoLoopAlignment);
__ BIND(L_singleBlock_loopTop_192);
__ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
__ movdqa(xmm_prev_block_cipher_save, xmm_result); // save for next r vector
__ pxor (xmm_result, xmm_key_first); // do the aes dec rounds
for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
__ aesdec(xmm_result, as_XMMRegister(rnum));
}
__ aesdec(xmm_result, xmm_key11);
__ aesdec(xmm_result, xmm_key12);
__ aesdeclast(xmm_result, xmm_key_last); // xmm15 always came from key+0
__ pxor (xmm_result, xmm_prev_block_cipher); // xor with the current r vector
__ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
__ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save); // set up next r vector with cipher input from this block
__ addptr(pos, AESBlockSize);
__ subptr(len_reg, AESBlockSize);
__ jcc(Assembler::notEqual,L_singleBlock_loopTop_192);
__ jmp(L_exit);
__ BIND(L_key_256);
__ movptr(pos, 0);
__ align(OptoLoopAlignment);
__ BIND(L_singleBlock_loopTop_256);
__ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
__ movdqa(xmm_prev_block_cipher_save, xmm_result); // save for next r vector
__ pxor (xmm_result, xmm_key_first); // do the aes dec rounds
for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
__ aesdec(xmm_result, as_XMMRegister(rnum));
}
__ aesdec(xmm_result, xmm_key11);
load_key(xmm_temp, key, 0xc0);
__ aesdec(xmm_result, xmm_temp);
load_key(xmm_temp, key, 0xd0);
__ aesdec(xmm_result, xmm_temp);
load_key(xmm_temp, key, 0xe0); // 256-bit key goes up to e0
__ aesdec(xmm_result, xmm_temp);
__ aesdeclast(xmm_result, xmm_key_last); // xmm15 came from key+0
__ pxor (xmm_result, xmm_prev_block_cipher); // xor with the current r vector
__ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
__ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save); // set up next r vector with cipher input from this block
__ addptr(pos, AESBlockSize);
__ subptr(len_reg, AESBlockSize);
__ jcc(Assembler::notEqual,L_singleBlock_loopTop_256);
__ jmp(L_exit);
return start;
}
address generate_ghash_long_swap_mask() {
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
address start = __ pc();
__ emit_data64(0x0f0e0d0c0b0a0908, relocInfo::none );
__ emit_data64(0x0706050403020100, relocInfo::none );
return start;
}
address generate_ghash_byte_swap_mask() {
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
address start = __ pc();
__ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none );
__ emit_data64(0x0001020304050607, relocInfo::none );
return start;
}
address generate_ghash_processBlocks() {
__ align(CodeEntryAlignment);
Label L_ghash_loop, L_exit;
StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
address start = __ pc();
const Register state = c_rarg0;
const Register subkeyH = c_rarg1;
const Register data = c_rarg2;
const Register blocks = c_rarg3;
#ifdef _WIN64
const int XMM_REG_LAST = 10;
#endif
const XMMRegister xmm_temp0 = xmm0;
const XMMRegister xmm_temp1 = xmm1;
const XMMRegister xmm_temp2 = xmm2;
const XMMRegister xmm_temp3 = xmm3;
const XMMRegister xmm_temp4 = xmm4;
const XMMRegister xmm_temp5 = xmm5;
const XMMRegister xmm_temp6 = xmm6;
const XMMRegister xmm_temp7 = xmm7;
const XMMRegister xmm_temp8 = xmm8;
const XMMRegister xmm_temp9 = xmm9;
const XMMRegister xmm_temp10 = xmm10;
__ enter();
#ifdef _WIN64
__ subptr(rsp, -rsp_after_call_off * wordSize);
for (int i = 6; i <= XMM_REG_LAST; i++) {
__ movdqu(xmm_save(i), as_XMMRegister(i));
}
#endif
__ movdqu(xmm_temp10, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
__ movdqu(xmm_temp0, Address(state, 0));
__ pshufb(xmm_temp0, xmm_temp10);
__ BIND(L_ghash_loop);
__ movdqu(xmm_temp2, Address(data, 0));
__ pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr()));
__ movdqu(xmm_temp1, Address(subkeyH, 0));
__ pshufb(xmm_temp1, xmm_temp10);
__ pxor(xmm_temp0, xmm_temp2);
__ movdqu(xmm_temp3, xmm_temp0);
__ pclmulqdq(xmm_temp3, xmm_temp1, 0); // xmm3 holds a0*b0
__ movdqu(xmm_temp4, xmm_temp0);
__ pclmulqdq(xmm_temp4, xmm_temp1, 16); // xmm4 holds a0*b1
__ movdqu(xmm_temp5, xmm_temp0);
__ pclmulqdq(xmm_temp5, xmm_temp1, 1); // xmm5 holds a1*b0
__ movdqu(xmm_temp6, xmm_temp0);
__ pclmulqdq(xmm_temp6, xmm_temp1, 17); // xmm6 holds a1*b1
__ pxor(xmm_temp4, xmm_temp5); // xmm4 holds a0*b1 + a1*b0
__ movdqu(xmm_temp5, xmm_temp4); // move the contents of xmm4 to xmm5
__ psrldq(xmm_temp4, 8); // shift by xmm4 64 bits to the right
__ pslldq(xmm_temp5, 8); // shift by xmm5 64 bits to the left
__ pxor(xmm_temp3, xmm_temp5);
__ pxor(xmm_temp6, xmm_temp4); // Register pair <xmm6:xmm3> holds the result
__ movdqu(xmm_temp7, xmm_temp3);
__ movdqu(xmm_temp8, xmm_temp6);
__ pslld(xmm_temp3, 1);
__ pslld(xmm_temp6, 1);
__ psrld(xmm_temp7, 31);
__ psrld(xmm_temp8, 31);
__ movdqu(xmm_temp9, xmm_temp7);
__ pslldq(xmm_temp8, 4);
__ pslldq(xmm_temp7, 4);
__ psrldq(xmm_temp9, 12);
__ por(xmm_temp3, xmm_temp7);
__ por(xmm_temp6, xmm_temp8);
__ por(xmm_temp6, xmm_temp9);
__ movdqu(xmm_temp7, xmm_temp3);
__ movdqu(xmm_temp8, xmm_temp3);
__ movdqu(xmm_temp9, xmm_temp3);
__ pslld(xmm_temp7, 31); // packed right shift shifting << 31
__ pslld(xmm_temp8, 30); // packed right shift shifting << 30
__ pslld(xmm_temp9, 25); // packed right shift shifting << 25
__ pxor(xmm_temp7, xmm_temp8); // xor the shifted versions
__ pxor(xmm_temp7, xmm_temp9);
__ movdqu(xmm_temp8, xmm_temp7);
__ pslldq(xmm_temp7, 12);
__ psrldq(xmm_temp8, 4);
__ pxor(xmm_temp3, xmm_temp7); // first phase of the reduction complete
__ movdqu(xmm_temp2, xmm_temp3);
__ movdqu(xmm_temp4, xmm_temp3);
__ movdqu(xmm_temp5, xmm_temp3);
__ psrld(xmm_temp2, 1); // packed left shifting >> 1
__ psrld(xmm_temp4, 2); // packed left shifting >> 2
__ psrld(xmm_temp5, 7); // packed left shifting >> 7
__ pxor(xmm_temp2, xmm_temp4); // xor the shifted versions
__ pxor(xmm_temp2, xmm_temp5);
__ pxor(xmm_temp2, xmm_temp8);
__ pxor(xmm_temp3, xmm_temp2);
__ pxor(xmm_temp6, xmm_temp3); // the result is in xmm6
__ decrement(blocks);
__ jcc(Assembler::zero, L_exit);
__ movdqu(xmm_temp0, xmm_temp6);
__ addptr(data, 16);
__ jmp(L_ghash_loop);
__ BIND(L_exit);
__ pshufb(xmm_temp6, xmm_temp10); // Byte swap 16-byte result
__ movdqu(Address(state, 0), xmm_temp6); // store the result
#ifdef _WIN64
for (int i = 6; i <= XMM_REG_LAST; i++) {
__ movdqu(as_XMMRegister(i), xmm_save(i));
}
#endif
__ leave();
__ ret(0);
return start;
}
address generate_updateBytesCRC32() {
assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
address start = __ pc();
const Register crc = c_rarg0; // crc
const Register buf = c_rarg1; // source java byte array address
const Register len = c_rarg2; // length
const Register table = c_rarg3; // crc_table address (reuse register)
const Register tmp = r11;
assert_different_registers(crc, buf, len, table, tmp, rax);
BLOCK_COMMENT("Entry:");
__ enter(); // required for proper stackwalking of RuntimeStub frame
__ kernel_crc32(crc, buf, len, table, tmp);
__ movl(rax, crc);
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
return start;
}
address generate_multiplyToLen() {
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", "multiplyToLen");
address start = __ pc();
const Register x = rdi;
const Register xlen = rax;
const Register y = rsi;
const Register ylen = rcx;
const Register z = r8;
const Register zlen = r11;
const Register tmp1 = r12;
const Register tmp2 = r13;
const Register tmp3 = r14;
const Register tmp4 = r15;
const Register tmp5 = rbx;
BLOCK_COMMENT("Entry:");
__ enter(); // required for proper stackwalking of RuntimeStub frame
#ifndef _WIN64
__ movptr(zlen, r9); // Save r9 in r11 - zlen
#endif
setup_arg_regs(4); // x => rdi, xlen => rsi, y => rdx
#ifdef _WIN64
__ movptr(z, Address(rsp, 6 * wordSize));
__ movptr(zlen, Address(rsp, 7 * wordSize));
#endif
__ movptr(xlen, rsi);
__ movptr(y, rdx);
__ multiply_to_len(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5);
restore_arg_regs();
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
return start;
}
address generate_squareToLen() {
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", "squareToLen");
address start = __ pc();
const Register x = rdi;
const Register len = rsi;
const Register z = r8;
const Register zlen = rcx;
const Register tmp1 = r12;
const Register tmp2 = r13;
const Register tmp3 = r14;
const Register tmp4 = r15;
const Register tmp5 = rbx;
BLOCK_COMMENT("Entry:");
__ enter(); // required for proper stackwalking of RuntimeStub frame
setup_arg_regs(4); // x => rdi, len => rsi, z => rdx
__ movptr(r8, rdx);
__ square_to_len(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax);
restore_arg_regs();
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
return start;
}
address generate_mulAdd() {
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", "mulAdd");
address start = __ pc();
const Register out = rdi;
const Register in = rsi;
const Register offset = r11;
const Register len = rcx;
const Register k = r8;
const Register tmp1 = r12;
const Register tmp2 = r13;
const Register tmp3 = r14;
const Register tmp4 = r15;
const Register tmp5 = rbx;
BLOCK_COMMENT("Entry:");
__ enter(); // required for proper stackwalking of RuntimeStub frame
setup_arg_regs(4); // out => rdi, in => rsi, offset => rdx
#ifdef _WIN64
__ movl(k, Address(rsp, 6 * wordSize));
#endif
__ movptr(r11, rdx); // move offset in rdx to offset(r11)
__ mul_add(out, in, offset, len, k, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax);
restore_arg_regs();
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
return start;
}
#undef __
#define __ masm->
address generate_throw_exception(const char* name,
address runtime_entry,
Register arg1 = noreg,
Register arg2 = noreg) {
enum layout {
rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,
rbp_off2,
return_off,
return_off2,
framesize // inclusive of return address
};
int insts_size = 512;
int locs_size = 64;
CodeBuffer code(name, insts_size, locs_size);
OopMapSet* oop_maps = new OopMapSet();
MacroAssembler* masm = new MacroAssembler(&code);
address start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
assert(is_even(framesize/2), "sp not 16-byte aligned");
__ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog
int frame_complete = __ pc() - start;
address the_pc = __ pc();
__ set_last_Java_frame(rsp, rbp, the_pc);
__ andptr(rsp, -(StackAlignmentInBytes)); // Align stack
if (arg1 != noreg) {
assert(arg2 != c_rarg1, "clobbered");
__ movptr(c_rarg1, arg1);
}
if (arg2 != noreg) {
__ movptr(c_rarg2, arg2);
}
__ movptr(c_rarg0, r15_thread);
BLOCK_COMMENT("call runtime_entry");
__ call(RuntimeAddress(runtime_entry));
OopMap* map = new OopMap(framesize, 0);
oop_maps->add_gc_map(the_pc - start, map);
__ reset_last_Java_frame(true);
__ leave(); // required for proper stackwalking of RuntimeStub frame
#ifdef ASSERT
Label L;
__ cmpptr(Address(r15_thread, Thread::pending_exception_offset()),
(int32_t) NULL_WORD);
__ jcc(Assembler::notEqual, L);
__ should_not_reach_here();
__ bind(L);
#endif // ASSERT
__ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
RuntimeStub* stub =
RuntimeStub::new_runtime_stub(name,
&code,
frame_complete,
(framesize >> (LogBytesPerWord - LogBytesPerInt)),
oop_maps, false);
return stub->entry_point();
}
void create_control_words() {
StubRoutines::_fpu_cntrl_wrd_std = 0x027F;
StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
StubRoutines::_fpu_cntrl_wrd_24 = 0x007F;
StubRoutines::_fpu_cntrl_wrd_64 = 0x037F;
StubRoutines::_mxcsr_std = 0x1F80;
StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
}
void generate_initial() {
create_control_words();
StubRoutines::_forward_exception_entry = generate_forward_exception();
StubRoutines::_call_stub_entry =
generate_call_stub(StubRoutines::_call_stub_return_address);
StubRoutines::_catch_exception_entry = generate_catch_exception();
StubRoutines::_atomic_xchg_entry = generate_atomic_xchg();
StubRoutines::_atomic_xchg_ptr_entry = generate_atomic_xchg_ptr();
StubRoutines::_atomic_cmpxchg_entry = generate_atomic_cmpxchg();
StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
StubRoutines::_atomic_add_entry = generate_atomic_add();
StubRoutines::_atomic_add_ptr_entry = generate_atomic_add_ptr();
StubRoutines::_fence_entry = generate_orderaccess_fence();
StubRoutines::_handler_for_unsafe_access_entry =
generate_handler_for_unsafe_access();
StubRoutines::x86::_get_previous_fp_entry = generate_get_previous_fp();
StubRoutines::x86::_get_previous_sp_entry = generate_get_previous_sp();
StubRoutines::x86::_verify_mxcsr_entry = generate_verify_mxcsr();
StubRoutines::_throw_StackOverflowError_entry =
generate_throw_exception("StackOverflowError throw_exception",
CAST_FROM_FN_PTR(address,
SharedRuntime::
throw_StackOverflowError));
if (UseCRC32Intrinsics) {
StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
}
}
void generate_all() {
StubRoutines::_throw_AbstractMethodError_entry =
generate_throw_exception("AbstractMethodError throw_exception",
CAST_FROM_FN_PTR(address,
SharedRuntime::
throw_AbstractMethodError));
StubRoutines::_throw_IncompatibleClassChangeError_entry =
generate_throw_exception("IncompatibleClassChangeError throw_exception",
CAST_FROM_FN_PTR(address,
SharedRuntime::
throw_IncompatibleClassChangeError));
StubRoutines::_throw_NullPointerException_at_call_entry =
generate_throw_exception("NullPointerException at call throw_exception",
CAST_FROM_FN_PTR(address,
SharedRuntime::
throw_NullPointerException_at_call));
StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
StubRoutines::x86::_float_sign_mask = generate_fp_mask("float_sign_mask", 0x7FFFFFFF7FFFFFFF);
StubRoutines::x86::_float_sign_flip = generate_fp_mask("float_sign_flip", 0x8000000080000000);
StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF);
StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000);
StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
generate_arraycopy_stubs();
generate_math_stubs();
if (UseAESIntrinsics) {
StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask(); // needed by the others
StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
}
if (UseGHASHIntrinsics) {
StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
}
generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry,
&StubRoutines::_safefetch32_fault_pc,
&StubRoutines::_safefetch32_continuation_pc);
generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
&StubRoutines::_safefetchN_fault_pc,
&StubRoutines::_safefetchN_continuation_pc);
#ifdef COMPILER2
if (UseMultiplyToLenIntrinsic) {
StubRoutines::_multiplyToLen = generate_multiplyToLen();
}
if (UseSquareToLenIntrinsic) {
StubRoutines::_squareToLen = generate_squareToLen();
}
if (UseMulAddIntrinsic) {
StubRoutines::_mulAdd = generate_mulAdd();
}
#ifndef _WINDOWS
if (UseMontgomeryMultiplyIntrinsic) {
StubRoutines::_montgomeryMultiply
= CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_multiply);
}
if (UseMontgomerySquareIntrinsic) {
StubRoutines::_montgomerySquare
= CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square);
}
#endif // WINDOWS
#endif // COMPILER2
}
public:
StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
if (all) {
generate_all();
} else {
generate_initial();
}
}
}; // end class declaration
void StubGenerator_generate(CodeBuffer* code, bool all) {
StubGenerator g(code, all);
}
C:\hotspot-69087d08d473\src\cpu\x86\vm/stubRoutines_x86.cpp
#include "precompiled.hpp"
#include "runtime/deoptimization.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/stubRoutines.hpp"
#include "runtime/thread.inline.hpp"
address StubRoutines::x86::_verify_mxcsr_entry = NULL;
address StubRoutines::x86::_key_shuffle_mask_addr = NULL;
address StubRoutines::x86::_ghash_long_swap_mask_addr = NULL;
address StubRoutines::x86::_ghash_byte_swap_mask_addr = NULL;
uint64_t StubRoutines::x86::_crc_by128_masks[] =
{
((uint64_t) 0xffffffffUL), /* low of K_M_64 */
((uint64_t) 0xb1e6b092U << 1), /* high of K_M_64 */
((uint64_t) 0xba8ccbe8U << 1), /* low of K_160_96 */
((uint64_t) 0x6655004fU << 1), /* high of K_160_96 */
((uint64_t) 0xaa2215eaU << 1), /* low of K_544_480 */
((uint64_t) 0xe3720acbU << 1) /* high of K_544_480 */
};
juint StubRoutines::x86::_crc_table[] =
{
0x00000000UL, 0x77073096UL, 0xee0e612cUL, 0x990951baUL, 0x076dc419UL,
0x706af48fUL, 0xe963a535UL, 0x9e6495a3UL, 0x0edb8832UL, 0x79dcb8a4UL,
0xe0d5e91eUL, 0x97d2d988UL, 0x09b64c2bUL, 0x7eb17cbdUL, 0xe7b82d07UL,
0x90bf1d91UL, 0x1db71064UL, 0x6ab020f2UL, 0xf3b97148UL, 0x84be41deUL,
0x1adad47dUL, 0x6ddde4ebUL, 0xf4d4b551UL, 0x83d385c7UL, 0x136c9856UL,
0x646ba8c0UL, 0xfd62f97aUL, 0x8a65c9ecUL, 0x14015c4fUL, 0x63066cd9UL,
0xfa0f3d63UL, 0x8d080df5UL, 0x3b6e20c8UL, 0x4c69105eUL, 0xd56041e4UL,
0xa2677172UL, 0x3c03e4d1UL, 0x4b04d447UL, 0xd20d85fdUL, 0xa50ab56bUL,
0x35b5a8faUL, 0x42b2986cUL, 0xdbbbc9d6UL, 0xacbcf940UL, 0x32d86ce3UL,
0x45df5c75UL, 0xdcd60dcfUL, 0xabd13d59UL, 0x26d930acUL, 0x51de003aUL,
0xc8d75180UL, 0xbfd06116UL, 0x21b4f4b5UL, 0x56b3c423UL, 0xcfba9599UL,
0xb8bda50fUL, 0x2802b89eUL, 0x5f058808UL, 0xc60cd9b2UL, 0xb10be924UL,
0x2f6f7c87UL, 0x58684c11UL, 0xc1611dabUL, 0xb6662d3dUL, 0x76dc4190UL,
0x01db7106UL, 0x98d220bcUL, 0xefd5102aUL, 0x71b18589UL, 0x06b6b51fUL,
0x9fbfe4a5UL, 0xe8b8d433UL, 0x7807c9a2UL, 0x0f00f934UL, 0x9609a88eUL,
0xe10e9818UL, 0x7f6a0dbbUL, 0x086d3d2dUL, 0x91646c97UL, 0xe6635c01UL,
0x6b6b51f4UL, 0x1c6c6162UL, 0x856530d8UL, 0xf262004eUL, 0x6c0695edUL,
0x1b01a57bUL, 0x8208f4c1UL, 0xf50fc457UL, 0x65b0d9c6UL, 0x12b7e950UL,
0x8bbeb8eaUL, 0xfcb9887cUL, 0x62dd1ddfUL, 0x15da2d49UL, 0x8cd37cf3UL,
0xfbd44c65UL, 0x4db26158UL, 0x3ab551ceUL, 0xa3bc0074UL, 0xd4bb30e2UL,
0x4adfa541UL, 0x3dd895d7UL, 0xa4d1c46dUL, 0xd3d6f4fbUL, 0x4369e96aUL,
0x346ed9fcUL, 0xad678846UL, 0xda60b8d0UL, 0x44042d73UL, 0x33031de5UL,
0xaa0a4c5fUL, 0xdd0d7cc9UL, 0x5005713cUL, 0x270241aaUL, 0xbe0b1010UL,
0xc90c2086UL, 0x5768b525UL, 0x206f85b3UL, 0xb966d409UL, 0xce61e49fUL,
0x5edef90eUL, 0x29d9c998UL, 0xb0d09822UL, 0xc7d7a8b4UL, 0x59b33d17UL,
0x2eb40d81UL, 0xb7bd5c3bUL, 0xc0ba6cadUL, 0xedb88320UL, 0x9abfb3b6UL,
0x03b6e20cUL, 0x74b1d29aUL, 0xead54739UL, 0x9dd277afUL, 0x04db2615UL,
0x73dc1683UL, 0xe3630b12UL, 0x94643b84UL, 0x0d6d6a3eUL, 0x7a6a5aa8UL,
0xe40ecf0bUL, 0x9309ff9dUL, 0x0a00ae27UL, 0x7d079eb1UL, 0xf00f9344UL,
0x8708a3d2UL, 0x1e01f268UL, 0x6906c2feUL, 0xf762575dUL, 0x806567cbUL,
0x196c3671UL, 0x6e6b06e7UL, 0xfed41b76UL, 0x89d32be0UL, 0x10da7a5aUL,
0x67dd4accUL, 0xf9b9df6fUL, 0x8ebeeff9UL, 0x17b7be43UL, 0x60b08ed5UL,
0xd6d6a3e8UL, 0xa1d1937eUL, 0x38d8c2c4UL, 0x4fdff252UL, 0xd1bb67f1UL,
0xa6bc5767UL, 0x3fb506ddUL, 0x48b2364bUL, 0xd80d2bdaUL, 0xaf0a1b4cUL,
0x36034af6UL, 0x41047a60UL, 0xdf60efc3UL, 0xa867df55UL, 0x316e8eefUL,
0x4669be79UL, 0xcb61b38cUL, 0xbc66831aUL, 0x256fd2a0UL, 0x5268e236UL,
0xcc0c7795UL, 0xbb0b4703UL, 0x220216b9UL, 0x5505262fUL, 0xc5ba3bbeUL,
0xb2bd0b28UL, 0x2bb45a92UL, 0x5cb36a04UL, 0xc2d7ffa7UL, 0xb5d0cf31UL,
0x2cd99e8bUL, 0x5bdeae1dUL, 0x9b64c2b0UL, 0xec63f226UL, 0x756aa39cUL,
0x026d930aUL, 0x9c0906a9UL, 0xeb0e363fUL, 0x72076785UL, 0x05005713UL,
0x95bf4a82UL, 0xe2b87a14UL, 0x7bb12baeUL, 0x0cb61b38UL, 0x92d28e9bUL,
0xe5d5be0dUL, 0x7cdcefb7UL, 0x0bdbdf21UL, 0x86d3d2d4UL, 0xf1d4e242UL,
0x68ddb3f8UL, 0x1fda836eUL, 0x81be16cdUL, 0xf6b9265bUL, 0x6fb077e1UL,
0x18b74777UL, 0x88085ae6UL, 0xff0f6a70UL, 0x66063bcaUL, 0x11010b5cUL,
0x8f659effUL, 0xf862ae69UL, 0x616bffd3UL, 0x166ccf45UL, 0xa00ae278UL,
0xd70dd2eeUL, 0x4e048354UL, 0x3903b3c2UL, 0xa7672661UL, 0xd06016f7UL,
0x4969474dUL, 0x3e6e77dbUL, 0xaed16a4aUL, 0xd9d65adcUL, 0x40df0b66UL,
0x37d83bf0UL, 0xa9bcae53UL, 0xdebb9ec5UL, 0x47b2cf7fUL, 0x30b5ffe9UL,
0xbdbdf21cUL, 0xcabac28aUL, 0x53b39330UL, 0x24b4a3a6UL, 0xbad03605UL,
0xcdd70693UL, 0x54de5729UL, 0x23d967bfUL, 0xb3667a2eUL, 0xc4614ab8UL,
0x5d681b02UL, 0x2a6f2b94UL, 0xb40bbe37UL, 0xc30c8ea1UL, 0x5a05df1bUL,
0x2d02ef8dUL
};
C:\hotspot-69087d08d473\src\cpu\x86\vm/stubRoutines_x86.hpp
#ifndef CPU_X86_VM_STUBROUTINES_X86_HPP
#define CPU_X86_VM_STUBROUTINES_X86_HPP
private:
static address _verify_mxcsr_entry;
static address _key_shuffle_mask_addr;
static uint64_t _crc_by128_masks[];
static juint _crc_table[];
static address _ghash_long_swap_mask_addr;
static address _ghash_byte_swap_mask_addr;
public:
static address verify_mxcsr_entry() { return _verify_mxcsr_entry; }
static address key_shuffle_mask_addr() { return _key_shuffle_mask_addr; }
static address crc_by128_masks_addr() { return (address)_crc_by128_masks; }
static address ghash_long_swap_mask_addr() { return _ghash_long_swap_mask_addr; }
static address ghash_byte_swap_mask_addr() { return _ghash_byte_swap_mask_addr; }
#endif // CPU_X86_VM_STUBROUTINES_X86_32_HPP
C:\hotspot-69087d08d473\src\cpu\x86\vm/stubRoutines_x86_32.cpp
#include "precompiled.hpp"
#include "runtime/deoptimization.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/stubRoutines.hpp"
#include "runtime/thread.inline.hpp"
address StubRoutines::x86::_verify_fpu_cntrl_wrd_entry = NULL;
C:\hotspot-69087d08d473\src\cpu\x86\vm/stubRoutines_x86_32.hpp
#ifndef CPU_X86_VM_STUBROUTINES_X86_32_HPP
#define CPU_X86_VM_STUBROUTINES_X86_32_HPP
enum platform_dependent_constants {
code_size1 = 9000, // simply increase if too small (assembler will crash if too small)
code_size2 = 22000 // simply increase if too small (assembler will crash if too small)
};
class x86 {
friend class StubGenerator;
friend class VMStructs;
private:
static address _verify_fpu_cntrl_wrd_entry;
public:
static address verify_fpu_cntrl_wrd_entry() { return _verify_fpu_cntrl_wrd_entry; }
# include "stubRoutines_x86.hpp"
};
static bool returns_to_call_stub(address return_pc) { return return_pc == _call_stub_return_address; }
#endif // CPU_X86_VM_STUBROUTINES_X86_32_HPP
C:\hotspot-69087d08d473\src\cpu\x86\vm/stubRoutines_x86_64.cpp
#include "precompiled.hpp"
#include "runtime/deoptimization.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/stubRoutines.hpp"
#include "runtime/thread.inline.hpp"
address StubRoutines::x86::_get_previous_fp_entry = NULL;
address StubRoutines::x86::_get_previous_sp_entry = NULL;
address StubRoutines::x86::_f2i_fixup = NULL;
address StubRoutines::x86::_f2l_fixup = NULL;
address StubRoutines::x86::_d2i_fixup = NULL;
address StubRoutines::x86::_d2l_fixup = NULL;
address StubRoutines::x86::_float_sign_mask = NULL;
address StubRoutines::x86::_float_sign_flip = NULL;
address StubRoutines::x86::_double_sign_mask = NULL;
address StubRoutines::x86::_double_sign_flip = NULL;
C:\hotspot-69087d08d473\src\cpu\x86\vm/stubRoutines_x86_64.hpp
#ifndef CPU_X86_VM_STUBROUTINES_X86_64_HPP
#define CPU_X86_VM_STUBROUTINES_X86_64_HPP
static bool returns_to_call_stub(address return_pc) { return return_pc == _call_stub_return_address; }
enum platform_dependent_constants {
code_size1 = 19000, // simply increase if too small (assembler will crash if too small)
code_size2 = 24000 // simply increase if too small (assembler will crash if too small)
};
class x86 {
friend class StubGenerator;
private:
static address _get_previous_fp_entry;
static address _get_previous_sp_entry;
static address _f2i_fixup;
static address _f2l_fixup;
static address _d2i_fixup;
static address _d2l_fixup;
static address _float_sign_mask;
static address _float_sign_flip;
static address _double_sign_mask;
static address _double_sign_flip;
public:
static address get_previous_fp_entry()
{
return _get_previous_fp_entry;
}
static address get_previous_sp_entry()
{
return _get_previous_sp_entry;
}
static address f2i_fixup()
{
return _f2i_fixup;
}
static address f2l_fixup()
{
return _f2l_fixup;
}
static address d2i_fixup()
{
return _d2i_fixup;
}
static address d2l_fixup()
{
return _d2l_fixup;
}
static address float_sign_mask()
{
return _float_sign_mask;
}
static address float_sign_flip()
{
return _float_sign_flip;
}
static address double_sign_mask()
{
return _double_sign_mask;
}
static address double_sign_flip()
{
return _double_sign_flip;
}
# include "stubRoutines_x86.hpp"
};
#endif // CPU_X86_VM_STUBROUTINES_X86_64_HPP
C:\hotspot-69087d08d473\src\cpu\x86\vm/templateInterpreterGenerator_x86.hpp
#ifndef CPU_X86_VM_TEMPLATEINTERPRETERGENERATOR_X86_HPP
#define CPU_X86_VM_TEMPLATEINTERPRETERGENERATOR_X86_HPP
protected:
void generate_fixed_frame(bool native_call);
#endif // CPU_X86_VM_TEMPLATEINTERPRETERGENERATOR_X86_HPP
C:\hotspot-69087d08d473\src\cpu\x86\vm/templateInterpreter_x86.cpp
#include "precompiled.hpp"
#include "ci/ciMethod.hpp"
#include "interpreter/interpreter.hpp"
#include "runtime/frame.inline.hpp"
#ifndef CC_INTERP
int AbstractInterpreter::size_activation(int max_stack,
int temps,
int extra_args,
int monitors,
int callee_params,
int callee_locals,
bool is_top_frame) {
int overhead = frame::sender_sp_offset -
frame::interpreter_frame_initial_sp_offset;
int size = overhead +
(callee_locals - callee_params)*Interpreter::stackElementWords +
monitors * frame::interpreter_frame_monitor_size() +
temps* Interpreter::stackElementWords + extra_args;
return size;
}
void AbstractInterpreter::layout_activation(Method* method,
int tempcount,
int popframe_extra_args,
int moncount,
int caller_actual_parameters,
int callee_param_count,
int callee_locals,
frame* caller,
frame* interpreter_frame,
bool is_top_frame,
bool is_bottom_frame) {
int max_locals = method->max_locals() * Interpreter::stackElementWords;
int extra_locals = (method->max_locals() - method->size_of_parameters()) *
Interpreter::stackElementWords;
#ifdef ASSERT
if (!EnableInvokeDynamic) {
assert(caller->unextended_sp() == interpreter_frame->interpreter_frame_sender_sp(), "Frame not properly walkable");
}
assert(caller->sp() == interpreter_frame->sender_sp(), "Frame not properly walkable(2)");
#endif
interpreter_frame->interpreter_frame_set_method(method);
intptr_t* locals = interpreter_frame->sender_sp() + max_locals - 1;
#ifdef ASSERT
if (caller->is_interpreted_frame()) {
assert(locals < caller->fp() + frame::interpreter_frame_initial_sp_offset, "bad placement");
}
#endif
interpreter_frame->interpreter_frame_set_locals(locals);
BasicObjectLock* montop = interpreter_frame->interpreter_frame_monitor_begin();
BasicObjectLock* monbot = montop - moncount;
interpreter_frame->interpreter_frame_set_monitor_end(monbot);
intptr_t* esp = (intptr_t*) monbot -
tempcount*Interpreter::stackElementWords -
popframe_extra_args;
interpreter_frame->interpreter_frame_set_last_sp(esp);
if (extra_locals != 0 &&
interpreter_frame->sender_sp() ==
interpreter_frame->interpreter_frame_sender_sp()) {
interpreter_frame->set_interpreter_frame_sender_sp(caller->sp() +
extra_locals);
}
method->constants()->cache();
}
#endif // CC_INTERP
C:\hotspot-69087d08d473\src\cpu\x86\vm/templateInterpreter_x86.hpp
#ifndef CPU_X86_VM_TEMPLATEINTERPRETER_X86_HPP
#define CPU_X86_VM_TEMPLATEINTERPRETER_X86_HPP
protected:
#ifdef AMD64
const static int InterpreterCodeSize = 256 * 1024;
#else
const static int InterpreterCodeSize = 224 * 1024;
#endif // AMD64
#endif // CPU_X86_VM_TEMPLATEINTERPRETER_X86_HPP
C:\hotspot-69087d08d473\src\cpu\x86\vm/templateInterpreter_x86_32.cpp
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
#include "interpreter/bytecodeHistogram.hpp"
#include "interpreter/interpreter.hpp"
#include "interpreter/interpreterGenerator.hpp"
#include "interpreter/interpreterRuntime.hpp"
#include "interpreter/templateTable.hpp"
#include "oops/arrayOop.hpp"
#include "oops/methodData.hpp"
#include "oops/method.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "prims/jvmtiThreadState.hpp"
#include "runtime/arguments.hpp"
#include "runtime/deoptimization.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
#include "runtime/synchronizer.hpp"
#include "runtime/timer.hpp"
#include "runtime/vframeArray.hpp"
#include "utilities/debug.hpp"
#include "utilities/macros.hpp"
#define __ _masm->
#ifndef CC_INTERP
const int method_offset = frame::interpreter_frame_method_offset * wordSize;
const int bci_offset = frame::interpreter_frame_bcx_offset * wordSize;
const int locals_offset = frame::interpreter_frame_locals_offset * wordSize;
address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
address entry = __ pc();
#ifdef ASSERT
{ Label L;
__ lea(rax, Address(rbp,
frame::interpreter_frame_monitor_block_top_offset * wordSize));
__ cmpptr(rax, rsp); // rax, = maximal rsp for current rbp,
__ jcc(Assembler::aboveEqual, L); // check if frame is complete
__ stop ("interpreter frame not set up");
__ bind(L);
}
#endif // ASSERT
__ restore_bcp();
__ empty_expression_stack();
__ empty_FPU_stack();
__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_StackOverflowError));
return entry;
}
address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler(const char* name) {
address entry = __ pc();
__ empty_expression_stack();
__ empty_FPU_stack();
__ lea(rax, ExternalAddress((address)name));
__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_ArrayIndexOutOfBoundsException), rax, rbx);
return entry;
}
address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
address entry = __ pc();
__ pop(rax);
__ empty_expression_stack();
__ empty_FPU_stack();
__ call_VM(noreg,
CAST_FROM_FN_PTR(address,
InterpreterRuntime::throw_ClassCastException),
rax);
return entry;
}
address TemplateInterpreterGenerator::generate_exception_handler_common(const char* name, const char* message, bool pass_oop) {
assert(!pass_oop || message == NULL, "either oop or message but not both");
address entry = __ pc();
if (pass_oop) {
__ pop(rbx);
}
__ empty_expression_stack();
__ empty_FPU_stack();
__ lea(rax, ExternalAddress((address)name));
if (pass_oop) {
__ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_klass_exception), rax, rbx);
} else {
if (message != NULL) {
__ lea(rbx, ExternalAddress((address)message));
} else {
__ movptr(rbx, NULL_WORD);
}
__ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception), rax, rbx);
}
__ jump(ExternalAddress(Interpreter::throw_exception_entry()));
return entry;
}
address TemplateInterpreterGenerator::generate_continuation_for(TosState state) {
address entry = __ pc();
__ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
__ dispatch_next(state);
return entry;
}
address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) {
address entry = __ pc();
#ifdef COMPILER2
if ((state == ftos && UseSSE < 1) || (state == dtos && UseSSE < 2)) {
for (int i = 1; i < 8; i++) {
__ ffree(i);
}
} else if (UseSSE < 2) {
__ empty_FPU_stack();
}
#endif
if ((state == ftos && UseSSE < 1) || (state == dtos && UseSSE < 2)) {
__ MacroAssembler::verify_FPU(1, "generate_return_entry_for compiled");
} else {
__ MacroAssembler::verify_FPU(0, "generate_return_entry_for compiled");
}
if (state == ftos && UseSSE >= 1) {
__ subptr(rsp, wordSize);
__ movflt(Address(rsp, 0), xmm0);
__ fld_s(Address(rsp, 0));
__ addptr(rsp, wordSize);
} else if (state == dtos && UseSSE >= 2) {
__ subptr(rsp, 2*wordSize);
__ movdbl(Address(rsp, 0), xmm0);
__ fld_d(Address(rsp, 0));
__ addptr(rsp, 2*wordSize);
}
__ MacroAssembler::verify_FPU(state == ftos || state == dtos ? 1 : 0, "generate_return_entry_for in interpreter");
__ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
__ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
__ restore_bcp();
__ restore_locals();
if (state == atos) {
Register mdp = rbx;
Register tmp = rcx;
__ profile_return_type(mdp, rax, tmp);
}
const Register cache = rbx;
const Register index = rcx;
__ get_cache_and_index_at_bcp(cache, index, 1, index_size);
const Register flags = cache;
__ movl(flags, Address(cache, index, Address::times_ptr, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()));
__ andl(flags, ConstantPoolCacheEntry::parameter_size_mask);
__ lea(rsp, Address(rsp, flags, Interpreter::stackElementScale()));
__ dispatch_next(state, step);
return entry;
}
address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state, int step) {
address entry = __ pc();
if (state == ftos && UseSSE > 0) {
__ subptr(rsp, wordSize);
__ movflt(Address(rsp, 0), xmm0);
__ fld_s(Address(rsp, 0));
__ addptr(rsp, wordSize);
} else if (state == dtos && UseSSE >= 2) {
__ subptr(rsp, 2*wordSize);
__ movdbl(Address(rsp, 0), xmm0);
__ fld_d(Address(rsp, 0));
__ addptr(rsp, 2*wordSize);
}
__ MacroAssembler::verify_FPU(state == ftos || state == dtos ? 1 : 0, "generate_deopt_entry_for in interpreter");
__ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
__ restore_bcp();
__ restore_locals();
{ Label L;
const Register thread = rcx;
__ get_thread(thread);
__ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
__ jcc(Assembler::zero, L);
__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_pending_exception));
__ should_not_reach_here();
__ bind(L);
}
__ dispatch_next(state, step);
return entry;
}
int AbstractInterpreter::BasicType_as_index(BasicType type) {
int i = 0;
switch (type) {
case T_BOOLEAN: i = 0; break;
case T_CHAR : i = 1; break;
case T_BYTE : i = 2; break;
case T_SHORT : i = 3; break;
case T_INT : // fall through
case T_LONG : // fall through
case T_VOID : i = 4; break;
case T_FLOAT : i = 5; break; // have to treat float and double separately for SSE
case T_DOUBLE : i = 6; break;
case T_OBJECT : // fall through
case T_ARRAY : i = 7; break;
default : ShouldNotReachHere();
}
assert(0 <= i && i < AbstractInterpreter::number_of_result_handlers, "index out of bounds");
return i;
}
address TemplateInterpreterGenerator::generate_result_handler_for(BasicType type) {
address entry = __ pc();
switch (type) {
case T_BOOLEAN: __ c2bool(rax); break;
case T_CHAR : __ andptr(rax, 0xFFFF); break;
case T_BYTE : __ sign_extend_byte (rax); break;
case T_SHORT : __ sign_extend_short(rax); break;
case T_INT : /* nothing to do */ break;
case T_DOUBLE :
case T_FLOAT :
{ const Register t = InterpreterRuntime::SignatureHandlerGenerator::temp();
__ pop(t); // remove return address first
if (type == T_FLOAT && UseSSE >= 1) {
__ fld_d(Address(rsp, 0));
__ fstp_s(Address(rsp, 0));
__ movflt(xmm0, Address(rsp, 0));
} else if (type == T_DOUBLE && UseSSE >= 2 ) {
__ movdbl(xmm0, Address(rsp, 0));
} else {
__ fld_d(Address(rsp, 0));
}
__ addptr(rsp, 2 * wordSize);
__ push(t); // restore return address
}
break;
case T_OBJECT :
__ movptr(rax, Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize));
__ verify_oop(rax);
break;
default : ShouldNotReachHere();
}
__ ret(0); // return from result handler
return entry;
}
address TemplateInterpreterGenerator::generate_safept_entry_for(TosState state, address runtime_entry) {
address entry = __ pc();
__ push(state);
__ call_VM(noreg, runtime_entry);
__ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos));
return entry;
}
void InterpreterGenerator::generate_counter_incr(Label* overflow, Label* profile_method, Label* profile_method_continue) {
Label done;
if (TieredCompilation) {
int increment = InvocationCounter::count_increment;
int mask = ((1 << Tier0InvokeNotifyFreqLog) - 1) << InvocationCounter::count_shift;
Label no_mdo;
if (ProfileInterpreter) {
__ movptr(rax, Address(rbx, Method::method_data_offset()));
__ testptr(rax, rax);
__ jccb(Assembler::zero, no_mdo);
const Address mdo_invocation_counter(rax, in_bytes(MethodData::invocation_counter_offset()) +
in_bytes(InvocationCounter::counter_offset()));
__ increment_mask_and_jump(mdo_invocation_counter, increment, mask, rcx, false, Assembler::zero, overflow);
__ jmp(done);
}
__ bind(no_mdo);
const Address invocation_counter(rax,
MethodCounters::invocation_counter_offset() +
InvocationCounter::counter_offset());
__ get_method_counters(rbx, rax, done);
__ increment_mask_and_jump(invocation_counter, increment, mask,
rcx, false, Assembler::zero, overflow);
__ bind(done);
} else {
const Address backedge_counter (rax,
MethodCounters::backedge_counter_offset() +
InvocationCounter::counter_offset());
const Address invocation_counter(rax,
MethodCounters::invocation_counter_offset() +
InvocationCounter::counter_offset());
__ get_method_counters(rbx, rax, done);
if (ProfileInterpreter) {
__ incrementl(Address(rax,
MethodCounters::interpreter_invocation_counter_offset()));
}
__ movl(rcx, invocation_counter);
__ incrementl(rcx, InvocationCounter::count_increment);
__ movl(invocation_counter, rcx); // save invocation count
__ movl(rax, backedge_counter); // load backedge counter
__ andl(rax, InvocationCounter::count_mask_value); // mask out the status bits
__ addl(rcx, rax); // add both counters
if (ProfileInterpreter && profile_method != NULL) {
__ cmp32(rcx,
ExternalAddress((address)&InvocationCounter::InterpreterProfileLimit));
__ jcc(Assembler::less, *profile_method_continue);
__ test_method_data_pointer(rax, *profile_method);
}
__ cmp32(rcx,
ExternalAddress((address)&InvocationCounter::InterpreterInvocationLimit));
__ jcc(Assembler::aboveEqual, *overflow);
__ bind(done);
}
}
void InterpreterGenerator::generate_counter_overflow(Label* do_continue) {
__ movptr(rax, (intptr_t)false);
__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow), rax);
__ movptr(rbx, Address(rbp, method_offset)); // restore Method*
__ jmp(*do_continue, relocInfo::none);
}
void InterpreterGenerator::generate_stack_overflow_check(void) {
const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
const int overhead_size = -(frame::interpreter_frame_initial_sp_offset*wordSize) + entry_size;
const int page_size = os::vm_page_size();
Label after_frame_check;
__ cmpl(rdx, (page_size - overhead_size)/Interpreter::stackElementSize);
__ jcc(Assembler::belowEqual, after_frame_check);
Label after_frame_check_pop;
__ push(rsi);
const Register thread = rsi;
__ get_thread(thread);
const Address stack_base(thread, Thread::stack_base_offset());
const Address stack_size(thread, Thread::stack_size_offset());
__ lea(rax, Address(noreg, rdx, Interpreter::stackElementScale(), overhead_size));
#ifdef ASSERT
Label stack_base_okay, stack_size_okay;
__ cmpptr(stack_base, (int32_t)NULL_WORD);
__ jcc(Assembler::notEqual, stack_base_okay);
__ stop("stack base is zero");
__ bind(stack_base_okay);
__ cmpptr(stack_size, 0);
__ jcc(Assembler::notEqual, stack_size_okay);
__ stop("stack size is zero");
__ bind(stack_size_okay);
#endif
__ addptr(rax, stack_base);
__ subptr(rax, stack_size);
const int max_pages = StackShadowPages > (StackRedPages+StackYellowPages) ? StackShadowPages :
(StackRedPages+StackYellowPages);
__ addptr(rax, max_pages * page_size);
__ cmpptr(rsp, rax);
__ jcc(Assembler::above, after_frame_check_pop);
__ pop(rsi); // get saved bcp / (c++ prev state ).
__ pop(rax); // return address must be moved if SP is changed
__ mov(rsp, rsi);
__ push(rax);
assert(StubRoutines::throw_StackOverflowError_entry() != NULL, "stub not yet generated");
__ jump(ExternalAddress(StubRoutines::throw_StackOverflowError_entry()));
__ bind(after_frame_check_pop);
__ pop(rsi);
__ bind(after_frame_check);
}
void InterpreterGenerator::lock_method(void) {
const Address access_flags (rbx, Method::access_flags_offset());
const Address monitor_block_top (rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
#ifdef ASSERT
{ Label L;
__ movl(rax, access_flags);
__ testl(rax, JVM_ACC_SYNCHRONIZED);
__ jcc(Assembler::notZero, L);
__ stop("method doesn't need synchronization");
__ bind(L);
}
#endif // ASSERT
{ Label done;
const int mirror_offset = in_bytes(Klass::java_mirror_offset());
__ movl(rax, access_flags);
__ testl(rax, JVM_ACC_STATIC);
__ movptr(rax, Address(rdi, Interpreter::local_offset_in_bytes(0))); // get receiver (assume this is frequent case)
__ jcc(Assembler::zero, done);
__ movptr(rax, Address(rbx, Method::const_offset()));
__ movptr(rax, Address(rax, ConstMethod::constants_offset()));
__ movptr(rax, Address(rax, ConstantPool::pool_holder_offset_in_bytes()));
__ movptr(rax, Address(rax, mirror_offset));
__ bind(done);
}
__ subptr(rsp, entry_size); // add space for a monitor entry
__ movptr(monitor_block_top, rsp); // set new monitor block top
__ movptr(Address(rsp, BasicObjectLock::obj_offset_in_bytes()), rax); // store object
__ mov(rdx, rsp); // object address
__ lock_object(rdx);
}
void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
__ push(rax); // save return address
__ enter(); // save old & set new rbp,
__ push(rsi); // set sender sp
__ push((int32_t)NULL_WORD); // leave last_sp as null
__ movptr(rsi, Address(rbx,Method::const_offset())); // get ConstMethod*
__ lea(rsi, Address(rsi,ConstMethod::codes_offset())); // get codebase
__ push(rbx); // save Method*
if (ProfileInterpreter) {
Label method_data_continue;
__ movptr(rdx, Address(rbx, in_bytes(Method::method_data_offset())));
__ testptr(rdx, rdx);
__ jcc(Assembler::zero, method_data_continue);
__ addptr(rdx, in_bytes(MethodData::data_offset()));
__ bind(method_data_continue);
__ push(rdx); // set the mdp (method data pointer)
} else {
__ push(0);
}
__ movptr(rdx, Address(rbx, Method::const_offset()));
__ movptr(rdx, Address(rdx, ConstMethod::constants_offset()));
__ movptr(rdx, Address(rdx, ConstantPool::cache_offset_in_bytes()));
__ push(rdx); // set constant pool cache
__ push(rdi); // set locals pointer
if (native_call) {
__ push(0); // no bcp
} else {
__ push(rsi); // set bcp
}
__ push(0); // reserve word for pointer to expression stack bottom
__ movptr(Address(rsp, 0), rsp); // set expression stack bottom
}
address InterpreterGenerator::generate_accessor_entry(void) {
address entry_point = __ pc();
Label xreturn_path;
if (UseFastAccessorMethods) {
Label slow_path;
ExternalAddress state(SafepointSynchronize::address_of_state());
__ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
SafepointSynchronize::_not_synchronized);
__ jcc(Assembler::notEqual, slow_path);
__ movptr(rax, Address(rsp, wordSize));
__ testptr(rax, rax);
__ jcc(Assembler::zero, slow_path);
__ movptr(rdx, Address(rbx, Method::const_offset()));
__ movptr(rdi, Address(rdx, ConstMethod::constants_offset()));
__ movl(rdx, Address(rdx, ConstMethod::codes_offset()));
__ shrl(rdx, 2*BitsPerByte);
__ shll(rdx, exact_log2(in_words(ConstantPoolCacheEntry::size())));
__ movptr(rdi, Address(rdi, ConstantPool::cache_offset_in_bytes()));
assert(in_words(ConstantPoolCacheEntry::size()) == 4, "adjust shift below");
__ movl(rcx,
Address(rdi,
rdx,
Address::times_ptr, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset()));
__ shrl(rcx, 2*BitsPerByte);
__ andl(rcx, 0xFF);
__ cmpl(rcx, Bytecodes::_getfield);
__ jcc(Assembler::notEqual, slow_path);
__ movptr(rcx,
Address(rdi,
rdx,
Address::times_ptr, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f2_offset()));
__ movl(rdx,
Address(rdi,
rdx,
Address::times_ptr, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()));
Label notByte, notBool, notShort, notChar;
const Address field_address (rax, rcx, Address::times_1);
__ shrl(rdx, ConstantPoolCacheEntry::tos_state_shift);
ConstantPoolCacheEntry::verify_tos_state_shift();
__ cmpl(rdx, btos);
__ jcc(Assembler::notEqual, notByte);
__ load_signed_byte(rax, field_address);
__ jmp(xreturn_path);
__ bind(notByte);
__ cmpl(rdx, ztos);
__ jcc(Assembler::notEqual, notBool);
__ load_signed_byte(rax, field_address);
__ jmp(xreturn_path);
__ bind(notBool);
__ cmpl(rdx, stos);
__ jcc(Assembler::notEqual, notShort);
__ load_signed_short(rax, field_address);
__ jmp(xreturn_path);
__ bind(notShort);
__ cmpl(rdx, ctos);
__ jcc(Assembler::notEqual, notChar);
__ load_unsigned_short(rax, field_address);
__ jmp(xreturn_path);
__ bind(notChar);
#ifdef ASSERT
Label okay;
__ cmpl(rdx, atos);
__ jcc(Assembler::equal, okay);
__ cmpl(rdx, itos);
__ jcc(Assembler::equal, okay);
__ stop("what type is this?");
__ bind(okay);
#endif // ASSERT
__ movptr(rax, field_address);
__ bind(xreturn_path);
__ pop(rdi); // get return address
__ mov(rsp, rsi); // set sp to sender sp
__ jmp(rdi);
__ bind(slow_path);
(void) generate_normal_entry(false);
return entry_point;
}
return NULL;
}
address InterpreterGenerator::generate_Reference_get_entry(void) {
#if INCLUDE_ALL_GCS
address entry = __ pc();
const int referent_offset = java_lang_ref_Reference::referent_offset;
guarantee(referent_offset > 0, "referent offset not initialized");
if (UseG1GC) {
Label slow_path;
__ movptr(rax, Address(rsp, wordSize));
__ testptr(rax, rax);
__ jcc(Assembler::zero, slow_path);
__ push(rsi);
const Address field_address(rax, referent_offset);
__ movptr(rax, field_address);
__ get_thread(rcx);
__ g1_write_barrier_pre(noreg /* obj */,
rax /* pre_val */,
rcx /* thread */,
rbx /* tmp */,
true /* tosca_save */,
true /* expand_call */);
__ pop(rsi); // get sender sp
__ pop(rdi); // get return address
__ mov(rsp, rsi); // set sp to sender sp
__ jmp(rdi);
__ bind(slow_path);
(void) generate_normal_entry(false);
return entry;
}
#endif // INCLUDE_ALL_GCS
return generate_accessor_entry();
}
address InterpreterGenerator::generate_CRC32_update_entry() {
if (UseCRC32Intrinsics) {
address entry = __ pc();
Label slow_path;
ExternalAddress state(SafepointSynchronize::address_of_state());
__ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
SafepointSynchronize::_not_synchronized);
__ jcc(Assembler::notEqual, slow_path);
const Register crc = rax; // crc
const Register val = rdx; // source java byte value
const Register tbl = rdi; // scratch
__ movl(val, Address(rsp, wordSize)); // byte value
__ movl(crc, Address(rsp, 2*wordSize)); // Initial CRC
__ lea(tbl, ExternalAddress(StubRoutines::crc_table_addr()));
__ notl(crc); // ~crc
__ update_byte_crc32(crc, val, tbl);
__ notl(crc); // ~crc
__ pop(rdi); // get return address
__ mov(rsp, rsi); // set sp to sender sp
__ jmp(rdi);
__ bind(slow_path);
(void) generate_native_entry(false);
return entry;
}
return generate_native_entry(false);
}
address InterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractInterpreter::MethodKind kind) {
if (UseCRC32Intrinsics) {
address entry = __ pc();
Label slow_path;
ExternalAddress state(SafepointSynchronize::address_of_state());
__ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
SafepointSynchronize::_not_synchronized);
__ jcc(Assembler::notEqual, slow_path);
const Register crc = rax; // crc
const Register buf = rdx; // source java byte array address
const Register len = rdi; // length
__ movl(len, Address(rsp, wordSize)); // Length
if (kind == Interpreter::java_util_zip_CRC32_updateByteBuffer) {
__ movptr(buf, Address(rsp, 3*wordSize)); // long buf
__ addptr(buf, Address(rsp, 2*wordSize)); // + offset
__ movl(crc, Address(rsp, 5*wordSize)); // Initial CRC
} else {
__ movptr(buf, Address(rsp, 3*wordSize)); // byte[] array
__ addptr(buf, arrayOopDesc::base_offset_in_bytes(T_BYTE)); // + header size
__ addptr(buf, Address(rsp, 2*wordSize)); // + offset
__ movl(crc, Address(rsp, 4*wordSize)); // Initial CRC
}
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, StubRoutines::updateBytesCRC32()), crc, buf, len);
__ pop(rdi); // get return address
__ mov(rsp, rsi); // set sp to sender sp
__ jmp(rdi);
__ bind(slow_path);
(void) generate_native_entry(false);
return entry;
}
return generate_native_entry(false);
}
address InterpreterGenerator::generate_native_entry(bool synchronized) {
bool inc_counter = UseCompiler || CountCompiledCalls;
address entry_point = __ pc();
const Address constMethod (rbx, Method::const_offset());
const Address access_flags (rbx, Method::access_flags_offset());
const Address size_of_parameters(rcx, ConstMethod::size_of_parameters_offset());
__ movptr(rcx, constMethod);
__ load_unsigned_short(rcx, size_of_parameters);
__ pop(rax); // get return address
__ lea(rdi, Address(rsp, rcx, Interpreter::stackElementScale(), -wordSize));
__ push((int32_t)NULL_WORD);
__ push((int32_t)NULL_WORD);
generate_fixed_frame(true);
#ifdef ASSERT
__ movl(rax, access_flags);
{
Label L;
__ testl(rax, JVM_ACC_NATIVE);
__ jcc(Assembler::notZero, L);
__ stop("tried to execute non-native method as native");
__ bind(L);
}
{ Label L;
__ testl(rax, JVM_ACC_ABSTRACT);
__ jcc(Assembler::zero, L);
__ stop("tried to execute abstract method in interpreter");
__ bind(L);
}
#endif
__ get_thread(rax);
const Address do_not_unlock_if_synchronized(rax,
in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
__ movbool(do_not_unlock_if_synchronized, true);
Label invocation_counter_overflow;
if (inc_counter) {
generate_counter_incr(&invocation_counter_overflow, NULL, NULL);
}
Label continue_after_compile;
__ bind(continue_after_compile);
bang_stack_shadow_pages(true);
__ get_thread(rax);
__ movbool(do_not_unlock_if_synchronized, false);
if (synchronized) {
lock_method();
} else {
#ifdef ASSERT
{ Label L;
__ movl(rax, access_flags);
__ testl(rax, JVM_ACC_SYNCHRONIZED);
__ jcc(Assembler::zero, L);
__ stop("method needs synchronization");
__ bind(L);
}
#endif
}
#ifdef ASSERT
{ Label L;
const Address monitor_block_top (rbp,
frame::interpreter_frame_monitor_block_top_offset * wordSize);
__ movptr(rax, monitor_block_top);
__ cmpptr(rax, rsp);
__ jcc(Assembler::equal, L);
__ stop("broken stack frame setup in interpreter");
__ bind(L);
}
#endif
__ notify_method_entry();
const Register method = rbx;
const Register thread = rdi;
const Register t = rcx;
__ get_method(method);
__ movptr(t, Address(method, Method::const_offset()));
__ load_unsigned_short(t, Address(t, ConstMethod::size_of_parameters_offset()));
__ shlptr(t, Interpreter::logStackElementSize);
__ addptr(t, 2*wordSize); // allocate two more slots for JNIEnv and possible mirror
__ subptr(rsp, t);
__ andptr(rsp, -(StackAlignmentInBytes)); // gcc needs 16 byte aligned stacks to do XMM intrinsics
{ Label L;
__ movptr(t, Address(method, Method::signature_handler_offset()));
__ testptr(t, t);
__ jcc(Assembler::notZero, L);
__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), method);
__ get_method(method);
__ movptr(t, Address(method, Method::signature_handler_offset()));
__ bind(L);
}
assert(InterpreterRuntime::SignatureHandlerGenerator::from() == rdi, "adjust this code");
assert(InterpreterRuntime::SignatureHandlerGenerator::to () == rsp, "adjust this code");
assert(InterpreterRuntime::SignatureHandlerGenerator::temp() == t , "adjust this code");
__ call(t);
__ get_method(method); // slow path call blows RBX on DevStudio 5.0
__ movptr(Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize), rax);
{ Label L;
const int mirror_offset = in_bytes(Klass::java_mirror_offset());
__ movl(t, Address(method, Method::access_flags_offset()));
__ testl(t, JVM_ACC_STATIC);
__ jcc(Assembler::zero, L);
__ movptr(t, Address(method, Method:: const_offset()));
__ movptr(t, Address(t, ConstMethod::constants_offset()));
__ movptr(t, Address(t, ConstantPool::pool_holder_offset_in_bytes()));
__ movptr(t, Address(t, mirror_offset));
__ movptr(Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize), t);
__ lea(t, Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize));
__ movptr(Address(rsp, wordSize), t);
__ bind(L);
}
{ Label L;
__ movptr(rax, Address(method, Method::native_function_offset()));
ExternalAddress unsatisfied(SharedRuntime::native_method_throw_unsatisfied_link_error_entry());
__ cmpptr(rax, unsatisfied.addr());
__ jcc(Assembler::notEqual, L);
__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), method);
__ get_method(method);
__ movptr(rax, Address(method, Method::native_function_offset()));
__ bind(L);
}
__ get_thread(thread);
__ lea(t, Address(thread, JavaThread::jni_environment_offset()));
__ movptr(Address(rsp, 0), t);
__ set_last_Java_frame(thread, noreg, rbp, __ pc());
#ifdef ASSERT
{ Label L;
__ movl(t, Address(thread, JavaThread::thread_state_offset()));
__ cmpl(t, _thread_in_Java);
__ jcc(Assembler::equal, L);
__ stop("Wrong thread state in native stub");
__ bind(L);
}
#endif
__ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_native);
__ call(rax);
__ restore_cpu_control_state_after_jni();
{ Label L;
Label push_double;
ExternalAddress float_handler(AbstractInterpreter::result_handler(T_FLOAT));
ExternalAddress double_handler(AbstractInterpreter::result_handler(T_DOUBLE));
__ cmpptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset + 1)*wordSize),
float_handler.addr());
__ jcc(Assembler::equal, push_double);
__ cmpptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset + 1)*wordSize),
double_handler.addr());
__ jcc(Assembler::notEqual, L);
__ bind(push_double);
__ push(dtos);
__ bind(L);
}
__ push(ltos);
__ get_thread(thread);
__ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
if(os::is_MP()) {
if (UseMembar) {
__ membar(Assembler::Membar_mask_bits(
Assembler::LoadLoad | Assembler::LoadStore |
Assembler::StoreLoad | Assembler::StoreStore));
} else {
__ serialize_memory(thread, rcx);
}
}
if (AlwaysRestoreFPU) {
__ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
}
{ Label Continue;
__ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
SafepointSynchronize::_not_synchronized);
Label L;
__ jcc(Assembler::notEqual, L);
__ cmpl(Address(thread, JavaThread::suspend_flags_offset()), 0);
__ jcc(Assembler::equal, Continue);
__ bind(L);
__ push(thread);
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address,
JavaThread::check_special_condition_for_native_trans)));
__ increment(rsp, wordSize);
__ get_thread(thread);
__ bind(Continue);
}
__ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_Java);
__ reset_last_Java_frame(thread, true);
__ movptr(t, Address(thread, JavaThread::active_handles_offset()));
__ movl(Address(t, JNIHandleBlock::top_offset_in_bytes()), NULL_WORD);
{
Label no_oop;
ExternalAddress handler(AbstractInterpreter::result_handler(T_OBJECT));
__ cmpptr(Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize),
handler.addr());
__ jcc(Assembler::notEqual, no_oop);
__ cmpptr(Address(rsp, 0), (int32_t)NULL_WORD);
__ pop(ltos);
__ resolve_jobject(rax /* value */,
thread /* thread */,
t /* tmp */);
__ movptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset)*wordSize), rax);
__ push(ltos);
__ bind(no_oop);
}
{
Label no_reguard;
__ cmpl(Address(thread, JavaThread::stack_guard_state_offset()), JavaThread::stack_guard_yellow_disabled);
__ jcc(Assembler::notEqual, no_reguard);
__ pusha();
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
__ popa();
__ bind(no_reguard);
}
__ get_method(method); // method is junk from thread_in_native to now.
__ movptr(rsi, Address(method,Method::const_offset())); // get ConstMethod*
__ lea(rsi, Address(rsi,ConstMethod::codes_offset())); // get codebase
{ Label L;
__ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
__ jcc(Assembler::zero, L);
__ MacroAssembler::call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_pending_exception));
__ should_not_reach_here();
__ bind(L);
}
{ Label L;
__ movl(t, Address(method, Method::access_flags_offset()));
__ testl(t, JVM_ACC_SYNCHRONIZED);
__ jcc(Assembler::zero, L);
{ Label unlock;
const Address monitor(rbp, frame::interpreter_frame_initial_sp_offset * wordSize - (int)sizeof(BasicObjectLock));
__ lea(rdx, monitor); // address of first monitor
__ movptr(t, Address(rdx, BasicObjectLock::obj_offset_in_bytes()));
__ testptr(t, t);
__ jcc(Assembler::notZero, unlock);
__ MacroAssembler::call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
__ should_not_reach_here();
__ bind(unlock);
__ unlock_object(rdx);
}
__ bind(L);
}
__ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI);
__ pop(ltos);
__ movptr(t, Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize));
__ call(t);
__ movptr(t, Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize)); // get sender sp
__ leave(); // remove frame anchor
__ pop(rdi); // get return address
__ mov(rsp, t); // set sp to sender sp
__ jmp(rdi);
if (inc_counter) {
__ bind(invocation_counter_overflow);
generate_counter_overflow(&continue_after_compile);
}
return entry_point;
}
address InterpreterGenerator::generate_normal_entry(bool synchronized) {
bool inc_counter = UseCompiler || CountCompiledCalls;
address entry_point = __ pc();
const Address constMethod (rbx, Method::const_offset());
const Address access_flags (rbx, Method::access_flags_offset());
const Address size_of_parameters(rdx, ConstMethod::size_of_parameters_offset());
const Address size_of_locals (rdx, ConstMethod::size_of_locals_offset());
__ movptr(rdx, constMethod);
__ load_unsigned_short(rcx, size_of_parameters);
__ load_unsigned_short(rdx, size_of_locals); // get size of locals in words
__ subl(rdx, rcx); // rdx = no. of additional locals
generate_stack_overflow_check();
__ pop(rax);
__ lea(rdi, Address(rsp, rcx, Interpreter::stackElementScale(), -wordSize));
{
Label exit, loop;
__ testl(rdx, rdx);
__ jcc(Assembler::lessEqual, exit); // do nothing if rdx <= 0
__ bind(loop);
__ push((int32_t)NULL_WORD); // initialize local variables
__ decrement(rdx); // until everything initialized
__ jcc(Assembler::greater, loop);
__ bind(exit);
}
generate_fixed_frame(false);
#ifdef ASSERT
__ movl(rax, access_flags);
{
Label L;
__ testl(rax, JVM_ACC_NATIVE);
__ jcc(Assembler::zero, L);
__ stop("tried to execute native method as non-native");
__ bind(L);
}
{ Label L;
__ testl(rax, JVM_ACC_ABSTRACT);
__ jcc(Assembler::zero, L);
__ stop("tried to execute abstract method in interpreter");
__ bind(L);
}
#endif
__ get_thread(rax);
const Address do_not_unlock_if_synchronized(rax,
in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
__ movbool(do_not_unlock_if_synchronized, true);
__ profile_parameters_type(rax, rcx, rdx);
Label invocation_counter_overflow;
Label profile_method;
Label profile_method_continue;
if (inc_counter) {
generate_counter_incr(&invocation_counter_overflow, &profile_method, &profile_method_continue);
if (ProfileInterpreter) {
__ bind(profile_method_continue);
}
}
Label continue_after_compile;
__ bind(continue_after_compile);
bang_stack_shadow_pages(false);
__ get_thread(rax);
__ movbool(do_not_unlock_if_synchronized, false);
if (synchronized) {
lock_method();
} else {
#ifdef ASSERT
{ Label L;
__ movl(rax, access_flags);
__ testl(rax, JVM_ACC_SYNCHRONIZED);
__ jcc(Assembler::zero, L);
__ stop("method needs synchronization");
__ bind(L);
}
#endif
}
#ifdef ASSERT
{ Label L;
const Address monitor_block_top (rbp,
frame::interpreter_frame_monitor_block_top_offset * wordSize);
__ movptr(rax, monitor_block_top);
__ cmpptr(rax, rsp);
__ jcc(Assembler::equal, L);
__ stop("broken stack frame setup in interpreter");
__ bind(L);
}
#endif
__ notify_method_entry();
__ dispatch_next(vtos);
if (inc_counter) {
if (ProfileInterpreter) {
__ bind(profile_method);
__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
__ set_method_data_pointer_for_bcp();
__ get_method(rbx);
__ jmp(profile_method_continue);
}
__ bind(invocation_counter_overflow);
generate_counter_overflow(&continue_after_compile);
}
return entry_point;
}
address AbstractInterpreterGenerator::generate_method_entry(AbstractInterpreter::MethodKind kind) {
bool synchronized = false;
address entry_point = NULL;
InterpreterGenerator* ig_this = (InterpreterGenerator*)this;
switch (kind) {
case Interpreter::zerolocals : break;
case Interpreter::zerolocals_synchronized: synchronized = true; break;
case Interpreter::native : entry_point = ig_this->generate_native_entry(false); break;
case Interpreter::native_synchronized : entry_point = ig_this->generate_native_entry(true); break;
case Interpreter::empty : entry_point = ig_this->generate_empty_entry(); break;
case Interpreter::accessor : entry_point = ig_this->generate_accessor_entry(); break;
case Interpreter::abstract : entry_point = ig_this->generate_abstract_entry(); break;
case Interpreter::java_lang_math_sin : // fall thru
case Interpreter::java_lang_math_cos : // fall thru
case Interpreter::java_lang_math_tan : // fall thru
case Interpreter::java_lang_math_abs : // fall thru
case Interpreter::java_lang_math_log : // fall thru
case Interpreter::java_lang_math_log10 : // fall thru
case Interpreter::java_lang_math_sqrt : // fall thru
case Interpreter::java_lang_math_pow : // fall thru
case Interpreter::java_lang_math_exp : entry_point = ig_this->generate_math_entry(kind); break;
case Interpreter::java_lang_ref_reference_get
: entry_point = ig_this->generate_Reference_get_entry(); break;
case Interpreter::java_util_zip_CRC32_update
: entry_point = ig_this->generate_CRC32_update_entry(); break;
case Interpreter::java_util_zip_CRC32_updateBytes
: // fall thru
case Interpreter::java_util_zip_CRC32_updateByteBuffer
: entry_point = ig_this->generate_CRC32_updateBytes_entry(kind); break;
default:
fatal(err_msg("unexpected method kind: %d", kind));
break;
}
if (entry_point) return entry_point;
return ig_this->generate_normal_entry(synchronized);
}
bool AbstractInterpreter::can_be_compiled(methodHandle m) {
switch (method_kind(m)) {
case Interpreter::java_lang_math_sin : // fall thru
case Interpreter::java_lang_math_cos : // fall thru
case Interpreter::java_lang_math_tan : // fall thru
case Interpreter::java_lang_math_abs : // fall thru
case Interpreter::java_lang_math_log : // fall thru
case Interpreter::java_lang_math_log10 : // fall thru
case Interpreter::java_lang_math_sqrt : // fall thru
case Interpreter::java_lang_math_pow : // fall thru
case Interpreter::java_lang_math_exp :
return false;
default:
return true;
}
}
int AbstractInterpreter::size_top_interpreter_activation(Method* method) {
const int stub_code = 4; // see generate_call_stub
int monitor_size = method->is_synchronized() ?
1*frame::interpreter_frame_monitor_size() : 0;
const int overhead_size = -frame::interpreter_frame_initial_sp_offset;
const int method_stack = (method->max_locals() + method->max_stack()) *
Interpreter::stackElementWords;
return overhead_size + method_stack + stub_code;
}
void TemplateInterpreterGenerator::generate_throw_exception() {
Interpreter::_rethrow_exception_entry = __ pc();
const Register thread = rcx;
__ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
__ restore_bcp(); // rsi points to call/send
__ restore_locals();
Interpreter::_throw_exception_entry = __ pc();
__ verify_oop(rax);
__ empty_expression_stack();
__ empty_FPU_stack();
__ call_VM(rdx, CAST_FROM_FN_PTR(address, InterpreterRuntime::exception_handler_for_exception), rax);
__ push_ptr(rdx); // push exception which is now the only value on the stack
__ jmp(rax); // jump to exception handler (may be _remove_activation_entry!)
Interpreter::_remove_activation_preserving_args_entry = __ pc();
__ empty_expression_stack();
__ empty_FPU_stack();
__ get_thread(thread);
__ movl(rdx, Address(thread, JavaThread::popframe_condition_offset()));
__ orl(rdx, JavaThread::popframe_processing_bit);
__ movl(Address(thread, JavaThread::popframe_condition_offset()), rdx);
{
Label caller_not_deoptimized;
__ movptr(rdx, Address(rbp, frame::return_addr_offset * wordSize));
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::interpreter_contains), rdx);
__ testl(rax, rax);
__ jcc(Assembler::notZero, caller_not_deoptimized);
__ get_method(rax);
__ movptr(rax, Address(rax, Method::const_offset()));
__ load_unsigned_short(rax, Address(rax, ConstMethod::size_of_parameters_offset()));
__ shlptr(rax, Interpreter::logStackElementSize);
__ restore_locals();
__ subptr(rdi, rax);
__ addptr(rdi, wordSize);
__ get_thread(thread);
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::popframe_preserve_args), thread, rax, rdi);
__ remove_activation(vtos, rdx,
__ get_thread(thread);
__ movl(Address(thread, JavaThread::popframe_condition_offset()), JavaThread::popframe_force_deopt_reexecution_bit);
__ jmp(rdx);
__ bind(caller_not_deoptimized);
}
__ remove_activation(vtos, rdx,
__ mov(rax, rsp);
__ movptr(rbx, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
__ get_thread(thread);
__ set_last_Java_frame(thread, noreg, rbp, __ pc());
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), thread, rax, rbx);
__ get_thread(thread);
__ reset_last_Java_frame(thread, true);
__ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
__ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
__ restore_bcp();
__ restore_locals();
if (ProfileInterpreter) {
__ set_method_data_pointer_for_bcp();
}
__ get_thread(thread);
__ movl(Address(thread, JavaThread::popframe_condition_offset()), JavaThread::popframe_inactive);
#if INCLUDE_JVMTI
if (EnableInvokeDynamic) {
Label L_done;
const Register local0 = rdi;
__ cmpb(Address(rsi, 0), Bytecodes::_invokestatic);
__ jcc(Assembler::notEqual, L_done);
__ get_method(rdx);
__ movptr(rax, Address(local0, 0));
__ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::member_name_arg_or_null), rax, rdx, rsi);
__ testptr(rax, rax);
__ jcc(Assembler::zero, L_done);
__ movptr(Address(rbx, 0), rax);
__ bind(L_done);
}
#endif // INCLUDE_JVMTI
__ dispatch_next(vtos);
Interpreter::_remove_activation_entry = __ pc();
__ pop_ptr(rax);
__ get_thread(thread);
__ movptr(Address(thread, JavaThread::vm_result_offset()), rax);
__ remove_activation(vtos, rdx, false, true, false);
__ get_thread(thread);
__ get_vm_result(rax, thread);
__ push(rax); // save exception
__ push(rdx); // save return address
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, rdx);
__ mov(rbx, rax); // save exception handler
__ pop(rdx); // restore return address
__ pop(rax); // restore exception
__ jmp(rbx); // jump to exception handler of caller
}
address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
address entry = __ pc();
const Register thread = rcx;
__ restore_bcp();
__ restore_locals();
__ empty_expression_stack();
__ empty_FPU_stack();
__ load_earlyret_value(state);
__ get_thread(thread);
__ movptr(rcx, Address(thread, JavaThread::jvmti_thread_state_offset()));
const Address cond_addr(rcx, JvmtiThreadState::earlyret_state_offset());
__ movl(cond_addr, JvmtiThreadState::earlyret_inactive);
__ remove_activation(state, rsi,
false, /* throw_monitor_exception */
false, /* install_monitor_exception */
true); /* notify_jvmdi */
__ jmp(rsi);
return entry;
} // end of ForceEarlyReturn support
void TemplateInterpreterGenerator::set_vtos_entry_points (Template* t, address& bep, address& cep, address& sep, address& aep, address& iep, address& lep, address& fep, address& dep, address& vep) {
assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
Label L;
fep = __ pc(); __ push(ftos); __ jmp(L);
dep = __ pc(); __ push(dtos); __ jmp(L);
lep = __ pc(); __ push(ltos); __ jmp(L);
aep = __ pc(); __ push(atos); __ jmp(L);
bep = cep = sep = // fall through
iep = __ pc(); __ push(itos); // fall through
vep = __ pc(); __ bind(L); // fall through
generate_and_dispatch(t);
}
InterpreterGenerator::InterpreterGenerator(StubQueue* code)
: TemplateInterpreterGenerator(code) {
generate_all(); // down here so it can be "virtual"
}
#ifndef PRODUCT
address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
address entry = __ pc();
__ pop(rcx); // pop return address so expression stack is 'pure'
__ push(state); // save tosca
__ call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::trace_bytecode), rcx, rax, rdx);
__ mov(rcx, rax); // make sure return address is not destroyed by pop(state)
__ pop(state); // restore tosca
__ jmp(rcx);
return entry;
}
void TemplateInterpreterGenerator::count_bytecode() {
__ incrementl(ExternalAddress((address) &BytecodeCounter::_counter_value));
}
void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
__ incrementl(ExternalAddress((address) &BytecodeHistogram::_counters[t->bytecode()]));
}
void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
__ mov32(ExternalAddress((address) &BytecodePairHistogram::_index), rbx);
__ shrl(rbx, BytecodePairHistogram::log2_number_of_codes);
__ orl(rbx, ((int)t->bytecode()) << BytecodePairHistogram::log2_number_of_codes);
ExternalAddress table((address) BytecodePairHistogram::_counters);
Address index(noreg, rbx, Address::times_4);
__ incrementl(ArrayAddress(table, index));
}
void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
assert(Interpreter::trace_code(t->tos_in()) != NULL,
"entry must have been generated");
__ call(RuntimeAddress(Interpreter::trace_code(t->tos_in())));
}
void TemplateInterpreterGenerator::stop_interpreter_at() {
Label L;
__ cmp32(ExternalAddress((address) &BytecodeCounter::_counter_value),
StopInterpreterAt);
__ jcc(Assembler::notEqual, L);
__ int3();
__ bind(L);
}
#endif // !PRODUCT
#endif // CC_INTERP
C:\hotspot-69087d08d473\src\cpu\x86\vm/templateInterpreter_x86_64.cpp
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
#include "interpreter/bytecodeHistogram.hpp"
#include "interpreter/interpreter.hpp"
#include "interpreter/interpreterGenerator.hpp"
#include "interpreter/interpreterRuntime.hpp"
#include "interpreter/templateTable.hpp"
#include "oops/arrayOop.hpp"
#include "oops/methodData.hpp"
#include "oops/method.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "prims/jvmtiThreadState.hpp"
#include "runtime/arguments.hpp"
#include "runtime/deoptimization.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
#include "runtime/synchronizer.hpp"
#include "runtime/timer.hpp"
#include "runtime/vframeArray.hpp"
#include "utilities/debug.hpp"
#include "utilities/macros.hpp"
#define __ _masm->
#ifndef CC_INTERP
const int method_offset = frame::interpreter_frame_method_offset * wordSize;
const int bci_offset = frame::interpreter_frame_bcx_offset * wordSize;
const int locals_offset = frame::interpreter_frame_locals_offset * wordSize;
address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
address entry = __ pc();
#ifdef ASSERT
{
Label L;
__ lea(rax, Address(rbp,
frame::interpreter_frame_monitor_block_top_offset *
wordSize));
__ cmpptr(rax, rsp); // rax = maximal rsp for current rbp (stack
__ jcc(Assembler::aboveEqual, L); // check if frame is complete
__ stop ("interpreter frame not set up");
__ bind(L);
}
#endif // ASSERT
__ restore_bcp();
__ empty_expression_stack();
__ call_VM(noreg,
CAST_FROM_FN_PTR(address,
InterpreterRuntime::throw_StackOverflowError));
return entry;
}
address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler(
const char* name) {
address entry = __ pc();
__ empty_expression_stack();
__ lea(c_rarg1, ExternalAddress((address)name));
__ call_VM(noreg,
CAST_FROM_FN_PTR(address,
InterpreterRuntime::
throw_ArrayIndexOutOfBoundsException),
c_rarg1, rbx);
return entry;
}
address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
address entry = __ pc();
__ pop(c_rarg1);
__ empty_expression_stack();
__ call_VM(noreg,
CAST_FROM_FN_PTR(address,
InterpreterRuntime::
throw_ClassCastException),
c_rarg1);
return entry;
}
address TemplateInterpreterGenerator::generate_exception_handler_common(
const char* name, const char* message, bool pass_oop) {
assert(!pass_oop || message == NULL, "either oop or message but not both");
address entry = __ pc();
if (pass_oop) {
__ pop(c_rarg2);
}
__ empty_expression_stack();
__ lea(c_rarg1, ExternalAddress((address)name));
if (pass_oop) {
__ call_VM(rax, CAST_FROM_FN_PTR(address,
InterpreterRuntime::
create_klass_exception),
c_rarg1, c_rarg2);
} else {
if (message != NULL) {
__ lea(c_rarg2, ExternalAddress((address)message));
} else {
__ movptr(c_rarg2, NULL_WORD);
}
__ call_VM(rax,
CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception),
c_rarg1, c_rarg2);
}
__ jump(ExternalAddress(Interpreter::throw_exception_entry()));
return entry;
}
address TemplateInterpreterGenerator::generate_continuation_for(TosState state) {
address entry = __ pc();
__ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
__ dispatch_next(state);
return entry;
}
address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) {
address entry = __ pc();
__ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
__ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
__ restore_bcp();
__ restore_locals();
if (state == atos) {
Register mdp = rbx;
Register tmp = rcx;
__ profile_return_type(mdp, rax, tmp);
}
const Register cache = rbx;
const Register index = rcx;
__ get_cache_and_index_at_bcp(cache, index, 1, index_size);
const Register flags = cache;
__ movl(flags, Address(cache, index, Address::times_ptr, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()));
__ andl(flags, ConstantPoolCacheEntry::parameter_size_mask);
__ lea(rsp, Address(rsp, flags, Interpreter::stackElementScale()));
__ dispatch_next(state, step);
return entry;
}
address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state,
int step) {
address entry = __ pc();
__ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
__ restore_bcp();
__ restore_locals();
{
Label L;
__ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
__ jcc(Assembler::zero, L);
__ call_VM(noreg,
CAST_FROM_FN_PTR(address,
InterpreterRuntime::throw_pending_exception));
__ should_not_reach_here();
__ bind(L);
}
__ dispatch_next(state, step);
return entry;
}
int AbstractInterpreter::BasicType_as_index(BasicType type) {
int i = 0;
switch (type) {
case T_BOOLEAN: i = 0; break;
case T_CHAR : i = 1; break;
case T_BYTE : i = 2; break;
case T_SHORT : i = 3; break;
case T_INT : i = 4; break;
case T_LONG : i = 5; break;
case T_VOID : i = 6; break;
case T_FLOAT : i = 7; break;
case T_DOUBLE : i = 8; break;
case T_OBJECT : i = 9; break;
case T_ARRAY : i = 9; break;
default : ShouldNotReachHere();
}
assert(0 <= i && i < AbstractInterpreter::number_of_result_handlers,
"index out of bounds");
return i;
}
address TemplateInterpreterGenerator::generate_result_handler_for(
BasicType type) {
address entry = __ pc();
switch (type) {
case T_BOOLEAN: __ c2bool(rax); break;
case T_CHAR : __ movzwl(rax, rax); break;
case T_BYTE : __ sign_extend_byte(rax); break;
case T_SHORT : __ sign_extend_short(rax); break;
case T_INT : /* nothing to do */ break;
case T_LONG : /* nothing to do */ break;
case T_VOID : /* nothing to do */ break;
case T_FLOAT : /* nothing to do */ break;
case T_DOUBLE : /* nothing to do */ break;
case T_OBJECT :
__ movptr(rax, Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize));
__ verify_oop(rax);
break;
default : ShouldNotReachHere();
}
__ ret(0); // return from result handler
return entry;
}
address TemplateInterpreterGenerator::generate_safept_entry_for(
TosState state,
address runtime_entry) {
address entry = __ pc();
__ push(state);
__ call_VM(noreg, runtime_entry);
__ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos));
return entry;
}
void InterpreterGenerator::generate_counter_incr(
Label* overflow,
Label* profile_method,
Label* profile_method_continue) {
Label done;
if (TieredCompilation) {
int increment = InvocationCounter::count_increment;
int mask = ((1 << Tier0InvokeNotifyFreqLog) - 1) << InvocationCounter::count_shift;
Label no_mdo;
if (ProfileInterpreter) {
__ movptr(rax, Address(rbx, Method::method_data_offset()));
__ testptr(rax, rax);
__ jccb(Assembler::zero, no_mdo);
const Address mdo_invocation_counter(rax, in_bytes(MethodData::invocation_counter_offset()) +
in_bytes(InvocationCounter::counter_offset()));
__ increment_mask_and_jump(mdo_invocation_counter, increment, mask, rcx, false, Assembler::zero, overflow);
__ jmp(done);
}
__ bind(no_mdo);
const Address invocation_counter(rax,
MethodCounters::invocation_counter_offset() +
InvocationCounter::counter_offset());
__ get_method_counters(rbx, rax, done);
__ increment_mask_and_jump(invocation_counter, increment, mask, rcx,
false, Assembler::zero, overflow);
__ bind(done);
} else {
const Address backedge_counter(rax,
MethodCounters::backedge_counter_offset() +
InvocationCounter::counter_offset());
const Address invocation_counter(rax,
MethodCounters::invocation_counter_offset() +
InvocationCounter::counter_offset());
__ get_method_counters(rbx, rax, done);
if (ProfileInterpreter) {
__ incrementl(Address(rax,
MethodCounters::interpreter_invocation_counter_offset()));
}
__ movl(rcx, invocation_counter);
__ incrementl(rcx, InvocationCounter::count_increment);
__ movl(invocation_counter, rcx); // save invocation count
__ movl(rax, backedge_counter); // load backedge counter
__ andl(rax, InvocationCounter::count_mask_value); // mask out the status bits
__ addl(rcx, rax); // add both counters
if (ProfileInterpreter && profile_method != NULL) {
__ cmp32(rcx, ExternalAddress((address)&InvocationCounter::InterpreterProfileLimit));
__ jcc(Assembler::less, *profile_method_continue);
__ test_method_data_pointer(rax, *profile_method);
}
__ cmp32(rcx, ExternalAddress((address)&InvocationCounter::InterpreterInvocationLimit));
__ jcc(Assembler::aboveEqual, *overflow);
__ bind(done);
}
}
void InterpreterGenerator::generate_counter_overflow(Label* do_continue) {
__ movl(c_rarg1, 0);
__ call_VM(noreg,
CAST_FROM_FN_PTR(address,
InterpreterRuntime::frequency_counter_overflow),
c_rarg1);
__ movptr(rbx, Address(rbp, method_offset)); // restore Method*
__ jmp(*do_continue, relocInfo::none);
}
void InterpreterGenerator::generate_stack_overflow_check(void) {
const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
const int overhead_size =
-(frame::interpreter_frame_initial_sp_offset * wordSize) + entry_size;
const int page_size = os::vm_page_size();
Label after_frame_check;
__ cmpl(rdx, (page_size - overhead_size) / Interpreter::stackElementSize);
__ jcc(Assembler::belowEqual, after_frame_check);
const Address stack_base(r15_thread, Thread::stack_base_offset());
const Address stack_size(r15_thread, Thread::stack_size_offset());
__ mov(rax, rdx);
__ shlptr(rax, Interpreter::logStackElementSize); // 2 slots per parameter.
__ addptr(rax, overhead_size);
#ifdef ASSERT
Label stack_base_okay, stack_size_okay;
__ cmpptr(stack_base, (int32_t)NULL_WORD);
__ jcc(Assembler::notEqual, stack_base_okay);
__ stop("stack base is zero");
__ bind(stack_base_okay);
__ cmpptr(stack_size, 0);
__ jcc(Assembler::notEqual, stack_size_okay);
__ stop("stack size is zero");
__ bind(stack_size_okay);
#endif
__ addptr(rax, stack_base);
__ subptr(rax, stack_size);
const int max_pages = StackShadowPages > (StackRedPages+StackYellowPages) ? StackShadowPages :
(StackRedPages+StackYellowPages);
__ addptr(rax, max_pages * page_size);
__ cmpptr(rsp, rax);
__ jcc(Assembler::above, after_frame_check);
__ pop(rax); // return address must be moved if SP is changed
__ mov(rsp, r13);
__ push(rax);
assert(StubRoutines::throw_StackOverflowError_entry() != NULL, "stub not yet generated");
__ jump(ExternalAddress(StubRoutines::throw_StackOverflowError_entry()));
__ bind(after_frame_check);
}
void InterpreterGenerator::lock_method(void) {
const Address access_flags(rbx, Method::access_flags_offset());
const Address monitor_block_top(
rbp,
frame::interpreter_frame_monitor_block_top_offset * wordSize);
const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
#ifdef ASSERT
{
Label L;
__ movl(rax, access_flags);
__ testl(rax, JVM_ACC_SYNCHRONIZED);
__ jcc(Assembler::notZero, L);
__ stop("method doesn't need synchronization");
__ bind(L);
}
#endif // ASSERT
{
const int mirror_offset = in_bytes(Klass::java_mirror_offset());
Label done;
__ movl(rax, access_flags);
__ testl(rax, JVM_ACC_STATIC);
__ movptr(rax, Address(r14, Interpreter::local_offset_in_bytes(0)));
__ jcc(Assembler::zero, done);
__ movptr(rax, Address(rbx, Method::const_offset()));
__ movptr(rax, Address(rax, ConstMethod::constants_offset()));
__ movptr(rax, Address(rax,
ConstantPool::pool_holder_offset_in_bytes()));
__ movptr(rax, Address(rax, mirror_offset));
#ifdef ASSERT
{
Label L;
__ testptr(rax, rax);
__ jcc(Assembler::notZero, L);
__ stop("synchronization object is NULL");
__ bind(L);
}
#endif // ASSERT
__ bind(done);
}
__ subptr(rsp, entry_size); // add space for a monitor entry
__ movptr(monitor_block_top, rsp); // set new monitor block top
__ movptr(Address(rsp, BasicObjectLock::obj_offset_in_bytes()), rax);
__ movptr(c_rarg1, rsp); // object address
__ lock_object(c_rarg1);
}
void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
__ push(rax); // save return address
__ enter(); // save old & set new rbp
__ push(r13); // set sender sp
__ push((int)NULL_WORD); // leave last_sp as null
__ movptr(r13, Address(rbx, Method::const_offset())); // get ConstMethod*
__ lea(r13, Address(r13, ConstMethod::codes_offset())); // get codebase
__ push(rbx); // save Method*
if (ProfileInterpreter) {
Label method_data_continue;
__ movptr(rdx, Address(rbx, in_bytes(Method::method_data_offset())));
__ testptr(rdx, rdx);
__ jcc(Assembler::zero, method_data_continue);
__ addptr(rdx, in_bytes(MethodData::data_offset()));
__ bind(method_data_continue);
__ push(rdx); // set the mdp (method data pointer)
} else {
__ push(0);
}
__ movptr(rdx, Address(rbx, Method::const_offset()));
__ movptr(rdx, Address(rdx, ConstMethod::constants_offset()));
__ movptr(rdx, Address(rdx, ConstantPool::cache_offset_in_bytes()));
__ push(rdx); // set constant pool cache
__ push(r14); // set locals pointer
if (native_call) {
__ push(0); // no bcp
} else {
__ push(r13); // set bcp
}
__ push(0); // reserve word for pointer to expression stack bottom
__ movptr(Address(rsp, 0), rsp); // set expression stack bottom
}
address InterpreterGenerator::generate_accessor_entry(void) {
address entry_point = __ pc();
Label xreturn_path;
if (UseFastAccessorMethods) {
Label slow_path;
__ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
SafepointSynchronize::_not_synchronized);
__ jcc(Assembler::notEqual, slow_path);
__ movptr(rax, Address(rsp, wordSize));
__ testptr(rax, rax);
__ jcc(Assembler::zero, slow_path);
__ movptr(rdx, Address(rbx, Method::const_offset()));
__ movptr(rdi, Address(rdx, ConstMethod::constants_offset()));
__ movl(rdx, Address(rdx, ConstMethod::codes_offset()));
__ shrl(rdx, 2 * BitsPerByte);
__ shll(rdx, exact_log2(in_words(ConstantPoolCacheEntry::size())));
__ movptr(rdi, Address(rdi, ConstantPool::cache_offset_in_bytes()));
assert(in_words(ConstantPoolCacheEntry::size()) == 4,
"adjust shift below");
__ movl(rcx,
Address(rdi,
rdx,
Address::times_8,
ConstantPoolCache::base_offset() +
ConstantPoolCacheEntry::indices_offset()));
__ shrl(rcx, 2 * BitsPerByte);
__ andl(rcx, 0xFF);
__ cmpl(rcx, Bytecodes::_getfield);
__ jcc(Assembler::notEqual, slow_path);
__ movptr(rcx,
Address(rdi,
rdx,
Address::times_8,
ConstantPoolCache::base_offset() +
ConstantPoolCacheEntry::f2_offset()));
__ movl(rdx,
Address(rdi,
rdx,
Address::times_8,
ConstantPoolCache::base_offset() +
ConstantPoolCacheEntry::flags_offset()));
Label notObj, notInt, notByte, notBool, notShort;
const Address field_address(rax, rcx, Address::times_1);
__ shrl(rdx, ConstantPoolCacheEntry::tos_state_shift);
ConstantPoolCacheEntry::verify_tos_state_shift();
__ cmpl(rdx, atos);
__ jcc(Assembler::notEqual, notObj);
__ load_heap_oop(rax, field_address);
__ jmp(xreturn_path);
__ bind(notObj);
__ cmpl(rdx, itos);
__ jcc(Assembler::notEqual, notInt);
__ movl(rax, field_address);
__ jmp(xreturn_path);
__ bind(notInt);
__ cmpl(rdx, btos);
__ jcc(Assembler::notEqual, notByte);
__ load_signed_byte(rax, field_address);
__ jmp(xreturn_path);
__ bind(notByte);
__ cmpl(rdx, ztos);
__ jcc(Assembler::notEqual, notBool);
__ load_signed_byte(rax, field_address);
__ jmp(xreturn_path);
__ bind(notBool);
__ cmpl(rdx, stos);
__ jcc(Assembler::notEqual, notShort);
__ load_signed_short(rax, field_address);
__ jmp(xreturn_path);
__ bind(notShort);
#ifdef ASSERT
Label okay;
__ cmpl(rdx, ctos);
__ jcc(Assembler::equal, okay);
__ stop("what type is this?");
__ bind(okay);
#endif
__ load_unsigned_short(rax, field_address);
__ bind(xreturn_path);
__ pop(rdi);
__ mov(rsp, r13);
__ jmp(rdi);
__ ret(0);
__ bind(slow_path);
(void) generate_normal_entry(false);
} else {
(void) generate_normal_entry(false);
}
return entry_point;
}
address InterpreterGenerator::generate_Reference_get_entry(void) {
#if INCLUDE_ALL_GCS
address entry = __ pc();
const int referent_offset = java_lang_ref_Reference::referent_offset;
guarantee(referent_offset > 0, "referent offset not initialized");
if (UseG1GC) {
Label slow_path;
__ movptr(rax, Address(rsp, wordSize));
__ testptr(rax, rax);
__ jcc(Assembler::zero, slow_path);
const Address field_address(rax, referent_offset);
__ load_heap_oop(rax, field_address);
__ g1_write_barrier_pre(noreg /* obj */,
rax /* pre_val */,
r15_thread /* thread */,
rbx /* tmp */,
true /* tosca_live */,
true /* expand_call */);
__ pop(rdi); // get return address
__ mov(rsp, r13); // set sp to sender sp
__ jmp(rdi);
__ ret(0);
__ bind(slow_path);
(void) generate_normal_entry(false);
return entry;
}
#endif // INCLUDE_ALL_GCS
return generate_accessor_entry();
}
address InterpreterGenerator::generate_CRC32_update_entry() {
if (UseCRC32Intrinsics) {
address entry = __ pc();
Label slow_path;
ExternalAddress state(SafepointSynchronize::address_of_state());
__ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
SafepointSynchronize::_not_synchronized);
__ jcc(Assembler::notEqual, slow_path);
const Register crc = rax; // crc
const Register val = c_rarg0; // source java byte value
const Register tbl = c_rarg1; // scratch
__ movl(val, Address(rsp, wordSize)); // byte value
__ movl(crc, Address(rsp, 2*wordSize)); // Initial CRC
__ lea(tbl, ExternalAddress(StubRoutines::crc_table_addr()));
__ notl(crc); // ~crc
__ update_byte_crc32(crc, val, tbl);
__ notl(crc); // ~crc
__ pop(rdi); // get return address
__ mov(rsp, r13); // set sp to sender sp
__ jmp(rdi);
__ bind(slow_path);
(void) generate_native_entry(false);
return entry;
}
return generate_native_entry(false);
}
address InterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractInterpreter::MethodKind kind) {
if (UseCRC32Intrinsics) {
address entry = __ pc();
Label slow_path;
ExternalAddress state(SafepointSynchronize::address_of_state());
__ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
SafepointSynchronize::_not_synchronized);
__ jcc(Assembler::notEqual, slow_path);
const Register crc = c_rarg0; // crc
const Register buf = c_rarg1; // source java byte array address
const Register len = c_rarg2; // length
const Register off = len; // offset (never overlaps with 'len')
if (kind == Interpreter::java_util_zip_CRC32_updateByteBuffer) {
__ movptr(buf, Address(rsp, 3*wordSize)); // long buf
__ movl2ptr(off, Address(rsp, 2*wordSize)); // offset
__ addq(buf, off); // + offset
__ movl(crc, Address(rsp, 5*wordSize)); // Initial CRC
} else {
__ movptr(buf, Address(rsp, 3*wordSize)); // byte[] array
__ addptr(buf, arrayOopDesc::base_offset_in_bytes(T_BYTE)); // + header size
__ movl2ptr(off, Address(rsp, 2*wordSize)); // offset
__ addq(buf, off); // + offset
__ movl(crc, Address(rsp, 4*wordSize)); // Initial CRC
}
__ movl(len, Address(rsp, wordSize)); // Length
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, StubRoutines::updateBytesCRC32()), crc, buf, len);
__ pop(rdi); // get return address
__ mov(rsp, r13); // set sp to sender sp
__ jmp(rdi);
__ bind(slow_path);
(void) generate_native_entry(false);
return entry;
}
return generate_native_entry(false);
}
address InterpreterGenerator::generate_native_entry(bool synchronized) {
bool inc_counter = UseCompiler || CountCompiledCalls;
address entry_point = __ pc();
const Address constMethod (rbx, Method::const_offset());
const Address access_flags (rbx, Method::access_flags_offset());
const Address size_of_parameters(rcx, ConstMethod::
size_of_parameters_offset());
__ movptr(rcx, constMethod);
__ load_unsigned_short(rcx, size_of_parameters);
__ pop(rax); // get return address
__ lea(r14, Address(rsp, rcx, Address::times_8, -wordSize));
__ push((int) NULL_WORD);
__ push((int) NULL_WORD);
generate_fixed_frame(true);
#ifdef ASSERT
__ movl(rax, access_flags);
{
Label L;
__ testl(rax, JVM_ACC_NATIVE);
__ jcc(Assembler::notZero, L);
__ stop("tried to execute non-native method as native");
__ bind(L);
}
{
Label L;
__ testl(rax, JVM_ACC_ABSTRACT);
__ jcc(Assembler::zero, L);
__ stop("tried to execute abstract method in interpreter");
__ bind(L);
}
#endif
const Address do_not_unlock_if_synchronized(r15_thread,
in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
__ movbool(do_not_unlock_if_synchronized, true);
Label invocation_counter_overflow;
if (inc_counter) {
generate_counter_incr(&invocation_counter_overflow, NULL, NULL);
}
Label continue_after_compile;
__ bind(continue_after_compile);
bang_stack_shadow_pages(true);
__ movbool(do_not_unlock_if_synchronized, false);
if (synchronized) {
lock_method();
} else {
#ifdef ASSERT
{
Label L;
__ movl(rax, access_flags);
__ testl(rax, JVM_ACC_SYNCHRONIZED);
__ jcc(Assembler::zero, L);
__ stop("method needs synchronization");
__ bind(L);
}
#endif
}
#ifdef ASSERT
{
Label L;
const Address monitor_block_top(rbp,
frame::interpreter_frame_monitor_block_top_offset * wordSize);
__ movptr(rax, monitor_block_top);
__ cmpptr(rax, rsp);
__ jcc(Assembler::equal, L);
__ stop("broken stack frame setup in interpreter");
__ bind(L);
}
#endif
__ notify_method_entry();
const Register method = rbx;
const Register t = r11;
__ get_method(method);
__ movptr(t, Address(method, Method::const_offset()));
__ load_unsigned_short(t, Address(t, ConstMethod::size_of_parameters_offset()));
__ shll(t, Interpreter::logStackElementSize);
__ subptr(rsp, t);
__ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
__ andptr(rsp, -16); // must be 16 byte boundary (see amd64 ABI)
{
Label L;
__ movptr(t, Address(method, Method::signature_handler_offset()));
__ testptr(t, t);
__ jcc(Assembler::notZero, L);
__ call_VM(noreg,
CAST_FROM_FN_PTR(address,
InterpreterRuntime::prepare_native_call),
method);
__ get_method(method);
__ movptr(t, Address(method, Method::signature_handler_offset()));
__ bind(L);
}
assert(InterpreterRuntime::SignatureHandlerGenerator::from() == r14,
"adjust this code");
assert(InterpreterRuntime::SignatureHandlerGenerator::to() == rsp,
"adjust this code");
assert(InterpreterRuntime::SignatureHandlerGenerator::temp() == rscratch1,
"adjust this code");
__ call(t);
__ get_method(method); // slow path can do a GC, reload RBX
__ movptr(Address(rbp,
(frame::interpreter_frame_result_handler_offset) * wordSize),
rax);
{
Label L;
const int mirror_offset = in_bytes(Klass::java_mirror_offset());
__ movl(t, Address(method, Method::access_flags_offset()));
__ testl(t, JVM_ACC_STATIC);
__ jcc(Assembler::zero, L);
__ movptr(t, Address(method, Method::const_offset()));
__ movptr(t, Address(t, ConstMethod::constants_offset()));
__ movptr(t, Address(t, ConstantPool::pool_holder_offset_in_bytes()));
__ movptr(t, Address(t, mirror_offset));
__ movptr(Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize),
t);
__ lea(c_rarg1,
Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize));
__ bind(L);
}
{
Label L;
__ movptr(rax, Address(method, Method::native_function_offset()));
ExternalAddress unsatisfied(SharedRuntime::native_method_throw_unsatisfied_link_error_entry());
__ movptr(rscratch2, unsatisfied.addr());
__ cmpptr(rax, rscratch2);
__ jcc(Assembler::notEqual, L);
__ call_VM(noreg,
CAST_FROM_FN_PTR(address,
InterpreterRuntime::prepare_native_call),
method);
__ get_method(method);
__ movptr(rax, Address(method, Method::native_function_offset()));
__ bind(L);
}
__ lea(c_rarg0, Address(r15_thread, JavaThread::jni_environment_offset()));
__ set_last_Java_frame(rsp, rbp, (address) __ pc());
#ifdef ASSERT
{
Label L;
__ movl(t, Address(r15_thread, JavaThread::thread_state_offset()));
__ cmpl(t, _thread_in_Java);
__ jcc(Assembler::equal, L);
__ stop("Wrong thread state in native stub");
__ bind(L);
}
#endif
__ movl(Address(r15_thread, JavaThread::thread_state_offset()),
_thread_in_native);
__ call(rax);
__ restore_cpu_control_state_after_jni();
__ push(dtos);
__ push(ltos);
__ movl(Address(r15_thread, JavaThread::thread_state_offset()),
_thread_in_native_trans);
if (os::is_MP()) {
if (UseMembar) {
__ membar(Assembler::Membar_mask_bits(
Assembler::LoadLoad | Assembler::LoadStore |
Assembler::StoreLoad | Assembler::StoreStore));
} else {
__ serialize_memory(r15_thread, rscratch2);
}
}
{
Label Continue;
__ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
SafepointSynchronize::_not_synchronized);
Label L;
__ jcc(Assembler::notEqual, L);
__ cmpl(Address(r15_thread, JavaThread::suspend_flags_offset()), 0);
__ jcc(Assembler::equal, Continue);
__ bind(L);
__ mov(c_rarg0, r15_thread);
__ mov(r12, rsp); // remember sp (can only use r12 if not using call_VM)
__ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
__ andptr(rsp, -16); // align stack as required by ABI
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
__ mov(rsp, r12); // restore sp
__ reinit_heapbase();
__ bind(Continue);
}
__ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
__ reset_last_Java_frame(r15_thread, true);
__ movptr(t, Address(r15_thread, JavaThread::active_handles_offset()));
__ movl(Address(t, JNIHandleBlock::top_offset_in_bytes()), (int32_t)NULL_WORD);
{
Label no_oop;
__ lea(t, ExternalAddress(AbstractInterpreter::result_handler(T_OBJECT)));
__ cmpptr(t, Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize));
__ jcc(Assembler::notEqual, no_oop);
__ pop(ltos);
__ resolve_jobject(rax /* value */,
r15_thread /* thread */,
t /* tmp */);
__ movptr(Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize), rax);
__ push(ltos);
__ bind(no_oop);
}
{
Label no_reguard;
__ cmpl(Address(r15_thread, JavaThread::stack_guard_state_offset()),
JavaThread::stack_guard_yellow_disabled);
__ jcc(Assembler::notEqual, no_reguard);
__ pusha(); // XXX only save smashed registers
__ mov(r12, rsp); // remember sp (can only use r12 if not using call_VM)
__ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
__ andptr(rsp, -16); // align stack as required by ABI
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
__ mov(rsp, r12); // restore sp
__ popa(); // XXX only restore smashed registers
__ reinit_heapbase();
__ bind(no_reguard);
}
__ get_method(method);
__ movptr(r13, Address(method, Method::const_offset())); // get ConstMethod*
__ lea(r13, Address(r13, ConstMethod::codes_offset())); // get codebase
{
Label L;
__ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
__ jcc(Assembler::zero, L);
__ MacroAssembler::call_VM(noreg,
CAST_FROM_FN_PTR(address,
InterpreterRuntime::throw_pending_exception));
__ should_not_reach_here();
__ bind(L);
}
{
Label L;
__ movl(t, Address(method, Method::access_flags_offset()));
__ testl(t, JVM_ACC_SYNCHRONIZED);
__ jcc(Assembler::zero, L);
{
Label unlock;
const Address monitor(rbp,
(intptr_t)(frame::interpreter_frame_initial_sp_offset *
wordSize - sizeof(BasicObjectLock)));
__ lea(c_rarg1, monitor); // address of first monitor
__ movptr(t, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
__ testptr(t, t);
__ jcc(Assembler::notZero, unlock);
__ MacroAssembler::call_VM(noreg,
CAST_FROM_FN_PTR(address,
InterpreterRuntime::throw_illegal_monitor_state_exception));
__ should_not_reach_here();
__ bind(unlock);
__ unlock_object(c_rarg1);
}
__ bind(L);
}
__ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI);
__ pop(ltos);
__ pop(dtos);
__ movptr(t, Address(rbp,
(frame::interpreter_frame_result_handler_offset) * wordSize));
__ call(t);
__ movptr(t, Address(rbp,
frame::interpreter_frame_sender_sp_offset *
wordSize)); // get sender sp
__ leave(); // remove frame anchor
__ pop(rdi); // get return address
__ mov(rsp, t); // set sp to sender sp
__ jmp(rdi);
if (inc_counter) {
__ bind(invocation_counter_overflow);
generate_counter_overflow(&continue_after_compile);
}
return entry_point;
}
address InterpreterGenerator::generate_normal_entry(bool synchronized) {
bool inc_counter = UseCompiler || CountCompiledCalls;
address entry_point = __ pc();
const Address constMethod(rbx, Method::const_offset());
const Address access_flags(rbx, Method::access_flags_offset());
const Address size_of_parameters(rdx,
ConstMethod::size_of_parameters_offset());
const Address size_of_locals(rdx, ConstMethod::size_of_locals_offset());
__ movptr(rdx, constMethod);
__ load_unsigned_short(rcx, size_of_parameters);
__ load_unsigned_short(rdx, size_of_locals); // get size of locals in words
__ subl(rdx, rcx); // rdx = no. of additional locals
generate_stack_overflow_check();
__ pop(rax);
__ lea(r14, Address(rsp, rcx, Address::times_8, -wordSize));
{
Label exit, loop;
__ testl(rdx, rdx);
__ jcc(Assembler::lessEqual, exit); // do nothing if rdx <= 0
__ bind(loop);
__ push((int) NULL_WORD); // initialize local variables
__ decrementl(rdx); // until everything initialized
__ jcc(Assembler::greater, loop);
__ bind(exit);
}
generate_fixed_frame(false);
#ifdef ASSERT
__ movl(rax, access_flags);
{
Label L;
__ testl(rax, JVM_ACC_NATIVE);
__ jcc(Assembler::zero, L);
__ stop("tried to execute native method as non-native");
__ bind(L);
}
{
Label L;
__ testl(rax, JVM_ACC_ABSTRACT);
__ jcc(Assembler::zero, L);
__ stop("tried to execute abstract method in interpreter");
__ bind(L);
}
#endif
const Address do_not_unlock_if_synchronized(r15_thread,
in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
__ movbool(do_not_unlock_if_synchronized, true);
__ profile_parameters_type(rax, rcx, rdx);
Label invocation_counter_overflow;
Label profile_method;
Label profile_method_continue;
if (inc_counter) {
generate_counter_incr(&invocation_counter_overflow,
&profile_method,
&profile_method_continue);
if (ProfileInterpreter) {
__ bind(profile_method_continue);
}
}
Label continue_after_compile;
__ bind(continue_after_compile);
bang_stack_shadow_pages(false);
__ movbool(do_not_unlock_if_synchronized, false);
if (synchronized) {
lock_method();
} else {
#ifdef ASSERT
{
Label L;
__ movl(rax, access_flags);
__ testl(rax, JVM_ACC_SYNCHRONIZED);
__ jcc(Assembler::zero, L);
__ stop("method needs synchronization");
__ bind(L);
}
#endif
}
#ifdef ASSERT
{
Label L;
const Address monitor_block_top (rbp,
frame::interpreter_frame_monitor_block_top_offset * wordSize);
__ movptr(rax, monitor_block_top);
__ cmpptr(rax, rsp);
__ jcc(Assembler::equal, L);
__ stop("broken stack frame setup in interpreter");
__ bind(L);
}
#endif
__ notify_method_entry();
__ dispatch_next(vtos);
if (inc_counter) {
if (ProfileInterpreter) {
__ bind(profile_method);
__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
__ set_method_data_pointer_for_bcp();
__ get_method(rbx);
__ jmp(profile_method_continue);
}
__ bind(invocation_counter_overflow);
generate_counter_overflow(&continue_after_compile);
}
return entry_point;
}
address AbstractInterpreterGenerator::generate_method_entry(
AbstractInterpreter::MethodKind kind) {
bool synchronized = false;
address entry_point = NULL;
InterpreterGenerator* ig_this = (InterpreterGenerator*)this;
switch (kind) {
case Interpreter::zerolocals : break;
case Interpreter::zerolocals_synchronized: synchronized = true; break;
case Interpreter::native : entry_point = ig_this->generate_native_entry(false); break;
case Interpreter::native_synchronized : entry_point = ig_this->generate_native_entry(true); break;
case Interpreter::empty : entry_point = ig_this->generate_empty_entry(); break;
case Interpreter::accessor : entry_point = ig_this->generate_accessor_entry(); break;
case Interpreter::abstract : entry_point = ig_this->generate_abstract_entry(); break;
case Interpreter::java_lang_math_sin : // fall thru
case Interpreter::java_lang_math_cos : // fall thru
case Interpreter::java_lang_math_tan : // fall thru
case Interpreter::java_lang_math_abs : // fall thru
case Interpreter::java_lang_math_log : // fall thru
case Interpreter::java_lang_math_log10 : // fall thru
case Interpreter::java_lang_math_sqrt : // fall thru
case Interpreter::java_lang_math_pow : // fall thru
case Interpreter::java_lang_math_exp : entry_point = ig_this->generate_math_entry(kind); break;
case Interpreter::java_lang_ref_reference_get
: entry_point = ig_this->generate_Reference_get_entry(); break;
case Interpreter::java_util_zip_CRC32_update
: entry_point = ig_this->generate_CRC32_update_entry(); break;
case Interpreter::java_util_zip_CRC32_updateBytes
: // fall thru
case Interpreter::java_util_zip_CRC32_updateByteBuffer
: entry_point = ig_this->generate_CRC32_updateBytes_entry(kind); break;
default:
fatal(err_msg("unexpected method kind: %d", kind));
break;
}
if (entry_point) {
return entry_point;
}
return ig_this->generate_normal_entry(synchronized);
}
bool AbstractInterpreter::can_be_compiled(methodHandle m) {
switch (method_kind(m)) {
case Interpreter::java_lang_math_sin : // fall thru
case Interpreter::java_lang_math_cos : // fall thru
case Interpreter::java_lang_math_tan : // fall thru
case Interpreter::java_lang_math_abs : // fall thru
case Interpreter::java_lang_math_log : // fall thru
case Interpreter::java_lang_math_log10 : // fall thru
case Interpreter::java_lang_math_sqrt : // fall thru
case Interpreter::java_lang_math_pow : // fall thru
case Interpreter::java_lang_math_exp :
return false;
default:
return true;
}
}
int AbstractInterpreter::size_top_interpreter_activation(Method* method) {
const int entry_size = frame::interpreter_frame_monitor_size();
const int overhead_size =
-(frame::interpreter_frame_initial_sp_offset) + entry_size;
const int stub_code = frame::entry_frame_after_call_words;
const int method_stack = (method->max_locals() + method->max_stack()) *
Interpreter::stackElementWords;
return (overhead_size + method_stack + stub_code);
}
void TemplateInterpreterGenerator::generate_throw_exception() {
Interpreter::_rethrow_exception_entry = __ pc();
__ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
__ restore_bcp(); // r13 points to call/send
__ restore_locals();
__ reinit_heapbase(); // restore r12 as heapbase.
Interpreter::_throw_exception_entry = __ pc();
__ verify_oop(rax);
__ mov(c_rarg1, rax);
__ empty_expression_stack();
__ call_VM(rdx,
CAST_FROM_FN_PTR(address,
InterpreterRuntime::exception_handler_for_exception),
c_rarg1);
__ push_ptr(rdx); // push exception which is now the only value on the stack
__ jmp(rax); // jump to exception handler (may be _remove_activation_entry!)
Interpreter::_remove_activation_preserving_args_entry = __ pc();
__ empty_expression_stack();
__ movl(rdx, Address(r15_thread, JavaThread::popframe_condition_offset()));
__ orl(rdx, JavaThread::popframe_processing_bit);
__ movl(Address(r15_thread, JavaThread::popframe_condition_offset()), rdx);
{
Label caller_not_deoptimized;
__ movptr(c_rarg1, Address(rbp, frame::return_addr_offset * wordSize));
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
InterpreterRuntime::interpreter_contains), c_rarg1);
__ testl(rax, rax);
__ jcc(Assembler::notZero, caller_not_deoptimized);
__ get_method(rax);
__ movptr(rax, Address(rax, Method::const_offset()));
__ load_unsigned_short(rax, Address(rax, in_bytes(ConstMethod::
size_of_parameters_offset())));
__ shll(rax, Interpreter::logStackElementSize);
__ restore_locals(); // XXX do we need this?
__ subptr(r14, rax);
__ addptr(r14, wordSize);
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
Deoptimization::
popframe_preserve_args),
r15_thread, rax, r14);
__ remove_activation(vtos, rdx,
__ movl(Address(r15_thread, JavaThread::popframe_condition_offset()),
JavaThread::popframe_force_deopt_reexecution_bit);
__ jmp(rdx);
__ bind(caller_not_deoptimized);
}
__ remove_activation(vtos, rdx, /* rdx result (retaddr) is not used */
__ mov(c_rarg1, rsp);
__ movptr(c_rarg2, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
__ set_last_Java_frame(noreg, rbp, __ pc());
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), r15_thread, c_rarg1, c_rarg2);
__ reset_last_Java_frame(r15_thread, true);
__ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
__ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
__ restore_bcp(); // XXX do we need this?
__ restore_locals(); // XXX do we need this?
if (ProfileInterpreter) {
__ set_method_data_pointer_for_bcp();
}
__ movl(Address(r15_thread, JavaThread::popframe_condition_offset()),
JavaThread::popframe_inactive);
#if INCLUDE_JVMTI
if (EnableInvokeDynamic) {
Label L_done;
const Register local0 = r14;
__ cmpb(Address(r13, 0), Bytecodes::_invokestatic);
__ jcc(Assembler::notEqual, L_done);
__ get_method(rdx);
__ movptr(rax, Address(local0, 0));
__ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::member_name_arg_or_null), rax, rdx, r13);
__ testptr(rax, rax);
__ jcc(Assembler::zero, L_done);
__ movptr(Address(rbx, 0), rax);
__ bind(L_done);
}
#endif // INCLUDE_JVMTI
__ dispatch_next(vtos);
Interpreter::_remove_activation_entry = __ pc();
__ pop_ptr(rax);
__ movptr(Address(r15_thread, JavaThread::vm_result_offset()), rax);
__ remove_activation(vtos, rdx, false, true, false);
__ get_vm_result(rax, r15_thread);
__ push(rax); // save exception
__ push(rdx); // save return address
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
SharedRuntime::exception_handler_for_return_address),
r15_thread, rdx);
__ mov(rbx, rax); // save exception handler
__ pop(rdx); // restore return address
__ pop(rax); // restore exception
__ jmp(rbx); // jump to exception
}
address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
address entry = __ pc();
__ restore_bcp();
__ restore_locals();
__ empty_expression_stack();
__ load_earlyret_value(state);
__ movptr(rdx, Address(r15_thread, JavaThread::jvmti_thread_state_offset()));
Address cond_addr(rdx, JvmtiThreadState::earlyret_state_offset());
__ movl(cond_addr, JvmtiThreadState::earlyret_inactive);
__ remove_activation(state, rsi,
false, /* throw_monitor_exception */
false, /* install_monitor_exception */
true); /* notify_jvmdi */
__ jmp(rsi);
return entry;
} // end of ForceEarlyReturn support
void TemplateInterpreterGenerator::set_vtos_entry_points(Template* t,
address& bep,
address& cep,
address& sep,
address& aep,
address& iep,
address& lep,
address& fep,
address& dep,
address& vep) {
assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
Label L;
aep = __ pc(); __ push_ptr(); __ jmp(L);
fep = __ pc(); __ push_f(); __ jmp(L);
dep = __ pc(); __ push_d(); __ jmp(L);
lep = __ pc(); __ push_l(); __ jmp(L);
bep = cep = sep =
iep = __ pc(); __ push_i();
vep = __ pc();
__ bind(L);
generate_and_dispatch(t);
}
InterpreterGenerator::InterpreterGenerator(StubQueue* code)
: TemplateInterpreterGenerator(code) {
generate_all(); // down here so it can be "virtual"
}
#ifndef PRODUCT
address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
address entry = __ pc();
__ push(state);
__ push(c_rarg0);
__ push(c_rarg1);
__ push(c_rarg2);
__ push(c_rarg3);
__ mov(c_rarg2, rax); // Pass itos
#ifdef _WIN64
__ movflt(xmm3, xmm0); // Pass ftos
#endif
__ call_VM(noreg,
CAST_FROM_FN_PTR(address, SharedRuntime::trace_bytecode),
c_rarg1, c_rarg2, c_rarg3);
__ pop(c_rarg3);
__ pop(c_rarg2);
__ pop(c_rarg1);
__ pop(c_rarg0);
__ pop(state);
__ ret(0); // return from result handler
return entry;
}
void TemplateInterpreterGenerator::count_bytecode() {
__ incrementl(ExternalAddress((address) &BytecodeCounter::_counter_value));
}
void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
__ incrementl(ExternalAddress((address) &BytecodeHistogram::_counters[t->bytecode()]));
}
void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
__ mov32(rbx, ExternalAddress((address) &BytecodePairHistogram::_index));
__ shrl(rbx, BytecodePairHistogram::log2_number_of_codes);
__ orl(rbx,
((int) t->bytecode()) <<
BytecodePairHistogram::log2_number_of_codes);
__ mov32(ExternalAddress((address) &BytecodePairHistogram::_index), rbx);
__ lea(rscratch1, ExternalAddress((address) BytecodePairHistogram::_counters));
__ incrementl(Address(rscratch1, rbx, Address::times_4));
}
void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
assert(Interpreter::trace_code(t->tos_in()) != NULL,
"entry must have been generated");
__ mov(r12, rsp); // remember sp (can only use r12 if not using call_VM)
__ andptr(rsp, -16); // align stack as required by ABI
__ call(RuntimeAddress(Interpreter::trace_code(t->tos_in())));
__ mov(rsp, r12); // restore sp
__ reinit_heapbase();
}
void TemplateInterpreterGenerator::stop_interpreter_at() {
Label L;
__ cmp32(ExternalAddress((address) &BytecodeCounter::_counter_value),
StopInterpreterAt);
__ jcc(Assembler::notEqual, L);
__ int3();
__ bind(L);
}
#endif // !PRODUCT
#endif // ! CC_INTERP
C:\hotspot-69087d08d473\src\cpu\x86\vm/templateTable_x86_32.cpp
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
#include "interpreter/interpreter.hpp"
#include "interpreter/interpreterRuntime.hpp"
#include "interpreter/templateTable.hpp"
#include "memory/universe.inline.hpp"
#include "oops/methodData.hpp"
#include "oops/objArrayKlass.hpp"
#include "oops/oop.inline.hpp"
#include "prims/methodHandles.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
#include "runtime/synchronizer.hpp"
#include "utilities/macros.hpp"
#ifndef CC_INTERP
#define __ _masm->
void TemplateTable::pd_initialize() {
}
static inline Address iaddress(int n) {
return Address(rdi, Interpreter::local_offset_in_bytes(n));
}
static inline Address laddress(int n) { return iaddress(n + 1); }
static inline Address haddress(int n) { return iaddress(n + 0); }
static inline Address faddress(int n) { return iaddress(n); }
static inline Address daddress(int n) { return laddress(n); }
static inline Address aaddress(int n) { return iaddress(n); }
static inline Address iaddress(Register r) {
return Address(rdi, r, Interpreter::stackElementScale());
}
static inline Address laddress(Register r) {
return Address(rdi, r, Interpreter::stackElementScale(), Interpreter::local_offset_in_bytes(1));
}
static inline Address haddress(Register r) {
return Address(rdi, r, Interpreter::stackElementScale(), Interpreter::local_offset_in_bytes(0));
}
static inline Address faddress(Register r) { return iaddress(r); }
static inline Address daddress(Register r) { return laddress(r); }
static inline Address aaddress(Register r) { return iaddress(r); }
static inline Address at_rsp () {
return Address(rsp, 0);
}
static inline Address at_tos () {
Address tos = Address(rsp, Interpreter::expr_offset_in_bytes(0));
return tos;
}
static inline Address at_tos_p1() {
return Address(rsp, Interpreter::expr_offset_in_bytes(1));
}
static inline Address at_tos_p2() {
return Address(rsp, Interpreter::expr_offset_in_bytes(2));
}
static Assembler::Condition j_not(TemplateTable::Condition cc) {
switch (cc) {
case TemplateTable::equal : return Assembler::notEqual;
case TemplateTable::not_equal : return Assembler::equal;
case TemplateTable::less : return Assembler::greaterEqual;
case TemplateTable::less_equal : return Assembler::greater;
case TemplateTable::greater : return Assembler::lessEqual;
case TemplateTable::greater_equal: return Assembler::less;
}
ShouldNotReachHere();
return Assembler::zero;
}
static void do_oop_store(InterpreterMacroAssembler* _masm,
Address obj,
Register val,
BarrierSet::Name barrier,
bool precise) {
assert(val == noreg || val == rax, "parameter is just for looks");
switch (barrier) {
#if INCLUDE_ALL_GCS
case BarrierSet::G1SATBCT:
case BarrierSet::G1SATBCTLogging:
{
if (obj.index() == noreg && obj.disp() == 0) {
if (obj.base() != rdx) {
__ movl(rdx, obj.base());
}
} else {
__ leal(rdx, obj);
}
__ get_thread(rcx);
__ save_bcp();
__ g1_write_barrier_pre(rdx /* obj */,
rbx /* pre_val */,
rcx /* thread */,
rsi /* tmp */,
val != noreg /* tosca_live */,
false /* expand_call */);
if (val == noreg) {
__ movptr(Address(rdx, 0), NULL_WORD);
} else {
__ movl(Address(rdx, 0), val);
__ g1_write_barrier_post(rdx /* store_adr */,
val /* new_val */,
rcx /* thread */,
rbx /* tmp */,
rsi /* tmp2 */);
}
__ restore_bcp();
}
break;
#endif // INCLUDE_ALL_GCS
case BarrierSet::CardTableModRef:
case BarrierSet::CardTableExtension:
{
if (val == noreg) {
__ movptr(obj, NULL_WORD);
} else {
__ movl(obj, val);
if (!precise || (obj.index() == noreg && obj.disp() == 0)) {
__ store_check(obj.base());
} else {
__ leal(rdx, obj);
__ store_check(rdx);
}
}
}
break;
case BarrierSet::ModRef:
case BarrierSet::Other:
if (val == noreg) {
__ movptr(obj, NULL_WORD);
} else {
__ movl(obj, val);
}
break;
default :
ShouldNotReachHere();
}
}
Address TemplateTable::at_bcp(int offset) {
assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
return Address(rsi, offset);
}
void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
int byte_no) {
if (!RewriteBytecodes) return;
Label L_patch_done;
switch (bc) {
case Bytecodes::_fast_aputfield:
case Bytecodes::_fast_bputfield:
case Bytecodes::_fast_zputfield:
case Bytecodes::_fast_cputfield:
case Bytecodes::_fast_dputfield:
case Bytecodes::_fast_fputfield:
case Bytecodes::_fast_iputfield:
case Bytecodes::_fast_lputfield:
case Bytecodes::_fast_sputfield:
{
assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
assert(load_bc_into_bc_reg, "we use bc_reg as temp");
__ get_cache_and_index_and_bytecode_at_bcp(bc_reg, temp_reg, temp_reg, byte_no, 1);
__ movl(bc_reg, bc);
__ cmpl(temp_reg, (int) 0);
__ jcc(Assembler::zero, L_patch_done); // don't patch
}
break;
default:
assert(byte_no == -1, "sanity");
if (load_bc_into_bc_reg) {
__ movl(bc_reg, bc);
}
}
if (JvmtiExport::can_post_breakpoint()) {
Label L_fast_patch;
__ movzbl(temp_reg, at_bcp(0));
__ cmpl(temp_reg, Bytecodes::_breakpoint);
__ jcc(Assembler::notEqual, L_fast_patch);
__ get_method(temp_reg);
__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), temp_reg, rsi, bc_reg);
#ifndef ASSERT
__ jmpb(L_patch_done);
#else
__ jmp(L_patch_done);
#endif
__ bind(L_fast_patch);
}
#ifdef ASSERT
Label L_okay;
__ load_unsigned_byte(temp_reg, at_bcp(0));
__ cmpl(temp_reg, (int)Bytecodes::java_code(bc));
__ jccb(Assembler::equal, L_okay);
__ cmpl(temp_reg, bc_reg);
__ jcc(Assembler::equal, L_okay);
__ stop("patching the wrong bytecode");
__ bind(L_okay);
#endif
__ movb(at_bcp(0), bc_reg);
__ bind(L_patch_done);
}
void TemplateTable::nop() {
transition(vtos, vtos);
}
void TemplateTable::shouldnotreachhere() {
transition(vtos, vtos);
__ stop("shouldnotreachhere bytecode");
}
void TemplateTable::aconst_null() {
transition(vtos, atos);
__ xorptr(rax, rax);
}
void TemplateTable::iconst(int value) {
transition(vtos, itos);
if (value == 0) {
__ xorptr(rax, rax);
} else {
__ movptr(rax, value);
}
}
void TemplateTable::lconst(int value) {
transition(vtos, ltos);
if (value == 0) {
__ xorptr(rax, rax);
} else {
__ movptr(rax, value);
}
assert(value >= 0, "check this code");
__ xorptr(rdx, rdx);
}
void TemplateTable::fconst(int value) {
transition(vtos, ftos);
if (value == 0) { __ fldz();
} else if (value == 1) { __ fld1();
} else if (value == 2) { __ fld1(); __ fld1(); __ faddp(); // should do a better solution here
} else { ShouldNotReachHere();
}
}
void TemplateTable::dconst(int value) {
transition(vtos, dtos);
if (value == 0) { __ fldz();
} else if (value == 1) { __ fld1();
} else { ShouldNotReachHere();
}
}
void TemplateTable::bipush() {
transition(vtos, itos);
__ load_signed_byte(rax, at_bcp(1));
}
void TemplateTable::sipush() {
transition(vtos, itos);
__ load_unsigned_short(rax, at_bcp(1));
__ bswapl(rax);
__ sarl(rax, 16);
}
void TemplateTable::ldc(bool wide) {
transition(vtos, vtos);
Label call_ldc, notFloat, notClass, Done;
if (wide) {
__ get_unsigned_2_byte_index_at_bcp(rbx, 1);
} else {
__ load_unsigned_byte(rbx, at_bcp(1));
}
__ get_cpool_and_tags(rcx, rax);
const int base_offset = ConstantPool::header_size() * wordSize;
const int tags_offset = Array<u1>::base_offset_in_bytes();
__ xorptr(rdx, rdx);
__ movb(rdx, Address(rax, rbx, Address::times_1, tags_offset));
__ cmpl(rdx, JVM_CONSTANT_UnresolvedClass);
__ jccb(Assembler::equal, call_ldc);
__ cmpl(rdx, JVM_CONSTANT_UnresolvedClassInError);
__ jccb(Assembler::equal, call_ldc);
__ cmpl(rdx, JVM_CONSTANT_Class);
__ jcc(Assembler::notEqual, notClass);
__ bind(call_ldc);
__ movl(rcx, wide);
call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), rcx);
__ push(atos);
__ jmp(Done);
__ bind(notClass);
__ cmpl(rdx, JVM_CONSTANT_Float);
__ jccb(Assembler::notEqual, notFloat);
__ fld_s( Address(rcx, rbx, Address::times_ptr, base_offset));
__ push(ftos);
__ jmp(Done);
__ bind(notFloat);
#ifdef ASSERT
{ Label L;
__ cmpl(rdx, JVM_CONSTANT_Integer);
__ jcc(Assembler::equal, L);
__ stop("unexpected tag type in ldc");
__ bind(L);
}
#endif
__ movl(rax, Address(rcx, rbx, Address::times_ptr, base_offset));
__ push(itos);
__ bind(Done);
}
void TemplateTable::fast_aldc(bool wide) {
transition(vtos, atos);
Register result = rax;
Register tmp = rdx;
int index_size = wide ? sizeof(u2) : sizeof(u1);
Label resolved;
assert_different_registers(result, tmp);
__ get_cache_index_at_bcp(tmp, 1, index_size);
__ load_resolved_reference_at_index(result, tmp);
__ testl(result, result);
__ jcc(Assembler::notZero, resolved);
address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
__ movl(tmp, (int)bytecode());
__ call_VM(result, entry, tmp);
__ bind(resolved);
if (VerifyOops) {
__ verify_oop(result);
}
}
void TemplateTable::ldc2_w() {
transition(vtos, vtos);
Label Long, Done;
__ get_unsigned_2_byte_index_at_bcp(rbx, 1);
__ get_cpool_and_tags(rcx, rax);
const int base_offset = ConstantPool::header_size() * wordSize;
const int tags_offset = Array<u1>::base_offset_in_bytes();
__ cmpb(Address(rax, rbx, Address::times_1, tags_offset), JVM_CONSTANT_Double);
__ jccb(Assembler::notEqual, Long);
__ fld_d( Address(rcx, rbx, Address::times_ptr, base_offset));
__ push(dtos);
__ jmpb(Done);
__ bind(Long);
__ movptr(rax, Address(rcx, rbx, Address::times_ptr, base_offset + 0 * wordSize));
NOT_LP64(__ movptr(rdx, Address(rcx, rbx, Address::times_ptr, base_offset + 1 * wordSize)));
__ push(ltos);
__ bind(Done);
}
void TemplateTable::locals_index(Register reg, int offset) {
__ load_unsigned_byte(reg, at_bcp(offset));
__ negptr(reg);
}
void TemplateTable::iload() {
transition(vtos, itos);
if (RewriteFrequentPairs) {
Label rewrite, done;
__ load_unsigned_byte(rbx, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
__ cmpl(rbx, Bytecodes::_iload);
__ jcc(Assembler::equal, done);
__ cmpl(rbx, Bytecodes::_fast_iload);
__ movl(rcx, Bytecodes::_fast_iload2);
__ jccb(Assembler::equal, rewrite);
__ cmpl(rbx, Bytecodes::_caload);
__ movl(rcx, Bytecodes::_fast_icaload);
__ jccb(Assembler::equal, rewrite);
__ movl(rcx, Bytecodes::_fast_iload);
__ bind(rewrite);
patch_bytecode(Bytecodes::_iload, rcx, rbx, false);
__ bind(done);
}
locals_index(rbx);
__ movl(rax, iaddress(rbx));
}
void TemplateTable::fast_iload2() {
transition(vtos, itos);
locals_index(rbx);
__ movl(rax, iaddress(rbx));
__ push(itos);
locals_index(rbx, 3);
__ movl(rax, iaddress(rbx));
}
void TemplateTable::fast_iload() {
transition(vtos, itos);
locals_index(rbx);
__ movl(rax, iaddress(rbx));
}
void TemplateTable::lload() {
transition(vtos, ltos);
locals_index(rbx);
__ movptr(rax, laddress(rbx));
NOT_LP64(__ movl(rdx, haddress(rbx)));
}
void TemplateTable::fload() {
transition(vtos, ftos);
locals_index(rbx);
__ fld_s(faddress(rbx));
}
void TemplateTable::dload() {
transition(vtos, dtos);
locals_index(rbx);
__ fld_d(daddress(rbx));
}
void TemplateTable::aload() {
transition(vtos, atos);
locals_index(rbx);
__ movptr(rax, aaddress(rbx));
}
void TemplateTable::locals_index_wide(Register reg) {
__ load_unsigned_short(reg, at_bcp(2));
__ bswapl(reg);
__ shrl(reg, 16);
__ negptr(reg);
}
void TemplateTable::wide_iload() {
transition(vtos, itos);
locals_index_wide(rbx);
__ movl(rax, iaddress(rbx));
}
void TemplateTable::wide_lload() {
transition(vtos, ltos);
locals_index_wide(rbx);
__ movptr(rax, laddress(rbx));
NOT_LP64(__ movl(rdx, haddress(rbx)));
}
void TemplateTable::wide_fload() {
transition(vtos, ftos);
locals_index_wide(rbx);
__ fld_s(faddress(rbx));
}
void TemplateTable::wide_dload() {
transition(vtos, dtos);
locals_index_wide(rbx);
__ fld_d(daddress(rbx));
}
void TemplateTable::wide_aload() {
transition(vtos, atos);
locals_index_wide(rbx);
__ movptr(rax, aaddress(rbx));
}
void TemplateTable::index_check(Register array, Register index) {
__ pop_ptr(array);
index_check_without_pop(array, index);
}
void TemplateTable::index_check_without_pop(Register array, Register index) {
__ null_check(array, arrayOopDesc::length_offset_in_bytes());
LP64_ONLY(__ movslq(index, index));
__ cmpl(index, Address(array, arrayOopDesc::length_offset_in_bytes()));
if (index != rbx) {
assert(rbx != array, "different registers");
__ mov(rbx, index);
}
__ jump_cc(Assembler::aboveEqual,
ExternalAddress(Interpreter::_throw_ArrayIndexOutOfBoundsException_entry));
}
void TemplateTable::iaload() {
transition(itos, itos);
index_check(rdx, rax); // kills rbx,
__ movl(rax, Address(rdx, rax, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_INT)));
}
void TemplateTable::laload() {
transition(itos, ltos);
index_check(rdx, rax);
__ mov(rbx, rax);
__ movptr(rax, Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 0 * wordSize));
NOT_LP64(__ movl(rdx, Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 1 * wordSize)));
}
void TemplateTable::faload() {
transition(itos, ftos);
index_check(rdx, rax); // kills rbx,
__ fld_s(Address(rdx, rax, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
}
void TemplateTable::daload() {
transition(itos, dtos);
index_check(rdx, rax); // kills rbx,
__ fld_d(Address(rdx, rax, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
}
void TemplateTable::aaload() {
transition(itos, atos);
index_check(rdx, rax); // kills rbx,
__ movptr(rax, Address(rdx, rax, Address::times_ptr, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
}
void TemplateTable::baload() {
transition(itos, itos);
index_check(rdx, rax); // kills rbx,
__ load_signed_byte(rbx, Address(rdx, rax, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_BYTE)));
__ mov(rax, rbx);
}
void TemplateTable::caload() {
transition(itos, itos);
index_check(rdx, rax); // kills rbx,
__ load_unsigned_short(rbx, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
__ mov(rax, rbx);
}
void TemplateTable::fast_icaload() {
transition(vtos, itos);
locals_index(rbx);
__ movl(rax, iaddress(rbx));
index_check(rdx, rax);
__ load_unsigned_short(rbx, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
__ mov(rax, rbx);
}
void TemplateTable::saload() {
transition(itos, itos);
index_check(rdx, rax); // kills rbx,
__ load_signed_short(rbx, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_SHORT)));
__ mov(rax, rbx);
}
void TemplateTable::iload(int n) {
transition(vtos, itos);
__ movl(rax, iaddress(n));
}
void TemplateTable::lload(int n) {
transition(vtos, ltos);
__ movptr(rax, laddress(n));
NOT_LP64(__ movptr(rdx, haddress(n)));
}
void TemplateTable::fload(int n) {
transition(vtos, ftos);
__ fld_s(faddress(n));
}
void TemplateTable::dload(int n) {
transition(vtos, dtos);
__ fld_d(daddress(n));
}
void TemplateTable::aload(int n) {
transition(vtos, atos);
__ movptr(rax, aaddress(n));
}
void TemplateTable::aload_0() {
transition(vtos, atos);
if (RewriteFrequentPairs) {
Label rewrite, done;
__ load_unsigned_byte(rbx, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
aload(0);
__ cmpl(rbx, Bytecodes::_getfield);
__ jcc(Assembler::equal, done);
assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
__ cmpl(rbx, Bytecodes::_fast_igetfield);
__ movl(rcx, Bytecodes::_fast_iaccess_0);
__ jccb(Assembler::equal, rewrite);
assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
__ cmpl(rbx, Bytecodes::_fast_agetfield);
__ movl(rcx, Bytecodes::_fast_aaccess_0);
__ jccb(Assembler::equal, rewrite);
assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
__ cmpl(rbx, Bytecodes::_fast_fgetfield);
__ movl(rcx, Bytecodes::_fast_faccess_0);
__ jccb(Assembler::equal, rewrite);
assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
__ movl(rcx, Bytecodes::_fast_aload_0);
__ bind(rewrite);
patch_bytecode(Bytecodes::_aload_0, rcx, rbx, false);
__ bind(done);
} else {
aload(0);
}
}
void TemplateTable::istore() {
transition(itos, vtos);
locals_index(rbx);
__ movl(iaddress(rbx), rax);
}
void TemplateTable::lstore() {
transition(ltos, vtos);
locals_index(rbx);
__ movptr(laddress(rbx), rax);
NOT_LP64(__ movptr(haddress(rbx), rdx));
}
void TemplateTable::fstore() {
transition(ftos, vtos);
locals_index(rbx);
__ fstp_s(faddress(rbx));
}
void TemplateTable::dstore() {
transition(dtos, vtos);
locals_index(rbx);
__ fstp_d(daddress(rbx));
}
void TemplateTable::astore() {
transition(vtos, vtos);
__ pop_ptr(rax);
locals_index(rbx);
__ movptr(aaddress(rbx), rax);
}
void TemplateTable::wide_istore() {
transition(vtos, vtos);
__ pop_i(rax);
locals_index_wide(rbx);
__ movl(iaddress(rbx), rax);
}
void TemplateTable::wide_lstore() {
transition(vtos, vtos);
__ pop_l(rax, rdx);
locals_index_wide(rbx);
__ movptr(laddress(rbx), rax);
NOT_LP64(__ movl(haddress(rbx), rdx));
}
void TemplateTable::wide_fstore() {
wide_istore();
}
void TemplateTable::wide_dstore() {
wide_lstore();
}
void TemplateTable::wide_astore() {
transition(vtos, vtos);
__ pop_ptr(rax);
locals_index_wide(rbx);
__ movptr(aaddress(rbx), rax);
}
void TemplateTable::iastore() {
transition(itos, vtos);
__ pop_i(rbx);
index_check(rdx, rbx); // prefer index in rbx,
__ movl(Address(rdx, rbx, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_INT)), rax);
}
void TemplateTable::lastore() {
transition(ltos, vtos);
__ pop_i(rbx);
index_check(rcx, rbx); // prefer index in rbx,
__ movptr(Address(rcx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 0 * wordSize), rax);
NOT_LP64(__ movl(Address(rcx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 1 * wordSize), rdx));
}
void TemplateTable::fastore() {
transition(ftos, vtos);
__ pop_i(rbx);
index_check(rdx, rbx); // prefer index in rbx,
__ fstp_s(Address(rdx, rbx, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
}
void TemplateTable::dastore() {
transition(dtos, vtos);
__ pop_i(rbx);
index_check(rdx, rbx); // prefer index in rbx,
__ fstp_d(Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
}
void TemplateTable::aastore() {
Label is_null, ok_is_subtype, done;
transition(vtos, vtos);
__ movptr(rax, at_tos()); // Value
__ movl(rcx, at_tos_p1()); // Index
__ movptr(rdx, at_tos_p2()); // Array
Address element_address(rdx, rcx, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
index_check_without_pop(rdx, rcx); // kills rbx,
__ testptr(rax, rax);
__ jcc(Assembler::zero, is_null);
__ load_klass(rbx, rax);
__ load_klass(rax, rdx);
__ movptr(rax, Address(rax, ObjArrayKlass::element_klass_offset()));
__ lea(rdx, element_address);
__ gen_subtype_check( rbx, ok_is_subtype );
__ jump(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry));
__ bind(ok_is_subtype);
__ movptr(rax, at_rsp());
do_oop_store(_masm, Address(rdx, 0), rax, _bs->kind(), true);
__ jmp(done);
__ bind(is_null);
__ profile_null_seen(rbx);
do_oop_store(_masm, element_address, noreg, _bs->kind(), true);
__ bind(done);
__ addptr(rsp, 3 * Interpreter::stackElementSize);
}
void TemplateTable::bastore() {
transition(itos, vtos);
__ pop_i(rbx);
index_check(rdx, rbx); // prefer index in rbx
__ load_klass(rcx, rdx);
__ movl(rcx, Address(rcx, Klass::layout_helper_offset()));
int diffbit = Klass::layout_helper_boolean_diffbit();
__ testl(rcx, diffbit);
Label L_skip;
__ jccb(Assembler::zero, L_skip);
__ andl(rax, 1); // if it is a T_BOOLEAN array, mask the stored value to 0/1
__ bind(L_skip);
__ movb(Address(rdx, rbx, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_BYTE)), rax);
}
void TemplateTable::castore() {
transition(itos, vtos);
__ pop_i(rbx);
index_check(rdx, rbx); // prefer index in rbx,
__ movw(Address(rdx, rbx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)), rax);
}
void TemplateTable::sastore() {
castore();
}
void TemplateTable::istore(int n) {
transition(itos, vtos);
__ movl(iaddress(n), rax);
}
void TemplateTable::lstore(int n) {
transition(ltos, vtos);
__ movptr(laddress(n), rax);
NOT_LP64(__ movptr(haddress(n), rdx));
}
void TemplateTable::fstore(int n) {
transition(ftos, vtos);
__ fstp_s(faddress(n));
}
void TemplateTable::dstore(int n) {
transition(dtos, vtos);
__ fstp_d(daddress(n));
}
void TemplateTable::astore(int n) {
transition(vtos, vtos);
__ pop_ptr(rax);
__ movptr(aaddress(n), rax);
}
void TemplateTable::pop() {
transition(vtos, vtos);
__ addptr(rsp, Interpreter::stackElementSize);
}
void TemplateTable::pop2() {
transition(vtos, vtos);
__ addptr(rsp, 2*Interpreter::stackElementSize);
}
void TemplateTable::dup() {
transition(vtos, vtos);
__ load_ptr(0, rax);
__ push_ptr(rax);
}
void TemplateTable::dup_x1() {
transition(vtos, vtos);
__ load_ptr( 0, rax); // load b
__ load_ptr( 1, rcx); // load a
__ store_ptr(1, rax); // store b
__ store_ptr(0, rcx); // store a
__ push_ptr(rax); // push b
}
void TemplateTable::dup_x2() {
transition(vtos, vtos);
__ load_ptr( 0, rax); // load c
__ load_ptr( 2, rcx); // load a
__ store_ptr(2, rax); // store c in a
__ push_ptr(rax); // push c
__ load_ptr( 2, rax); // load b
__ store_ptr(2, rcx); // store a in b
__ store_ptr(1, rax); // store b in c
}
void TemplateTable::dup2() {
transition(vtos, vtos);
__ load_ptr(1, rax); // load a
__ push_ptr(rax); // push a
__ load_ptr(1, rax); // load b
__ push_ptr(rax); // push b
}
void TemplateTable::dup2_x1() {
transition(vtos, vtos);
__ load_ptr( 0, rcx); // load c
__ load_ptr( 1, rax); // load b
__ push_ptr(rax); // push b
__ push_ptr(rcx); // push c
__ store_ptr(3, rcx); // store c in b
__ load_ptr( 4, rcx); // load a
__ store_ptr(2, rcx); // store a in 2nd c
__ store_ptr(4, rax); // store b in a
}
void TemplateTable::dup2_x2() {
transition(vtos, vtos);
__ load_ptr( 0, rcx); // load d
__ load_ptr( 1, rax); // load c
__ push_ptr(rax); // push c
__ push_ptr(rcx); // push d
__ load_ptr( 4, rax); // load b
__ store_ptr(2, rax); // store b in d
__ store_ptr(4, rcx); // store d in b
__ load_ptr( 5, rcx); // load a
__ load_ptr( 3, rax); // load c
__ store_ptr(3, rcx); // store a in c
__ store_ptr(5, rax); // store c in a
}
void TemplateTable::swap() {
transition(vtos, vtos);
__ load_ptr( 1, rcx); // load a
__ load_ptr( 0, rax); // load b
__ store_ptr(0, rcx); // store a in b
__ store_ptr(1, rax); // store b in a
}
void TemplateTable::iop2(Operation op) {
transition(itos, itos);
switch (op) {
case add : __ pop_i(rdx); __ addl (rax, rdx); break;
case sub : __ mov(rdx, rax); __ pop_i(rax); __ subl (rax, rdx); break;
case mul : __ pop_i(rdx); __ imull(rax, rdx); break;
case _and : __ pop_i(rdx); __ andl (rax, rdx); break;
case _or : __ pop_i(rdx); __ orl (rax, rdx); break;
case _xor : __ pop_i(rdx); __ xorl (rax, rdx); break;
case shl : __ mov(rcx, rax); __ pop_i(rax); __ shll (rax); break; // implicit masking of lower 5 bits by Intel shift instr.
case shr : __ mov(rcx, rax); __ pop_i(rax); __ sarl (rax); break; // implicit masking of lower 5 bits by Intel shift instr.
case ushr : __ mov(rcx, rax); __ pop_i(rax); __ shrl (rax); break; // implicit masking of lower 5 bits by Intel shift instr.
default : ShouldNotReachHere();
}
}
void TemplateTable::lop2(Operation op) {
transition(ltos, ltos);
__ pop_l(rbx, rcx);
switch (op) {
case add : __ addl(rax, rbx); __ adcl(rdx, rcx); break;
case sub : __ subl(rbx, rax); __ sbbl(rcx, rdx);
__ mov (rax, rbx); __ mov (rdx, rcx); break;
case _and : __ andl(rax, rbx); __ andl(rdx, rcx); break;
case _or : __ orl (rax, rbx); __ orl (rdx, rcx); break;
case _xor : __ xorl(rax, rbx); __ xorl(rdx, rcx); break;
default : ShouldNotReachHere();
}
}
void TemplateTable::idiv() {
transition(itos, itos);
__ mov(rcx, rax);
__ pop_i(rax);
__ corrected_idivl(rcx);
}
void TemplateTable::irem() {
transition(itos, itos);
__ mov(rcx, rax);
__ pop_i(rax);
__ corrected_idivl(rcx);
__ mov(rax, rdx);
}
void TemplateTable::lmul() {
transition(ltos, ltos);
__ pop_l(rbx, rcx);
__ push(rcx); __ push(rbx);
__ push(rdx); __ push(rax);
__ lmul(2 * wordSize, 0);
__ addptr(rsp, 4 * wordSize); // take off temporaries
}
void TemplateTable::ldiv() {
transition(ltos, ltos);
__ pop_l(rbx, rcx);
__ push(rcx); __ push(rbx);
__ push(rdx); __ push(rax);
__ orl(rax, rdx);
__ jump_cc(Assembler::zero,
ExternalAddress(Interpreter::_throw_ArithmeticException_entry));
__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::ldiv));
__ addptr(rsp, 4 * wordSize); // take off temporaries
}
void TemplateTable::lrem() {
transition(ltos, ltos);
__ pop_l(rbx, rcx);
__ push(rcx); __ push(rbx);
__ push(rdx); __ push(rax);
__ orl(rax, rdx);
__ jump_cc(Assembler::zero,
ExternalAddress(Interpreter::_throw_ArithmeticException_entry));
__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::lrem));
__ addptr(rsp, 4 * wordSize);
}
void TemplateTable::lshl() {
transition(itos, ltos);
__ movl(rcx, rax); // get shift count
__ pop_l(rax, rdx); // get shift value
__ lshl(rdx, rax);
}
void TemplateTable::lshr() {
transition(itos, ltos);
__ mov(rcx, rax); // get shift count
__ pop_l(rax, rdx); // get shift value
__ lshr(rdx, rax, true);
}
void TemplateTable::lushr() {
transition(itos, ltos);
__ mov(rcx, rax); // get shift count
__ pop_l(rax, rdx); // get shift value
__ lshr(rdx, rax);
}
void TemplateTable::fop2(Operation op) {
transition(ftos, ftos);
switch (op) {
case add: __ fadd_s (at_rsp()); break;
case sub: __ fsubr_s(at_rsp()); break;
case mul: __ fmul_s (at_rsp()); break;
case div: __ fdivr_s(at_rsp()); break;
case rem: __ fld_s (at_rsp()); __ fremr(rax); break;
default : ShouldNotReachHere();
}
__ f2ieee();
__ pop(rax); // pop float thing off
}
void TemplateTable::dop2(Operation op) {
transition(dtos, dtos);
switch (op) {
case add: __ fadd_d (at_rsp()); break;
case sub: __ fsubr_d(at_rsp()); break;
case mul: {
Label L_strict;
Label L_join;
const Address access_flags (rcx, Method::access_flags_offset());
__ get_method(rcx);
__ movl(rcx, access_flags);
__ testl(rcx, JVM_ACC_STRICT);
__ jccb(Assembler::notZero, L_strict);
__ fmul_d (at_rsp());
__ jmpb(L_join);
__ bind(L_strict);
__ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias1()));
__ fmulp();
__ fmul_d (at_rsp());
__ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias2()));
__ fmulp();
__ bind(L_join);
break;
}
case div: {
Label L_strict;
Label L_join;
const Address access_flags (rcx, Method::access_flags_offset());
__ get_method(rcx);
__ movl(rcx, access_flags);
__ testl(rcx, JVM_ACC_STRICT);
__ jccb(Assembler::notZero, L_strict);
__ fdivr_d(at_rsp());
__ jmp(L_join);
__ bind(L_strict);
__ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias1()));
__ fmul_d (at_rsp());
__ fdivrp();
__ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias2()));
__ fmulp();
__ bind(L_join);
break;
}
case rem: __ fld_d (at_rsp()); __ fremr(rax); break;
default : ShouldNotReachHere();
}
__ d2ieee();
__ pop(rax);
__ pop(rdx);
}
void TemplateTable::ineg() {
transition(itos, itos);
__ negl(rax);
}
void TemplateTable::lneg() {
transition(ltos, ltos);
__ lneg(rdx, rax);
}
void TemplateTable::fneg() {
transition(ftos, ftos);
__ fchs();
}
void TemplateTable::dneg() {
transition(dtos, dtos);
__ fchs();
}
void TemplateTable::iinc() {
transition(vtos, vtos);
__ load_signed_byte(rdx, at_bcp(2)); // get constant
locals_index(rbx);
__ addl(iaddress(rbx), rdx);
}
void TemplateTable::wide_iinc() {
transition(vtos, vtos);
__ movl(rdx, at_bcp(4)); // get constant
locals_index_wide(rbx);
__ bswapl(rdx); // swap bytes & sign-extend constant
__ sarl(rdx, 16);
__ addl(iaddress(rbx), rdx);
}
void TemplateTable::convert() {
#ifdef ASSERT
{ TosState tos_in = ilgl;
TosState tos_out = ilgl;
switch (bytecode()) {
case Bytecodes::_i2l: // fall through
case Bytecodes::_i2f: // fall through
case Bytecodes::_i2d: // fall through
case Bytecodes::_i2b: // fall through
case Bytecodes::_i2c: // fall through
case Bytecodes::_i2s: tos_in = itos; break;
case Bytecodes::_l2i: // fall through
case Bytecodes::_l2f: // fall through
case Bytecodes::_l2d: tos_in = ltos; break;
case Bytecodes::_f2i: // fall through
case Bytecodes::_f2l: // fall through
case Bytecodes::_f2d: tos_in = ftos; break;
case Bytecodes::_d2i: // fall through
case Bytecodes::_d2l: // fall through
case Bytecodes::_d2f: tos_in = dtos; break;
default : ShouldNotReachHere();
}
switch (bytecode()) {
case Bytecodes::_l2i: // fall through
case Bytecodes::_f2i: // fall through
case Bytecodes::_d2i: // fall through
case Bytecodes::_i2b: // fall through
case Bytecodes::_i2c: // fall through
case Bytecodes::_i2s: tos_out = itos; break;
case Bytecodes::_i2l: // fall through
case Bytecodes::_f2l: // fall through
case Bytecodes::_d2l: tos_out = ltos; break;
case Bytecodes::_i2f: // fall through
case Bytecodes::_l2f: // fall through
case Bytecodes::_d2f: tos_out = ftos; break;
case Bytecodes::_i2d: // fall through
case Bytecodes::_l2d: // fall through
case Bytecodes::_f2d: tos_out = dtos; break;
default : ShouldNotReachHere();
}
transition(tos_in, tos_out);
}
#endif // ASSERT
switch (bytecode()) {
case Bytecodes::_i2l:
__ extend_sign(rdx, rax);
break;
case Bytecodes::_i2f:
__ push(rax); // store int on tos
__ fild_s(at_rsp()); // load int to ST0
__ f2ieee(); // truncate to float size
__ pop(rcx); // adjust rsp
break;
case Bytecodes::_i2d:
__ push(rax); // add one slot for d2ieee()
__ push(rax); // store int on tos
__ fild_s(at_rsp()); // load int to ST0
__ d2ieee(); // truncate to double size
__ pop(rcx); // adjust rsp
__ pop(rcx);
break;
case Bytecodes::_i2b:
__ shll(rax, 24); // truncate upper 24 bits
__ sarl(rax, 24); // and sign-extend byte
LP64_ONLY(__ movsbl(rax, rax));
break;
case Bytecodes::_i2c:
__ andl(rax, 0xFFFF); // truncate upper 16 bits
LP64_ONLY(__ movzwl(rax, rax));
break;
case Bytecodes::_i2s:
__ shll(rax, 16); // truncate upper 16 bits
__ sarl(rax, 16); // and sign-extend short
LP64_ONLY(__ movswl(rax, rax));
break;
case Bytecodes::_l2i:
break;
case Bytecodes::_l2f:
__ push(rdx); // store long on tos
__ push(rax);
__ fild_d(at_rsp()); // load long to ST0
__ f2ieee(); // truncate to float size
__ pop(rcx); // adjust rsp
__ pop(rcx);
break;
case Bytecodes::_l2d:
__ push(rdx); // store long on tos
__ push(rax);
__ fild_d(at_rsp()); // load long to ST0
__ d2ieee(); // truncate to double size
__ pop(rcx); // adjust rsp
__ pop(rcx);
break;
case Bytecodes::_f2i:
__ push(rcx); // reserve space for argument
__ fstp_s(at_rsp()); // pass float argument on stack
__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2i), 1);
break;
case Bytecodes::_f2l:
__ push(rcx); // reserve space for argument
__ fstp_s(at_rsp()); // pass float argument on stack
__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2l), 1);
break;
case Bytecodes::_f2d:
break;
case Bytecodes::_d2i:
__ push(rcx); // reserve space for argument
__ push(rcx);
__ fstp_d(at_rsp()); // pass double argument on stack
__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2i), 2);
break;
case Bytecodes::_d2l:
__ push(rcx); // reserve space for argument
__ push(rcx);
__ fstp_d(at_rsp()); // pass double argument on stack
__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2l), 2);
break;
case Bytecodes::_d2f:
__ push(rcx); // reserve space for f2ieee()
__ f2ieee(); // truncate to float size
__ pop(rcx); // adjust rsp
break;
default :
ShouldNotReachHere();
}
}
void TemplateTable::lcmp() {
transition(ltos, itos);
__ pop_l(rbx, rcx); // get x = rcx:rbx
__ lcmp2int(rcx, rbx, rdx, rax);// rcx := cmp(x, y)
__ mov(rax, rcx);
}
01-23
183
183
11-06
116
116
11-05
42
42
08-31
519
519
05-08
324
324
11-28
“相关推荐”对你有帮助么？
-
非常没帮助 -
没帮助 -
一般 -
有帮助 -
非常有帮助
提交
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
