cccccccc5

address InterpreterGenerator::generate_math_entry(AbstractInterpreter::MethodKind kind) {
  if (!InlineIntrinsics) return NULL; // Generate a vanilla entry
  address entry_point = __ pc();
  if (kind == Interpreter::java_lang_math_sqrt) {
    __ sqrtsd(xmm0, Address(rsp, wordSize));
  } else {
    __ fld_d(Address(rsp, wordSize));
    switch (kind) {
      case Interpreter::java_lang_math_sin :
          __ trigfunc('s');
          break;
      case Interpreter::java_lang_math_cos :
          __ trigfunc('c');
          break;
      case Interpreter::java_lang_math_tan :
          __ trigfunc('t');
          break;
      case Interpreter::java_lang_math_abs:
          __ fabs();
          break;
      case Interpreter::java_lang_math_log:
          __ flog();
          break;
      case Interpreter::java_lang_math_log10:
          __ flog10();
          break;
      case Interpreter::java_lang_math_pow:
          __ fld_d(Address(rsp, 3*wordSize)); // second argument (one
          __ pow_with_fallback(0);
          break;
      case Interpreter::java_lang_math_exp:
          __ exp_with_fallback(0);
           break;
      default                              :
          ShouldNotReachHere();
    }
    __ subptr(rsp, 2*wordSize);
    __ fstp_d(Address(rsp, 0));
    __ movdbl(xmm0, Address(rsp, 0));
    __ addptr(rsp, 2*wordSize);
  }
  __ pop(rax);
  __ mov(rsp, r13);
  __ jmp(rax);
  return entry_point;
}
address InterpreterGenerator::generate_abstract_entry(void) {
  address entry_point = __ pc();
  __ empty_expression_stack();
  __ restore_bcp();      // rsi must be correct for exception handler   (was destroyed)
  __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
  __ call_VM(noreg, CAST_FROM_FN_PTR(address,
                             InterpreterRuntime::throw_AbstractMethodError));
  __ should_not_reach_here();
  return entry_point;
}
address InterpreterGenerator::generate_empty_entry(void) {
  if (!UseFastEmptyMethods) {
    return NULL;
  }
  address entry_point = __ pc();
  Label slow_path;
  __ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
           SafepointSynchronize::_not_synchronized);
  __ jcc(Assembler::notEqual, slow_path);
  __ pop(rax);
  __ mov(rsp, r13);
  __ jmp(rax);
  __ bind(slow_path);
  (void) generate_normal_entry(false);
  return entry_point;
}
void Deoptimization::unwind_callee_save_values(frame* f, vframeArray* vframe_array) {
  assert(f->is_interpreted_frame(), "must be interpreted");
}
C:\hotspot-69087d08d473\src\cpu\x86\vm/interp_masm_x86.cpp
#include "precompiled.hpp"
#include "interp_masm_x86.hpp"
#include "interpreter/interpreter.hpp"
#include "oops/methodData.hpp"
void InterpreterMacroAssembler::narrow(Register result) {
  movptr(rcx, Address(rbp, frame::interpreter_frame_method_offset * wordSize));
  movptr(rcx, Address(rcx, Method::const_offset()));
  load_unsigned_byte(rcx, Address(rcx, ConstMethod::result_type_offset()));
  Label done, notBool, notByte, notChar;
  cmpl(rcx, T_INT);
  jcc(Assembler::equal, done);
  cmpl(rcx, T_BOOLEAN);
  jcc(Assembler::notEqual, notBool);
  andl(result, 0x1);
  jmp(done);
  bind(notBool);
  cmpl(rcx, T_BYTE);
  jcc(Assembler::notEqual, notByte);
  LP64_ONLY(movsbl(result, result);)
  NOT_LP64(shll(result, 24);)      // truncate upper 24 bits
  NOT_LP64(sarl(result, 24);)      // and sign-extend byte
  jmp(done);
  bind(notByte);
  cmpl(rcx, T_CHAR);
  jcc(Assembler::notEqual, notChar);
  LP64_ONLY(movzwl(result, result);)
  NOT_LP64(andl(result, 0xFFFF);)  // truncate upper 16 bits
  jmp(done);
  bind(notChar);
  LP64_ONLY(movswl(result, result);)
  NOT_LP64(shll(result, 16);)      // truncate upper 16 bits
  NOT_LP64(sarl(result, 16);)      // and sign-extend short
  bind(done);
}
#ifndef CC_INTERP
void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr) {
  Label update, next, none;
  verify_oop(obj);
  testptr(obj, obj);
  jccb(Assembler::notZero, update);
  orptr(mdo_addr, TypeEntries::null_seen);
  jmpb(next);
  bind(update);
  load_klass(obj, obj);
  xorptr(obj, mdo_addr);
  testptr(obj, TypeEntries::type_klass_mask);
  jccb(Assembler::zero, next); // klass seen before, nothing to
  testptr(obj, TypeEntries::type_unknown);
  jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
  cmpptr(mdo_addr, 0);
  jccb(Assembler::equal, none);
  cmpptr(mdo_addr, TypeEntries::null_seen);
  jccb(Assembler::equal, none);
  xorptr(obj, mdo_addr);
  testptr(obj, TypeEntries::type_klass_mask);
  jccb(Assembler::zero, next);
  orptr(mdo_addr, TypeEntries::type_unknown);
  jmpb(next);
  bind(none);
  movptr(mdo_addr, obj);
  bind(next);
}
void InterpreterMacroAssembler::profile_arguments_type(Register mdp, Register callee, Register tmp, bool is_virtual) {
  if (!ProfileInterpreter) {
    return;
  }
  if (MethodData::profile_arguments() || MethodData::profile_return()) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    int off_to_start = is_virtual ? in_bytes(VirtualCallData::virtual_call_data_size()) : in_bytes(CounterData::counter_data_size());
    cmpb(Address(mdp, in_bytes(DataLayout::tag_offset()) - off_to_start), is_virtual ? DataLayout::virtual_call_type_data_tag : DataLayout::call_type_data_tag);
    jcc(Assembler::notEqual, profile_continue);
    if (MethodData::profile_arguments()) {
      Label done;
      int off_to_args = in_bytes(TypeEntriesAtCall::args_data_offset());
      addptr(mdp, off_to_args);
      for (int i = 0; i < TypeProfileArgsLimit; i++) {
        if (i > 0 || MethodData::profile_return()) {
          movptr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())-off_to_args));
          subl(tmp, i*TypeStackSlotEntries::per_arg_count());
          cmpl(tmp, TypeStackSlotEntries::per_arg_count());
          jcc(Assembler::less, done);
        }
        movptr(tmp, Address(callee, Method::const_offset()));
        load_unsigned_short(tmp, Address(tmp, ConstMethod::size_of_parameters_offset()));
        subptr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::stack_slot_offset(i))-off_to_args));
        subl(tmp, 1);
        Address arg_addr = argument_address(tmp);
        movptr(tmp, arg_addr);
        Address mdo_arg_addr(mdp, in_bytes(TypeEntriesAtCall::argument_type_offset(i))-off_to_args);
        profile_obj_type(tmp, mdo_arg_addr);
        int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
        addptr(mdp, to_add);
        off_to_args += to_add;
      }
      if (MethodData::profile_return()) {
        movptr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())-off_to_args));
        subl(tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
      }
      bind(done);
      if (MethodData::profile_return()) {
        assert(ReturnTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
        shll(tmp, exact_log2(DataLayout::cell_size));
        addptr(mdp, tmp);
      }
      movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), mdp);
    } else {
      assert(MethodData::profile_return(), "either profile call args or call ret");
      update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
    }
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
  assert_different_registers(mdp, ret, tmp, _bcp_register);
  if (ProfileInterpreter && MethodData::profile_return()) {
    Label profile_continue, done;
    test_method_data_pointer(mdp, profile_continue);
    if (MethodData::profile_return_jsr292_only()) {
      Label do_profile;
      cmpb(Address(_bcp_register, 0), Bytecodes::_invokedynamic);
      jcc(Assembler::equal, do_profile);
      cmpb(Address(_bcp_register, 0), Bytecodes::_invokehandle);
      jcc(Assembler::equal, do_profile);
      get_method(tmp);
      cmpb(Address(tmp, Method::intrinsic_id_offset_in_bytes()), vmIntrinsics::_compiledLambdaForm);
      jcc(Assembler::notEqual, profile_continue);
      bind(do_profile);
    }
    Address mdo_ret_addr(mdp, -in_bytes(ReturnTypeEntry::size()));
    mov(tmp, ret);
    profile_obj_type(tmp, mdo_ret_addr);
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
  if (ProfileInterpreter && MethodData::profile_parameters()) {
    Label profile_continue, done;
    test_method_data_pointer(mdp, profile_continue);
    movl(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
    testl(tmp1, tmp1);
    jcc(Assembler::negative, profile_continue);
    addptr(mdp, tmp1);
    movptr(tmp1, Address(mdp, ArrayData::array_len_offset()));
    decrement(tmp1, TypeStackSlotEntries::per_arg_count());
    Label loop;
    bind(loop);
    int off_base = in_bytes(ParametersTypeData::stack_slot_offset(0));
    int type_base = in_bytes(ParametersTypeData::type_offset(0));
    Address::ScaleFactor per_arg_scale = Address::times(DataLayout::cell_size);
    Address arg_off(mdp, tmp1, per_arg_scale, off_base);
    Address arg_type(mdp, tmp1, per_arg_scale, type_base);
    movptr(tmp2, arg_off);
    negptr(tmp2);
    movptr(tmp2, Address(_locals_register, tmp2, Interpreter::stackElementScale()));
    profile_obj_type(tmp2, arg_type);
    decrement(tmp1, TypeStackSlotEntries::per_arg_count());
    jcc(Assembler::positive, loop);
    bind(profile_continue);
  }
}
#endif
C:\hotspot-69087d08d473\src\cpu\x86\vm/interp_masm_x86.hpp
#ifndef CPU_X86_VM_INTERP_MASM_X86_HPP
#define CPU_X86_VM_INTERP_MASM_X86_HPP
#include "asm/macroAssembler.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "interpreter/invocationCounter.hpp"
#include "runtime/frame.hpp"
class InterpreterMacroAssembler: public MacroAssembler {
#ifdef TARGET_ARCH_MODEL_x86_32
# include "interp_masm_x86_32.hpp"
#endif
#ifdef TARGET_ARCH_MODEL_x86_64
# include "interp_masm_x86_64.hpp"
#endif
 private:
  Register _locals_register; // register that contains the pointer to the locals
  Register _bcp_register; // register that contains the bcp
 public:
  void narrow(Register result);
#ifndef CC_INTERP
  void profile_obj_type(Register obj, const Address& mdo_addr);
  void profile_arguments_type(Register mdp, Register callee, Register tmp, bool is_virtual);
  void profile_return_type(Register mdp, Register ret, Register tmp);
  void profile_parameters_type(Register mdp, Register tmp1, Register tmp2);
#endif /* !CC_INTERP */
};
#endif // CPU_X86_VM_INTERP_MASM_X86_HPP
C:\hotspot-69087d08d473\src\cpu\x86\vm/interp_masm_x86_32.cpp
#include "precompiled.hpp"
#include "interp_masm_x86.hpp"
#include "interpreter/interpreter.hpp"
#include "interpreter/interpreterRuntime.hpp"
#include "oops/arrayOop.hpp"
#include "oops/markOop.hpp"
#include "oops/methodData.hpp"
#include "oops/method.hpp"
#include "prims/jvmtiExport.hpp"
#include "prims/jvmtiRedefineClassesTrace.hpp"
#include "prims/jvmtiThreadState.hpp"
#include "runtime/basicLock.hpp"
#include "runtime/biasedLocking.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/thread.inline.hpp"
#ifdef CC_INTERP
void InterpreterMacroAssembler::get_method(Register reg) {
  movptr(reg, Address(rbp, -(sizeof(BytecodeInterpreter) + 2 * wordSize)));
  movptr(reg, Address(reg, byte_offset_of(BytecodeInterpreter, _method)));
}
#endif // CC_INTERP
#ifndef CC_INTERP
void InterpreterMacroAssembler::call_VM_leaf_base(
  address entry_point,
  int     number_of_arguments
) {
#ifdef ASSERT
  { Label L;
    cmpptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
    jcc(Assembler::equal, L);
    stop("InterpreterMacroAssembler::call_VM_leaf_base: last_sp != NULL");
    bind(L);
  }
#endif
  MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
}
void InterpreterMacroAssembler::call_VM_base(
  Register oop_result,
  Register java_thread,
  Register last_java_sp,
  address  entry_point,
  int      number_of_arguments,
  bool     check_exceptions
) {
#ifdef ASSERT
  { Label L;
    cmpptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
    jcc(Assembler::equal, L);
    stop("InterpreterMacroAssembler::call_VM_base: last_sp != NULL");
    bind(L);
  }
#endif /* ASSERT */
  assert(java_thread == noreg , "not expecting a precomputed java thread");
  save_bcp();
  MacroAssembler::call_VM_base(oop_result, java_thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
  restore_bcp();
  restore_locals();
}
void InterpreterMacroAssembler::check_and_handle_popframe(Register java_thread) {
  if (JvmtiExport::can_pop_frame()) {
    Label L;
    Register pop_cond = java_thread;  // Not clear if any other register is available...
    movl(pop_cond, Address(java_thread, JavaThread::popframe_condition_offset()));
    testl(pop_cond, JavaThread::popframe_pending_bit);
    jcc(Assembler::zero, L);
    testl(pop_cond, JavaThread::popframe_processing_bit);
    jcc(Assembler::notZero, L);
    call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry));
    jmp(rax);
    bind(L);
    get_thread(java_thread);
  }
}
void InterpreterMacroAssembler::load_earlyret_value(TosState state) {
  get_thread(rcx);
  movl(rcx, Address(rcx, JavaThread::jvmti_thread_state_offset()));
  const Address tos_addr (rcx, JvmtiThreadState::earlyret_tos_offset());
  const Address oop_addr (rcx, JvmtiThreadState::earlyret_oop_offset());
  const Address val_addr (rcx, JvmtiThreadState::earlyret_value_offset());
  const Address val_addr1(rcx, JvmtiThreadState::earlyret_value_offset()
                             + in_ByteSize(wordSize));
  switch (state) {
    case atos: movptr(rax, oop_addr);
               movptr(oop_addr, NULL_WORD);
               verify_oop(rax, state);                break;
    case ltos:
               movl(rdx, val_addr1);               // fall through
    case btos:                                     // fall through
    case ztos:                                     // fall through
    case ctos:                                     // fall through
    case stos:                                     // fall through
    case itos: movl(rax, val_addr);                   break;
    case ftos: fld_s(val_addr);                       break;
    case dtos: fld_d(val_addr);                       break;
    case vtos: /* nothing to do */                    break;
    default  : ShouldNotReachHere();
  }
  movl(tos_addr,  (int32_t) ilgl);
  movptr(val_addr,  NULL_WORD);
  NOT_LP64(movptr(val_addr1, NULL_WORD));
}
void InterpreterMacroAssembler::check_and_handle_earlyret(Register java_thread) {
  if (JvmtiExport::can_force_early_return()) {
    Label L;
    Register tmp = java_thread;
    movptr(tmp, Address(tmp, JavaThread::jvmti_thread_state_offset()));
    testptr(tmp, tmp);
    jcc(Assembler::zero, L); // if (thread->jvmti_thread_state() == NULL) exit;
    movl(tmp, Address(tmp, JvmtiThreadState::earlyret_state_offset()));
    cmpl(tmp, JvmtiThreadState::earlyret_pending);
    jcc(Assembler::notEqual, L);
    get_thread(java_thread);
    movptr(tmp, Address(java_thread, JavaThread::jvmti_thread_state_offset()));
    pushl(Address(tmp, JvmtiThreadState::earlyret_tos_offset()));
    call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), 1);
    jmp(rax);
    bind(L);
    get_thread(java_thread);
  }
}
void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset) {
  assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode");
  load_unsigned_short(reg, Address(rsi, bcp_offset));
  bswapl(reg);
  shrl(reg, 16);
}
void InterpreterMacroAssembler::get_cache_index_at_bcp(Register reg, int bcp_offset, size_t index_size) {
  assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
  if (index_size == sizeof(u2)) {
    load_unsigned_short(reg, Address(rsi, bcp_offset));
  } else if (index_size == sizeof(u4)) {
    assert(EnableInvokeDynamic, "giant index used only for JSR 292");
    movl(reg, Address(rsi, bcp_offset));
    assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line");
    notl(reg);  // convert to plain index
  } else if (index_size == sizeof(u1)) {
    load_unsigned_byte(reg, Address(rsi, bcp_offset));
  } else {
    ShouldNotReachHere();
  }
}
void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register index,
                                                           int bcp_offset, size_t index_size) {
  assert_different_registers(cache, index);
  get_cache_index_at_bcp(index, bcp_offset, index_size);
  movptr(cache, Address(rbp, frame::interpreter_frame_cache_offset * wordSize));
  assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
  assert(exact_log2(in_words(ConstantPoolCacheEntry::size())) == 2, "else change next line");
  shlptr(index, 2); // convert from field index to ConstantPoolCacheEntry index
}
void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache,
                                                                        Register index,
                                                                        Register bytecode,
                                                                        int byte_no,
                                                                        int bcp_offset,
                                                                        size_t index_size) {
  get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size);
  movptr(bytecode, Address(cache, index, Address::times_ptr, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset()));
  const int shift_count = (1 + byte_no) * BitsPerByte;
  assert((byte_no == TemplateTable::f1_byte && shift_count == ConstantPoolCacheEntry::bytecode_1_shift) ||
         (byte_no == TemplateTable::f2_byte && shift_count == ConstantPoolCacheEntry::bytecode_2_shift),
         "correct shift count");
  shrptr(bytecode, shift_count);
  assert(ConstantPoolCacheEntry::bytecode_1_mask == ConstantPoolCacheEntry::bytecode_2_mask, "common mask");
  andptr(bytecode, ConstantPoolCacheEntry::bytecode_1_mask);
}
void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register tmp,
                                                               int bcp_offset, size_t index_size) {
  assert(cache != tmp, "must use different register");
  get_cache_index_at_bcp(tmp, bcp_offset, index_size);
  assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
  assert(exact_log2(in_bytes(ConstantPoolCacheEntry::size_in_bytes())) == 2 + LogBytesPerWord, "else change next line");
  shll(tmp, 2 + LogBytesPerWord);
  movptr(cache, Address(rbp, frame::interpreter_frame_cache_offset * wordSize));
  addptr(cache, in_bytes(ConstantPoolCache::base_offset()));
  addptr(cache, tmp);            // construct pointer to cache entry
}
void InterpreterMacroAssembler::get_method_counters(Register method,
                                                    Register mcs, Label& skip) {
  Label has_counters;
  movptr(mcs, Address(method, Method::method_counters_offset()));
  testptr(mcs, mcs);
  jcc(Assembler::notZero, has_counters);
  call_VM(noreg, CAST_FROM_FN_PTR(address,
          InterpreterRuntime::build_method_counters), method);
  movptr(mcs, Address(method,Method::method_counters_offset()));
  testptr(mcs, mcs);
  jcc(Assembler::zero, skip); // No MethodCounters allocated, OutOfMemory
  bind(has_counters);
}
void InterpreterMacroAssembler::load_resolved_reference_at_index(
                                           Register result, Register index) {
  assert_different_registers(result, index);
  Register tmp = index;  // reuse
  shll(tmp, LogBytesPerHeapOop);
  get_constant_pool(result);
  movptr(result, Address(result, ConstantPool::resolved_references_offset_in_bytes()));
  movptr(result, Address(result, 0));
  addptr(result, tmp);
  load_heap_oop(result, Address(result, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
}
void InterpreterMacroAssembler::gen_subtype_check( Register Rsub_klass, Label &ok_is_subtype ) {
  assert( Rsub_klass != rax, "rax, holds superklass" );
  assert( Rsub_klass != rcx, "used as a temp" );
  assert( Rsub_klass != rdi, "used as a temp, restored from locals" );
  profile_typecheck(rcx, Rsub_klass, rdi); // blows rcx, reloads rdi
  check_klass_subtype(Rsub_klass, rax, rcx, ok_is_subtype); // blows rcx
  profile_typecheck_failed(rcx); // blows rcx
}
void InterpreterMacroAssembler::f2ieee() {
  if (IEEEPrecision) {
    fstp_s(Address(rsp, 0));
    fld_s(Address(rsp, 0));
  }
}
void InterpreterMacroAssembler::d2ieee() {
  if (IEEEPrecision) {
    fstp_d(Address(rsp, 0));
    fld_d(Address(rsp, 0));
  }
}
void InterpreterMacroAssembler::pop_ptr(Register r) {
  pop(r);
}
void InterpreterMacroAssembler::pop_i(Register r) {
  pop(r);
}
void InterpreterMacroAssembler::pop_l(Register lo, Register hi) {
  pop(lo);
  pop(hi);
}
void InterpreterMacroAssembler::pop_f() {
  fld_s(Address(rsp, 0));
  addptr(rsp, 1 * wordSize);
}
void InterpreterMacroAssembler::pop_d() {
  fld_d(Address(rsp, 0));
  addptr(rsp, 2 * wordSize);
}
void InterpreterMacroAssembler::pop(TosState state) {
  switch (state) {
    case atos: pop_ptr(rax);                                 break;
    case btos:                                               // fall through
    case ztos:                                               // fall through
    case ctos:                                               // fall through
    case stos:                                               // fall through
    case itos: pop_i(rax);                                   break;
    case ltos: pop_l(rax, rdx);                              break;
    case ftos: pop_f();                                      break;
    case dtos: pop_d();                                      break;
    case vtos: /* nothing to do */                           break;
    default  : ShouldNotReachHere();
  }
  verify_oop(rax, state);
}
void InterpreterMacroAssembler::push_ptr(Register r) {
  push(r);
}
void InterpreterMacroAssembler::push_i(Register r) {
  push(r);
}
void InterpreterMacroAssembler::push_l(Register lo, Register hi) {
  push(hi);
  push(lo);
}
void InterpreterMacroAssembler::push_f() {
  subptr(rsp, 1 * wordSize);
  fstp_s(Address(rsp, 0));
}
void InterpreterMacroAssembler::push_d(Register r) {
  subptr(rsp, 2 * wordSize);
  fstp_d(Address(rsp, 0));
}
void InterpreterMacroAssembler::push(TosState state) {
  verify_oop(rax, state);
  switch (state) {
    case atos: push_ptr(rax); break;
    case btos:                                               // fall through
    case ztos:                                               // fall through
    case ctos:                                               // fall through
    case stos:                                               // fall through
    case itos: push_i(rax);                                    break;
    case ltos: push_l(rax, rdx);                               break;
    case ftos: push_f();                                       break;
    case dtos: push_d(rax);                                    break;
    case vtos: /* nothing to do */                             break;
    default  : ShouldNotReachHere();
  }
}
void InterpreterMacroAssembler::load_ptr(int n, Register val) {
  movptr(val, Address(rsp, Interpreter::expr_offset_in_bytes(n)));
}
void InterpreterMacroAssembler::store_ptr(int n, Register val) {
  movptr(Address(rsp, Interpreter::expr_offset_in_bytes(n)), val);
}
void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() {
  lea(rsi, Address(rsp, wordSize));
  movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), rsi);
}
void InterpreterMacroAssembler::jump_from_interpreted(Register method, Register temp) {
  prepare_to_jump_from_interpreted();
  if (JvmtiExport::can_post_interpreter_events()) {
    Label run_compiled_code;
    get_thread(temp);
    cmpb(Address(temp, JavaThread::interp_only_mode_offset()), 0);
    jccb(Assembler::zero, run_compiled_code);
    jmp(Address(method, Method::interpreter_entry_offset()));
    bind(run_compiled_code);
  }
  jmp(Address(method, Method::from_interpreted_offset()));
}
void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) {
}
void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) {
  dispatch_next(state, step);
}
void InterpreterMacroAssembler::dispatch_base(TosState state, address* table,
                                              bool verifyoop) {
  verify_FPU(1, state);
  if (VerifyActivationFrameSize) {
    Label L;
    mov(rcx, rbp);
    subptr(rcx, rsp);
    int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize;
    cmpptr(rcx, min_frame_size);
    jcc(Assembler::greaterEqual, L);
    stop("broken stack frame");
    bind(L);
  }
  if (verifyoop) verify_oop(rax, state);
  Address index(noreg, rbx, Address::times_ptr);
  ExternalAddress tbl((address)table);
  ArrayAddress dispatch(tbl, index);
  jump(dispatch);
}
void InterpreterMacroAssembler::dispatch_only(TosState state) {
  dispatch_base(state, Interpreter::dispatch_table(state));
}
void InterpreterMacroAssembler::dispatch_only_normal(TosState state) {
  dispatch_base(state, Interpreter::normal_table(state));
}
void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) {
  dispatch_base(state, Interpreter::normal_table(state), false);
}
void InterpreterMacroAssembler::dispatch_next(TosState state, int step) {
  load_unsigned_byte(rbx, Address(rsi, step));
  increment(rsi, step);
  dispatch_base(state, Interpreter::dispatch_table(state));
}
void InterpreterMacroAssembler::dispatch_via(TosState state, address* table) {
  load_unsigned_byte(rbx, Address(rsi, 0));
  dispatch_base(state, table);
}
void InterpreterMacroAssembler::remove_activation(TosState state, Register ret_addr,
                                                  bool throw_monitor_exception,
                                                  bool install_monitor_exception,
                                                  bool notify_jvmdi) {
  Label unlocked, unlock, no_unlock;
  get_thread(rcx);
  const Address do_not_unlock_if_synchronized(rcx,
    in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
  movbool(rbx, do_not_unlock_if_synchronized);
  mov(rdi,rbx);
  movbool(do_not_unlock_if_synchronized, false); // reset the flag
  movptr(rbx, Address(rbp, frame::interpreter_frame_method_offset * wordSize)); // get method access flags
  movl(rcx, Address(rbx, Method::access_flags_offset()));
  testl(rcx, JVM_ACC_SYNCHRONIZED);
  jcc(Assembler::zero, unlocked);
  mov(rcx,rdi);
  testbool(rcx);
  jcc(Assembler::notZero, no_unlock);
  push(state);                                   // save result
  const Address monitor(rbp, frame::interpreter_frame_initial_sp_offset * wordSize - (int)sizeof(BasicObjectLock));
  lea   (rdx, monitor);                          // address of first monitor
  movptr (rax, Address(rdx, BasicObjectLock::obj_offset_in_bytes()));
  testptr(rax, rax);
  jcc    (Assembler::notZero, unlock);
  pop(state);
  if (throw_monitor_exception) {
    empty_FPU_stack();  // remove possible return value from FPU-stack, otherwise stack could overflow
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
    should_not_reach_here();
  } else {
    if (install_monitor_exception) {
      empty_FPU_stack();  // remove possible return value from FPU-stack, otherwise stack could overflow
      call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
    }
    jmp(unlocked);
  }
  bind(unlock);
  unlock_object(rdx);
  pop(state);
  bind(unlocked);
  {
    Label loop, exception, entry, restart;
    const int entry_size               = frame::interpreter_frame_monitor_size()           * wordSize;
    const Address monitor_block_top(rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
    const Address monitor_block_bot(rbp, frame::interpreter_frame_initial_sp_offset        * wordSize);
    bind(restart);
    movptr(rcx, monitor_block_top);           // points to current entry, starting with top-most entry
    lea(rbx, monitor_block_bot);              // points to word before bottom of monitor block
    jmp(entry);
    bind(exception);
    if (throw_monitor_exception) {
      empty_FPU_stack();  // remove possible return value from FPU-stack, otherwise stack could overflow
      call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
      should_not_reach_here();
    } else {
      push(state);
      mov(rdx, rcx);
      unlock_object(rdx);
      pop(state);
      if (install_monitor_exception) {
        empty_FPU_stack();  // remove possible return value from FPU-stack, otherwise stack could overflow
        call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
      }
      jmp(restart);
    }
    bind(loop);
    cmpptr(Address(rcx, BasicObjectLock::obj_offset_in_bytes()), (int32_t)NULL_WORD);  // check if current entry is used
    jcc(Assembler::notEqual, exception);
    addptr(rcx, entry_size);                     // otherwise advance to next entry
    bind(entry);
    cmpptr(rcx, rbx);                            // check if bottom reached
    jcc(Assembler::notEqual, loop);              // if not at bottom then check this entry
  }
  bind(no_unlock);
  if (notify_jvmdi) {
    notify_method_exit(state, NotifyJVMTI);     // preserve TOSCA
  } else {
    notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
  }
  movptr(rbx, Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize)); // get sender sp
  leave();                                     // remove frame anchor
  pop(ret_addr);                               // get return address
  mov(rsp, rbx);                               // set sp to sender sp
  if (UseSSE) {
    if (state == ftos && UseSSE >= 1) {
      subptr(rsp, wordSize);
      fstp_s(Address(rsp, 0));
      movflt(xmm0, Address(rsp, 0));
      addptr(rsp, wordSize);
    } else if (state == dtos && UseSSE >= 2) {
      subptr(rsp, 2*wordSize);
      fstp_d(Address(rsp, 0));
      movdbl(xmm0, Address(rsp, 0));
      addptr(rsp, 2*wordSize);
    }
  }
}
#endif /* !CC_INTERP */
void InterpreterMacroAssembler::lock_object(Register lock_reg) {
  assert(lock_reg == rdx, "The argument is only for looks. It must be rdx");
  if (UseHeavyMonitors) {
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), lock_reg);
  } else {
    Label done;
    const Register swap_reg = rax;  // Must use rax, for cmpxchg instruction
    const Register obj_reg  = rcx;  // Will contain the oop
    const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
    const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
    const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes();
    Label slow_case;
    movptr(obj_reg, Address(lock_reg, obj_offset));
    if (UseBiasedLocking) {
      biased_locking_enter(lock_reg, obj_reg, swap_reg, noreg, false, done, &slow_case);
    }
    movptr(swap_reg, (int32_t)1);
    orptr(swap_reg, Address(obj_reg, 0));
    movptr(Address(lock_reg, mark_offset), swap_reg);
    assert(lock_offset == 0, "displached header must be first word in BasicObjectLock");
    if (os::is_MP()) {
      lock();
    }
    cmpxchgptr(lock_reg, Address(obj_reg, 0));
    if (PrintBiasedLockingStatistics) {
      cond_inc32(Assembler::zero,
                 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
    }
    jcc(Assembler::zero, done);
    subptr(swap_reg, rsp);
    andptr(swap_reg, 3 - os::vm_page_size());
    movptr(Address(lock_reg, mark_offset), swap_reg);
    if (PrintBiasedLockingStatistics) {
      cond_inc32(Assembler::zero,
                 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
    }
    jcc(Assembler::zero, done);
    bind(slow_case);
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), lock_reg);
    bind(done);
  }
}
void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
  assert(lock_reg == rdx, "The argument is only for looks. It must be rdx");
  if (UseHeavyMonitors) {
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
  } else {
    Label done;
    const Register swap_reg   = rax;  // Must use rax, for cmpxchg instruction
    const Register header_reg = rbx;  // Will contain the old oopMark
    const Register obj_reg    = rcx;  // Will contain the oop
    save_bcp(); // Save in case of exception
    lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
    movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes ()));
    movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), NULL_WORD);
    if (UseBiasedLocking) {
      biased_locking_exit(obj_reg, header_reg, done);
    }
    movptr(header_reg, Address(swap_reg, BasicLock::displaced_header_offset_in_bytes()));
    testptr(header_reg, header_reg);
    jcc(Assembler::zero, done);
    if (os::is_MP()) lock();
    cmpxchgptr(header_reg, Address(obj_reg, 0));
    jcc(Assembler::zero, done);
    movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), obj_reg); // restore obj
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
    bind(done);
    restore_bcp();
  }
}
#ifndef CC_INTERP
void InterpreterMacroAssembler::test_method_data_pointer(Register mdp, Label& zero_continue) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  movptr(mdp, Address(rbp, frame::interpreter_frame_mdx_offset * wordSize));
  testptr(mdp, mdp);
  jcc(Assembler::zero, zero_continue);
}
void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
  assert(ProfileInterpreter, "must be profiling interpreter");
  Label set_mdp;
  push(rax);
  push(rbx);
  get_method(rbx);
  movptr(rax, Address(rbx, in_bytes(Method::method_data_offset())));
  testptr(rax, rax);
  jcc(Assembler::zero, set_mdp);
  call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), rbx, rsi);
  movptr(rbx, Address(rbx, in_bytes(Method::method_data_offset())));
  addptr(rbx, in_bytes(MethodData::data_offset()));
  addptr(rax, rbx);
  bind(set_mdp);
  movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax);
  pop(rbx);
  pop(rax);
}
void InterpreterMacroAssembler::verify_method_data_pointer() {
  assert(ProfileInterpreter, "must be profiling interpreter");
#ifdef ASSERT
  Label verify_continue;
  push(rax);
  push(rbx);
  push(rcx);
  push(rdx);
  test_method_data_pointer(rcx, verify_continue); // If mdp is zero, continue
  get_method(rbx);
  load_unsigned_short(rdx, Address(rcx, in_bytes(DataLayout::bci_offset())));
  addptr(rdx, Address(rbx, Method::const_offset()));
  lea(rdx, Address(rdx, ConstMethod::codes_offset()));
  cmpptr(rdx, rsi);
  jcc(Assembler::equal, verify_continue);
  call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), rbx, rsi, rcx);
  bind(verify_continue);
  pop(rdx);
  pop(rcx);
  pop(rbx);
  pop(rax);
#endif // ASSERT
}
void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in, int constant, Register value) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  Address data(mdp_in, constant);
  movptr(data, value);
}
void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
                                                      int constant,
                                                      bool decrement) {
  Address data(mdp_in, constant);
  increment_mdp_data_at(data, decrement);
}
void InterpreterMacroAssembler::increment_mdp_data_at(Address data,
                                                      bool decrement) {
  assert( DataLayout::counter_increment==1, "flow-free idiom only works with 1" );
  assert(ProfileInterpreter, "must be profiling interpreter");
  if (decrement) {
    addl(data, -DataLayout::counter_increment);
    Label L;
    jcc(Assembler::negative, L);
    addl(data, DataLayout::counter_increment);
    bind(L);
  } else {
    assert(DataLayout::counter_increment == 1,
           "flow-free idiom only works with 1");
    addl(data, DataLayout::counter_increment);
    sbbl(data, 0);
  }
}
void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
                                                      Register reg,
                                                      int constant,
                                                      bool decrement) {
  Address data(mdp_in, reg, Address::times_1, constant);
  increment_mdp_data_at(data, decrement);
}
void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in, int flag_byte_constant) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  int header_offset = in_bytes(DataLayout::header_offset());
  int header_bits = DataLayout::flag_mask_to_header_mask(flag_byte_constant);
  orl(Address(mdp_in, header_offset), header_bits);
}
void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in,
                                                 int offset,
                                                 Register value,
                                                 Register test_value_out,
                                                 Label& not_equal_continue) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  if (test_value_out == noreg) {
    cmpptr(value, Address(mdp_in, offset));
  } else {
    movptr(test_value_out, Address(mdp_in, offset));
    cmpptr(test_value_out, value);
  }
  jcc(Assembler::notEqual, not_equal_continue);
}
void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, int offset_of_disp) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  Address disp_address(mdp_in, offset_of_disp);
  addptr(mdp_in,disp_address);
  movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), mdp_in);
}
void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, Register reg, int offset_of_disp) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  Address disp_address(mdp_in, reg, Address::times_1, offset_of_disp);
  addptr(mdp_in, disp_address);
  movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), mdp_in);
}
void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in, int constant) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  addptr(mdp_in, constant);
  movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), mdp_in);
}
void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  push(return_bci);             // save/restore across call_VM
  call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret), return_bci);
  pop(return_bci);
}
void InterpreterMacroAssembler::profile_taken_branch(Register mdp, Register bumped_count) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    Address data(mdp, in_bytes(JumpData::taken_offset()));
    movl(bumped_count,data);
    assert( DataLayout::counter_increment==1, "flow-free idiom only works with 1" );
    addl(bumped_count, DataLayout::counter_increment);
    sbbl(bumped_count, 0);
    movl(data,bumped_count);    // Store back out
    update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
    bind (profile_continue);
  }
}
void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
    update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
    bind (profile_continue);
  }
}
void InterpreterMacroAssembler::profile_call(Register mdp) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
    update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
    bind (profile_continue);
  }
}
void InterpreterMacroAssembler::profile_final_call(Register mdp) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
    update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
    bind (profile_continue);
  }
}
void InterpreterMacroAssembler::profile_virtual_call(Register receiver, Register mdp,
                                                     Register reg2,
                                                     bool receiver_can_be_null) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    Label skip_receiver_profile;
    if (receiver_can_be_null) {
      Label not_null;
      testptr(receiver, receiver);
      jccb(Assembler::notZero, not_null);
      increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
      jmp(skip_receiver_profile);
      bind(not_null);
    }
    record_klass_in_profile(receiver, mdp, reg2, true);
    bind(skip_receiver_profile);
    update_mdp_by_constant(mdp,
                           in_bytes(VirtualCallData::
                                    virtual_call_data_size()));
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::record_klass_in_profile_helper(
                                        Register receiver, Register mdp,
                                        Register reg2, int start_row,
                                        Label& done, bool is_virtual_call) {
  if (TypeProfileWidth == 0) {
    if (is_virtual_call) {
      increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
    }
    return;
  }
  int last_row = VirtualCallData::row_limit() - 1;
  assert(start_row <= last_row, "must be work left to do");
  for (int row = start_row; row <= last_row; row++) {
    Label next_test;
    bool test_for_null_also = (row == start_row);
    int recvr_offset = in_bytes(VirtualCallData::receiver_offset(row));
    test_mdp_data_at(mdp, recvr_offset, receiver,
                     (test_for_null_also ? reg2 : noreg),
                     next_test);
    int count_offset = in_bytes(VirtualCallData::receiver_count_offset(row));
    increment_mdp_data_at(mdp, count_offset);
    jmp(done);
    bind(next_test);
    if (row == start_row) {
      Label found_null;
      testptr(reg2, reg2);
      if (start_row == last_row) {
        if (is_virtual_call) {
          jccb(Assembler::zero, found_null);
          increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
          jmp(done);
          bind(found_null);
        } else {
          jcc(Assembler::notZero, done);
        }
        break;
      }
      jcc(Assembler::zero, found_null);
      record_klass_in_profile_helper(receiver, mdp, reg2, start_row + 1, done, is_virtual_call);
      bind(found_null);
    }
  }
  int recvr_offset = in_bytes(VirtualCallData::receiver_offset(start_row));
  set_mdp_data_at(mdp, recvr_offset, receiver);
  int count_offset = in_bytes(VirtualCallData::receiver_count_offset(start_row));
  movptr(reg2, (intptr_t)DataLayout::counter_increment);
  set_mdp_data_at(mdp, count_offset, reg2);
  if (start_row > 0) {
    jmp(done);
  }
}
void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
                                                        Register mdp, Register reg2,
                                                        bool is_virtual_call) {
  assert(ProfileInterpreter, "must be profiling");
  Label done;
  record_klass_in_profile_helper(receiver, mdp, reg2, 0, done, is_virtual_call);
  bind (done);
}
void InterpreterMacroAssembler::profile_ret(Register return_bci, Register mdp) {
  if (ProfileInterpreter) {
    Label profile_continue;
    uint row;
    test_method_data_pointer(mdp, profile_continue);
    increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
    for (row = 0; row < RetData::row_limit(); row++) {
      Label next_test;
      test_mdp_data_at(mdp, in_bytes(RetData::bci_offset(row)), return_bci,
                       noreg, next_test);
      increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)));
      update_mdp_by_offset(mdp, in_bytes(RetData::bci_displacement_offset(row)));
      jmp(profile_continue);
      bind(next_test);
    }
    update_mdp_for_ret(return_bci);
    bind (profile_continue);
  }
}
void InterpreterMacroAssembler::profile_null_seen(Register mdp) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
    int mdp_delta = in_bytes(BitData::bit_data_size());
    if (TypeProfileCasts) {
      mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
    }
    update_mdp_by_constant(mdp, mdp_delta);
    bind (profile_continue);
  }
}
void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) {
  if (ProfileInterpreter && TypeProfileCasts) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    int count_offset = in_bytes(CounterData::count_offset());
    count_offset -= in_bytes(VirtualCallData::virtual_call_data_size());
    increment_mdp_data_at(mdp, count_offset, true);
    bind (profile_continue);
  }
}
void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2)
{
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    int mdp_delta = in_bytes(BitData::bit_data_size());
    if (TypeProfileCasts) {
      mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
      record_klass_in_profile(klass, mdp, reg2, false);
      assert(reg2 == rdi, "we know how to fix this blown reg");
      restore_locals();         // Restore EDI
    }
    update_mdp_by_constant(mdp, mdp_delta);
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::profile_switch_default(Register mdp) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    increment_mdp_data_at(mdp, in_bytes(MultiBranchData::default_count_offset()));
    update_mdp_by_offset(mdp, in_bytes(MultiBranchData::default_displacement_offset()));
    bind (profile_continue);
  }
}
void InterpreterMacroAssembler::profile_switch_case(Register index, Register mdp, Register reg2) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    movptr(reg2, (intptr_t)in_bytes(MultiBranchData::per_case_size()));
    imulptr(index, reg2);
    addptr(index, in_bytes(MultiBranchData::case_array_offset()));
    increment_mdp_data_at(mdp, index, in_bytes(MultiBranchData::relative_count_offset()));
    update_mdp_by_offset(mdp, index, in_bytes(MultiBranchData::relative_displacement_offset()));
    bind (profile_continue);
  }
}
#endif // !CC_INTERP
void InterpreterMacroAssembler::verify_oop(Register reg, TosState state) {
  if (state == atos) MacroAssembler::verify_oop(reg);
}
#ifndef CC_INTERP
void InterpreterMacroAssembler::verify_FPU(int stack_depth, TosState state) {
  if (state == ftos || state == dtos) MacroAssembler::verify_FPU(stack_depth);
}
#endif /* CC_INTERP */
void InterpreterMacroAssembler::notify_method_entry() {
  if (JvmtiExport::can_post_interpreter_events()) {
    Label L;
    get_thread(rcx);
    movl(rcx, Address(rcx, JavaThread::interp_only_mode_offset()));
    testl(rcx,rcx);
    jcc(Assembler::zero, L);
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_entry));
    bind(L);
  }
  {
    SkipIfEqual skip_if(this, &DTraceMethodProbes, 0);
    get_thread(rcx);
    get_method(rbx);
    call_VM_leaf(
      CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), rcx, rbx);
  }
  if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) {
    get_thread(rcx);
    get_method(rbx);
    call_VM_leaf(
      CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
      rcx, rbx);
  }
}
void InterpreterMacroAssembler::notify_method_exit(
    TosState state, NotifyMethodExitMode mode) {
  if (mode == NotifyJVMTI && JvmtiExport::can_post_interpreter_events()) {
    Label L;
    NOT_CC_INTERP(push(state);)
    get_thread(rcx);
    movl(rcx, Address(rcx, JavaThread::interp_only_mode_offset()));
    testl(rcx,rcx);
    jcc(Assembler::zero, L);
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
    bind(L);
    NOT_CC_INTERP(pop(state);)
  }
  {
    SkipIfEqual skip_if(this, &DTraceMethodProbes, 0);
    NOT_CC_INTERP(push(state));
    get_thread(rbx);
    get_method(rcx);
    call_VM_leaf(
      CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
      rbx, rcx);
    NOT_CC_INTERP(pop(state));
  }
}
void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr,
                                                        int increment, int mask,
                                                        Register scratch, bool preloaded,
                                                        Condition cond, Label* where) {
  if (!preloaded) {
    movl(scratch, counter_addr);
  }
  incrementl(scratch, increment);
  movl(counter_addr, scratch);
  andl(scratch, mask);
  if (where != NULL) {
    jcc(cond, *where);
  }
}
C:\hotspot-69087d08d473\src\cpu\x86\vm/interp_masm_x86_32.hpp
#ifndef CC_INTERP
 protected:
  virtual void call_VM_leaf_base(
    address entry_point,
    int     number_of_arguments
  );
  virtual void call_VM_base(
    Register oop_result,
    Register java_thread,
    Register last_java_sp,
    address  entry_point,
    int      number_of_arguments,
    bool     check_exceptions
  );
  virtual void check_and_handle_popframe(Register java_thread);
  virtual void check_and_handle_earlyret(Register java_thread);
  void dispatch_base(TosState state, address* table, bool verifyoop = true);
#endif /* CC_INTERP */
 public:
  InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code), _locals_register(rdi), _bcp_register(rsi) {}
  void load_earlyret_value(TosState state);
#ifdef CC_INTERP
  void save_bcp()                                          { /*  not needed in c++ interpreter and harmless */ }
  void restore_bcp()                                       { /*  not needed in c++ interpreter and harmless */ }
  void get_method(Register reg);
#else
  void save_bcp()                                          { movptr(Address(rbp, frame::interpreter_frame_bcx_offset * wordSize), rsi); }
  void restore_bcp()                                       { movptr(rsi, Address(rbp, frame::interpreter_frame_bcx_offset * wordSize)); }
  void restore_locals()                                    { movptr(rdi, Address(rbp, frame::interpreter_frame_locals_offset * wordSize)); }
  void get_method(Register reg)                            { movptr(reg, Address(rbp, frame::interpreter_frame_method_offset * wordSize)); }
  void get_const(Register reg)                             { get_method(reg); movptr(reg, Address(reg, Method::const_offset())); }
  void get_constant_pool(Register reg)                     { get_const(reg); movptr(reg, Address(reg, ConstMethod::constants_offset())); }
  void get_constant_pool_cache(Register reg)               { get_constant_pool(reg); movptr(reg, Address(reg, ConstantPool::cache_offset_in_bytes())); }
  void get_cpool_and_tags(Register cpool, Register tags)   { get_constant_pool(cpool); movptr(tags, Address(cpool, ConstantPool::tags_offset_in_bytes()));
  }
  void get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset);
  void get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size = sizeof(u2));
  void get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size = sizeof(u2));
  void get_cache_entry_pointer_at_bcp(Register cache, Register tmp, int bcp_offset, size_t index_size = sizeof(u2));
  void get_cache_index_at_bcp(Register index, int bcp_offset, size_t index_size = sizeof(u2));
  void get_method_counters(Register method, Register mcs, Label& skip);
  void load_resolved_reference_at_index(Register result, Register index);
  void f2ieee();                                           // truncate ftos to 32bits
  void d2ieee();                                           // truncate dtos to 64bits
  void pop_ptr(Register r = rax);
  void pop_i(Register r = rax);
  void pop_l(Register lo = rax, Register hi = rdx);
  void pop_f();
  void pop_d();
  void push_ptr(Register r = rax);
  void push_i(Register r = rax);
  void push_l(Register lo = rax, Register hi = rdx);
  void push_d(Register r = rax);
  void push_f();
  void pop(TosState state);        // transition vtos -> state
  void push(TosState state);       // transition state -> vtos
  void pop(Register r ) { ((MacroAssembler*)this)->pop(r); }
  void push(Register r ) { ((MacroAssembler*)this)->push(r); }
  void push(int32_t imm ) { ((MacroAssembler*)this)->push(imm); }
  void pop(void* v ); // Add unimplemented ambiguous method
  void push(void* v );   // Add unimplemented ambiguous method
  void empty_expression_stack() {
    movptr(rsp, Address(rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize));
    movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
  }
  void load_ptr(int n, Register val);
  void store_ptr(int n, Register val);
  void gen_subtype_check( Register sub_klass, Label &ok_is_subtype );
  void dispatch_prolog(TosState state, int step = 0);
  void dispatch_epilog(TosState state, int step = 0);
  void dispatch_only(TosState state);                      // dispatch via rbx, (assume rbx, is loaded already)
  void dispatch_only_normal(TosState state);               // dispatch normal table via rbx, (assume rbx, is loaded already)
  void dispatch_only_noverify(TosState state);
  void dispatch_next(TosState state, int step = 0);        // load rbx, from [esi + step] and dispatch via rbx,
  void dispatch_via (TosState state, address* table);      // load rbx, from [esi] and dispatch via rbx, and table
  void prepare_to_jump_from_interpreted();
  void jump_from_interpreted(Register method, Register temp);
  void remove_activation(TosState state, Register ret_addr,
                         bool throw_monitor_exception = true,
                         bool install_monitor_exception = true,
                         bool notify_jvmdi = true);
#endif /* !CC_INTERP */
  void verify_oop(Register reg, TosState state = atos);    // only if +VerifyOops && state == atos
#ifndef CC_INTERP
  void verify_FPU(int stack_depth, TosState state = ftos); // only if +VerifyFPU  && (state == ftos || state == dtos)
#endif /* !CC_INTERP */
  void lock_object  (Register lock_reg);
  void unlock_object(Register lock_reg);
#ifndef CC_INTERP
  void set_method_data_pointer_for_bcp();
  void test_method_data_pointer(Register mdp, Label& zero_continue);
  void verify_method_data_pointer();
  void set_mdp_data_at(Register mdp_in, int constant, Register value);
  void increment_mdp_data_at(Address data, bool decrement = false);
  void increment_mdp_data_at(Register mdp_in, int constant,
                             bool decrement = false);
  void increment_mdp_data_at(Register mdp_in, Register reg, int constant,
                             bool decrement = false);
  void increment_mask_and_jump(Address counter_addr,
                               int increment, int mask,
                               Register scratch, bool preloaded,
                               Condition cond, Label* where);
  void set_mdp_flag_at(Register mdp_in, int flag_constant);
  void test_mdp_data_at(Register mdp_in, int offset, Register value,
                        Register test_value_out,
                        Label& not_equal_continue);
  void record_klass_in_profile(Register receiver, Register mdp,
                               Register reg2, bool is_virtual_call);
  void record_klass_in_profile_helper(Register receiver, Register mdp,
                                      Register reg2, int start_row,
                                      Label& done, bool is_virtual_call);
  void update_mdp_by_offset(Register mdp_in, int offset_of_offset);
  void update_mdp_by_offset(Register mdp_in, Register reg, int offset_of_disp);
  void update_mdp_by_constant(Register mdp_in, int constant);
  void update_mdp_for_ret(Register return_bci);
  void profile_taken_branch(Register mdp, Register bumped_count);
  void profile_not_taken_branch(Register mdp);
  void profile_call(Register mdp);
  void profile_final_call(Register mdp);
  void profile_virtual_call(Register receiver, Register mdp, Register scratch2,
                            bool receiver_can_be_null = false);
  void profile_ret(Register return_bci, Register mdp);
  void profile_null_seen(Register mdp);
  void profile_typecheck(Register mdp, Register klass, Register scratch);
  void profile_typecheck_failed(Register mdp);
  void profile_switch_default(Register mdp);
  void profile_switch_case(Register index_in_scratch, Register mdp, Register scratch2);
#endif /* !CC_INTERP */
  typedef enum { NotifyJVMTI, SkipNotifyJVMTI } NotifyMethodExitMode;
  void notify_method_entry();
  void notify_method_exit(TosState state, NotifyMethodExitMode mode);
C:\hotspot-69087d08d473\src\cpu\x86\vm/interp_masm_x86_64.cpp
#include "precompiled.hpp"
#include "interp_masm_x86.hpp"
#include "interpreter/interpreter.hpp"
#include "interpreter/interpreterRuntime.hpp"
#include "oops/arrayOop.hpp"
#include "oops/markOop.hpp"
#include "oops/methodData.hpp"
#include "oops/method.hpp"
#include "prims/jvmtiExport.hpp"
#include "prims/jvmtiRedefineClassesTrace.hpp"
#include "prims/jvmtiThreadState.hpp"
#include "runtime/basicLock.hpp"
#include "runtime/biasedLocking.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/thread.inline.hpp"
#ifdef CC_INTERP
void InterpreterMacroAssembler::get_method(Register reg) {
  movptr(reg, Address(rbp, -((int)sizeof(BytecodeInterpreter) + 2 * wordSize)));
  movptr(reg, Address(reg, byte_offset_of(BytecodeInterpreter, _method)));
}
#endif // CC_INTERP
#ifndef CC_INTERP
void InterpreterMacroAssembler::call_VM_leaf_base(address entry_point,
                                                  int number_of_arguments) {
#ifdef ASSERT
  {
    Label L;
    cmpptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
    jcc(Assembler::equal, L);
    stop("InterpreterMacroAssembler::call_VM_leaf_base:"
         " last_sp != NULL");
    bind(L);
  }
#endif
  MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
}
void InterpreterMacroAssembler::call_VM_base(Register oop_result,
                                             Register java_thread,
                                             Register last_java_sp,
                                             address  entry_point,
                                             int      number_of_arguments,
                                             bool     check_exceptions) {
  save_bcp();
#ifdef ASSERT
  {
    Label L;
    cmpptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
    jcc(Assembler::equal, L);
    stop("InterpreterMacroAssembler::call_VM_leaf_base:"
         " last_sp != NULL");
    bind(L);
  }
#endif /* ASSERT */
  MacroAssembler::call_VM_base(oop_result, noreg, last_java_sp,
                               entry_point, number_of_arguments,
                               check_exceptions);
  restore_bcp();
  restore_locals();
}
void InterpreterMacroAssembler::check_and_handle_popframe(Register java_thread) {
  if (JvmtiExport::can_pop_frame()) {
    Label L;
    movl(c_rarg0, Address(r15_thread, JavaThread::popframe_condition_offset()));
    testl(c_rarg0, JavaThread::popframe_pending_bit);
    jcc(Assembler::zero, L);
    testl(c_rarg0, JavaThread::popframe_processing_bit);
    jcc(Assembler::notZero, L);
    call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry));
    jmp(rax);
    bind(L);
  }
}
void InterpreterMacroAssembler::load_earlyret_value(TosState state) {
  movptr(rcx, Address(r15_thread, JavaThread::jvmti_thread_state_offset()));
  const Address tos_addr(rcx, JvmtiThreadState::earlyret_tos_offset());
  const Address oop_addr(rcx, JvmtiThreadState::earlyret_oop_offset());
  const Address val_addr(rcx, JvmtiThreadState::earlyret_value_offset());
  switch (state) {
    case atos: movptr(rax, oop_addr);
               movptr(oop_addr, (int32_t)NULL_WORD);
               verify_oop(rax, state);              break;
    case ltos: movptr(rax, val_addr);                 break;
    case btos:                                   // fall through
    case ztos:                                   // fall through
    case ctos:                                   // fall through
    case stos:                                   // fall through
    case itos: movl(rax, val_addr);                 break;
    case ftos: movflt(xmm0, val_addr);              break;
    case dtos: movdbl(xmm0, val_addr);              break;
    case vtos: /* nothing to do */                  break;
    default  : ShouldNotReachHere();
  }
  movl(tos_addr,  (int) ilgl);
  movl(val_addr,  (int32_t) NULL_WORD);
}
void InterpreterMacroAssembler::check_and_handle_earlyret(Register java_thread) {
  if (JvmtiExport::can_force_early_return()) {
    Label L;
    movptr(c_rarg0, Address(r15_thread, JavaThread::jvmti_thread_state_offset()));
    testptr(c_rarg0, c_rarg0);
    jcc(Assembler::zero, L); // if (thread->jvmti_thread_state() == NULL) exit;
    movl(c_rarg0, Address(c_rarg0, JvmtiThreadState::earlyret_state_offset()));
    cmpl(c_rarg0, JvmtiThreadState::earlyret_pending);
    jcc(Assembler::notEqual, L);
    movptr(c_rarg0, Address(r15_thread, JavaThread::jvmti_thread_state_offset()));
    movl(c_rarg0, Address(c_rarg0, JvmtiThreadState::earlyret_tos_offset()));
    call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), c_rarg0);
    jmp(rax);
    bind(L);
  }
}
void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(
  Register reg,
  int bcp_offset) {
  assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode");
  load_unsigned_short(reg, Address(r13, bcp_offset));
  bswapl(reg);
  shrl(reg, 16);
}
void InterpreterMacroAssembler::get_cache_index_at_bcp(Register index,
                                                       int bcp_offset,
                                                       size_t index_size) {
  assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
  if (index_size == sizeof(u2)) {
    load_unsigned_short(index, Address(r13, bcp_offset));
  } else if (index_size == sizeof(u4)) {
    assert(EnableInvokeDynamic, "giant index used only for JSR 292");
    movl(index, Address(r13, bcp_offset));
    assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line");
    notl(index);  // convert to plain index
  } else if (index_size == sizeof(u1)) {
    load_unsigned_byte(index, Address(r13, bcp_offset));
  } else {
    ShouldNotReachHere();
  }
}
void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache,
                                                           Register index,
                                                           int bcp_offset,
                                                           size_t index_size) {
  assert_different_registers(cache, index);
  get_cache_index_at_bcp(index, bcp_offset, index_size);
  movptr(cache, Address(rbp, frame::interpreter_frame_cache_offset * wordSize));
  assert(sizeof(ConstantPoolCacheEntry) == 4 * wordSize, "adjust code below");
  assert(exact_log2(in_words(ConstantPoolCacheEntry::size())) == 2, "else change next line");
  shll(index, 2);
}
void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache,
                                                                        Register index,
                                                                        Register bytecode,
                                                                        int byte_no,
                                                                        int bcp_offset,
                                                                        size_t index_size) {
  get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size);
  movl(bytecode, Address(cache, index, Address::times_ptr, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset()));
  const int shift_count = (1 + byte_no) * BitsPerByte;
  assert((byte_no == TemplateTable::f1_byte && shift_count == ConstantPoolCacheEntry::bytecode_1_shift) ||
         (byte_no == TemplateTable::f2_byte && shift_count == ConstantPoolCacheEntry::bytecode_2_shift),
         "correct shift count");
  shrl(bytecode, shift_count);
  assert(ConstantPoolCacheEntry::bytecode_1_mask == ConstantPoolCacheEntry::bytecode_2_mask, "common mask");
  andl(bytecode, ConstantPoolCacheEntry::bytecode_1_mask);
}
void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache,
                                                               Register tmp,
                                                               int bcp_offset,
                                                               size_t index_size) {
  assert(cache != tmp, "must use different register");
  get_cache_index_at_bcp(tmp, bcp_offset, index_size);
  assert(sizeof(ConstantPoolCacheEntry) == 4 * wordSize, "adjust code below");
  assert(exact_log2(in_bytes(ConstantPoolCacheEntry::size_in_bytes())) == 2 + LogBytesPerWord, "else change next line");
  shll(tmp, 2 + LogBytesPerWord);
  movptr(cache, Address(rbp, frame::interpreter_frame_cache_offset * wordSize));
  addptr(cache, in_bytes(ConstantPoolCache::base_offset()));
  addptr(cache, tmp);  // construct pointer to cache entry
}
void InterpreterMacroAssembler::get_method_counters(Register method,
                                                    Register mcs, Label& skip) {
  Label has_counters;
  movptr(mcs, Address(method, Method::method_counters_offset()));
  testptr(mcs, mcs);
  jcc(Assembler::notZero, has_counters);
  call_VM(noreg, CAST_FROM_FN_PTR(address,
          InterpreterRuntime::build_method_counters), method);
  movptr(mcs, Address(method,Method::method_counters_offset()));
  testptr(mcs, mcs);
  jcc(Assembler::zero, skip); // No MethodCounters allocated, OutOfMemory
  bind(has_counters);
}
void InterpreterMacroAssembler::load_resolved_reference_at_index(
                                           Register result, Register index) {
  assert_different_registers(result, index);
  Register tmp = index;  // reuse
  shll(tmp, LogBytesPerHeapOop);
  get_constant_pool(result);
  movptr(result, Address(result, ConstantPool::resolved_references_offset_in_bytes()));
  movptr(result, Address(result, 0));
  addptr(result, tmp);
  load_heap_oop(result, Address(result, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
}
void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
                                                  Label& ok_is_subtype) {
  assert(Rsub_klass != rax, "rax holds superklass");
  assert(Rsub_klass != r14, "r14 holds locals");
  assert(Rsub_klass != r13, "r13 holds bcp");
  assert(Rsub_klass != rcx, "rcx holds 2ndary super array length");
  assert(Rsub_klass != rdi, "rdi holds 2ndary super array scan ptr");
  profile_typecheck(rcx, Rsub_klass, rdi); // blows rcx, reloads rdi
  check_klass_subtype(Rsub_klass, rax, rcx, ok_is_subtype); // blows rcx
  profile_typecheck_failed(rcx); // blows rcx
}
void InterpreterMacroAssembler::pop_ptr(Register r) {
  pop(r);
}
void InterpreterMacroAssembler::pop_i(Register r) {
  movl(r, Address(rsp, 0));
  addptr(rsp, wordSize);
}
void InterpreterMacroAssembler::pop_l(Register r) {
  movq(r, Address(rsp, 0));
  addptr(rsp, 2 * Interpreter::stackElementSize);
}
void InterpreterMacroAssembler::pop_f(XMMRegister r) {
  movflt(r, Address(rsp, 0));
  addptr(rsp, wordSize);
}
void InterpreterMacroAssembler::pop_d(XMMRegister r) {
  movdbl(r, Address(rsp, 0));
  addptr(rsp, 2 * Interpreter::stackElementSize);
}
void InterpreterMacroAssembler::push_ptr(Register r) {
  push(r);
}
void InterpreterMacroAssembler::push_i(Register r) {
  push(r);
}
void InterpreterMacroAssembler::push_l(Register r) {
  subptr(rsp, 2 * wordSize);
  movq(Address(rsp, 0), r);
}
void InterpreterMacroAssembler::push_f(XMMRegister r) {
  subptr(rsp, wordSize);
  movflt(Address(rsp, 0), r);
}
void InterpreterMacroAssembler::push_d(XMMRegister r) {
  subptr(rsp, 2 * wordSize);
  movdbl(Address(rsp, 0), r);
}
void InterpreterMacroAssembler::pop(TosState state) {
  switch (state) {
  case atos: pop_ptr();                 break;
  case btos:
  case ztos:
  case ctos:
  case stos:
  case itos: pop_i();                   break;
  case ltos: pop_l();                   break;
  case ftos: pop_f();                   break;
  case dtos: pop_d();                   break;
  case vtos: /* nothing to do */        break;
  default:   ShouldNotReachHere();
  }
  verify_oop(rax, state);
}
void InterpreterMacroAssembler::push(TosState state) {
  verify_oop(rax, state);
  switch (state) {
  case atos: push_ptr();                break;
  case btos:
  case ztos:
  case ctos:
  case stos:
  case itos: push_i();                  break;
  case ltos: push_l();                  break;
  case ftos: push_f();                  break;
  case dtos: push_d();                  break;
  case vtos: /* nothing to do */        break;
  default  : ShouldNotReachHere();
  }
}
void InterpreterMacroAssembler::load_ptr(int n, Register val) {
  movptr(val, Address(rsp, Interpreter::expr_offset_in_bytes(n)));
}
void InterpreterMacroAssembler::store_ptr(int n, Register val) {
  movptr(Address(rsp, Interpreter::expr_offset_in_bytes(n)), val);
}
void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() {
  lea(r13, Address(rsp, wordSize));
  movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), r13);
}
void InterpreterMacroAssembler::jump_from_interpreted(Register method, Register temp) {
  prepare_to_jump_from_interpreted();
  if (JvmtiExport::can_post_interpreter_events()) {
    Label run_compiled_code;
    cmpb(Address(r15_thread, JavaThread::interp_only_mode_offset()), 0);
    jccb(Assembler::zero, run_compiled_code);
    jmp(Address(method, Method::interpreter_entry_offset()));
    bind(run_compiled_code);
  }
  jmp(Address(method, Method::from_interpreted_offset()));
}
void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) {
}
void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) {
  dispatch_next(state, step);
}
void InterpreterMacroAssembler::dispatch_base(TosState state,
                                              address* table,
                                              bool verifyoop) {
  verify_FPU(1, state);
  if (VerifyActivationFrameSize) {
    Label L;
    mov(rcx, rbp);
    subptr(rcx, rsp);
    int32_t min_frame_size =
      (frame::link_offset - frame::interpreter_frame_initial_sp_offset) *
      wordSize;
    cmpptr(rcx, (int32_t)min_frame_size);
    jcc(Assembler::greaterEqual, L);
    stop("broken stack frame");
    bind(L);
  }
  if (verifyoop) {
    verify_oop(rax, state);
  }
  lea(rscratch1, ExternalAddress((address)table));
  jmp(Address(rscratch1, rbx, Address::times_8));
}
void InterpreterMacroAssembler::dispatch_only(TosState state) {
  dispatch_base(state, Interpreter::dispatch_table(state));
}
void InterpreterMacroAssembler::dispatch_only_normal(TosState state) {
  dispatch_base(state, Interpreter::normal_table(state));
}
void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) {
  dispatch_base(state, Interpreter::normal_table(state), false);
}
void InterpreterMacroAssembler::dispatch_next(TosState state, int step) {
  load_unsigned_byte(rbx, Address(r13, step));
  increment(r13, step);
  dispatch_base(state, Interpreter::dispatch_table(state));
}
void InterpreterMacroAssembler::dispatch_via(TosState state, address* table) {
  load_unsigned_byte(rbx, Address(r13, 0));
  dispatch_base(state, table);
}
void InterpreterMacroAssembler::remove_activation(
        TosState state,
        Register ret_addr,
        bool throw_monitor_exception,
        bool install_monitor_exception,
        bool notify_jvmdi) {
  Label unlocked, unlock, no_unlock;
  const Address do_not_unlock_if_synchronized(r15_thread,
    in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
  movbool(rdx, do_not_unlock_if_synchronized);
  movbool(do_not_unlock_if_synchronized, false); // reset the flag
  movptr(rbx, Address(rbp, frame::interpreter_frame_method_offset * wordSize));
  movl(rcx, Address(rbx, Method::access_flags_offset()));
  testl(rcx, JVM_ACC_SYNCHRONIZED);
  jcc(Assembler::zero, unlocked);
  testbool(rdx);
  jcc(Assembler::notZero, no_unlock);
  push(state); // save result
  const Address monitor(rbp, frame::interpreter_frame_initial_sp_offset *
                        wordSize - (int) sizeof(BasicObjectLock));
  lea(c_rarg1, monitor); // address of first monitor
  movptr(rax, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
  testptr(rax, rax);
  jcc(Assembler::notZero, unlock);
  pop(state);
  if (throw_monitor_exception) {
    call_VM(noreg, CAST_FROM_FN_PTR(address,
                   InterpreterRuntime::throw_illegal_monitor_state_exception));
    should_not_reach_here();
  } else {
    if (install_monitor_exception) {
      call_VM(noreg, CAST_FROM_FN_PTR(address,
                     InterpreterRuntime::new_illegal_monitor_state_exception));
    }
    jmp(unlocked);
  }
  bind(unlock);
  unlock_object(c_rarg1);
  pop(state);
  bind(unlocked);
  {
    Label loop, exception, entry, restart;
    const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
    const Address monitor_block_top(
        rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
    const Address monitor_block_bot(
        rbp, frame::interpreter_frame_initial_sp_offset * wordSize);
    bind(restart);
    movptr(c_rarg1, monitor_block_top); // points to current entry, starting
    lea(rbx, monitor_block_bot);  // points to word before bottom of
    jmp(entry);
    bind(exception);
    if (throw_monitor_exception) {
      MacroAssembler::call_VM(noreg,
                              CAST_FROM_FN_PTR(address, InterpreterRuntime::
                                   throw_illegal_monitor_state_exception));
      should_not_reach_here();
    } else {
      push(state);
      unlock_object(c_rarg1);
      pop(state);
      if (install_monitor_exception) {
        call_VM(noreg, CAST_FROM_FN_PTR(address,
                                        InterpreterRuntime::
                                        new_illegal_monitor_state_exception));
      }
      jmp(restart);
    }
    bind(loop);
    cmpptr(Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()), (int32_t) NULL);
    jcc(Assembler::notEqual, exception);
    addptr(c_rarg1, entry_size); // otherwise advance to next entry
    bind(entry);
    cmpptr(c_rarg1, rbx); // check if bottom reached
    jcc(Assembler::notEqual, loop); // if not at bottom then check this entry
  }
  bind(no_unlock);
  if (notify_jvmdi) {
    notify_method_exit(state, NotifyJVMTI);    // preserve TOSCA
  } else {
    notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
  }
  movptr(rbx,
         Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize));
  leave();                           // remove frame anchor
  pop(ret_addr);                     // get return address
  mov(rsp, rbx);                     // set sp to sender sp
}
#endif // C_INTERP
void InterpreterMacroAssembler::lock_object(Register lock_reg) {
  assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1");
  if (UseHeavyMonitors) {
    call_VM(noreg,
            CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
            lock_reg);
  } else {
    Label done;
    const Register swap_reg = rax; // Must use rax for cmpxchg instruction
    const Register obj_reg = c_rarg3; // Will contain the oop
    const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
    const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
    const int mark_offset = lock_offset +
                            BasicLock::displaced_header_offset_in_bytes();
    Label slow_case;
    movptr(obj_reg, Address(lock_reg, obj_offset));
    if (UseBiasedLocking) {
      biased_locking_enter(lock_reg, obj_reg, swap_reg, rscratch1, false, done, &slow_case);
    }
    movl(swap_reg, 1);
    orptr(swap_reg, Address(obj_reg, 0));
    movptr(Address(lock_reg, mark_offset), swap_reg);
    assert(lock_offset == 0,
           "displached header must be first word in BasicObjectLock");
    if (os::is_MP()) lock();
    cmpxchgptr(lock_reg, Address(obj_reg, 0));
    if (PrintBiasedLockingStatistics) {
      cond_inc32(Assembler::zero,
                 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
    }
    jcc(Assembler::zero, done);
    subptr(swap_reg, rsp);
    andptr(swap_reg, 7 - os::vm_page_size());
    movptr(Address(lock_reg, mark_offset), swap_reg);
    if (PrintBiasedLockingStatistics) {
      cond_inc32(Assembler::zero,
                 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
    }
    jcc(Assembler::zero, done);
    bind(slow_case);
    call_VM(noreg,
            CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
            lock_reg);
    bind(done);
  }
}
void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
  assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
  if (UseHeavyMonitors) {
    call_VM(noreg,
            CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit),
            lock_reg);
  } else {
    Label done;
    const Register swap_reg   = rax;  // Must use rax for cmpxchg instruction
    const Register header_reg = c_rarg2;  // Will contain the old oopMark
    const Register obj_reg    = c_rarg3;  // Will contain the oop
    save_bcp(); // Save in case of exception
    lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
    movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
    movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), (int32_t)NULL_WORD);
    if (UseBiasedLocking) {
      biased_locking_exit(obj_reg, header_reg, done);
    }
    movptr(header_reg, Address(swap_reg,
                               BasicLock::displaced_header_offset_in_bytes()));
    testptr(header_reg, header_reg);
    jcc(Assembler::zero, done);
    if (os::is_MP()) lock();
    cmpxchgptr(header_reg, Address(obj_reg, 0));
    jcc(Assembler::zero, done);
    movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()),
         obj_reg); // restore obj
    call_VM(noreg,
            CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit),
            lock_reg);
    bind(done);
    restore_bcp();
  }
}
#ifndef CC_INTERP
void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
                                                         Label& zero_continue) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  movptr(mdp, Address(rbp, frame::interpreter_frame_mdx_offset * wordSize));
  testptr(mdp, mdp);
  jcc(Assembler::zero, zero_continue);
}
void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
  assert(ProfileInterpreter, "must be profiling interpreter");
  Label set_mdp;
  push(rax);
  push(rbx);
  get_method(rbx);
  movptr(rax, Address(rbx, in_bytes(Method::method_data_offset())));
  testptr(rax, rax);
  jcc(Assembler::zero, set_mdp);
  call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), rbx, r13);
  movptr(rbx, Address(rbx, in_bytes(Method::method_data_offset())));
  addptr(rbx, in_bytes(MethodData::data_offset()));
  addptr(rax, rbx);
  bind(set_mdp);
  movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax);
  pop(rbx);
  pop(rax);
}
void InterpreterMacroAssembler::verify_method_data_pointer() {
  assert(ProfileInterpreter, "must be profiling interpreter");
#ifdef ASSERT
  Label verify_continue;
  push(rax);
  push(rbx);
  push(c_rarg3);
  push(c_rarg2);
  test_method_data_pointer(c_rarg3, verify_continue); // If mdp is zero, continue
  get_method(rbx);
  load_unsigned_short(c_rarg2,
                      Address(c_rarg3, in_bytes(DataLayout::bci_offset())));
  addptr(c_rarg2, Address(rbx, Method::const_offset()));
  lea(c_rarg2, Address(c_rarg2, ConstMethod::codes_offset()));
  cmpptr(c_rarg2, r13);
  jcc(Assembler::equal, verify_continue);
  call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp),
               rbx, r13, c_rarg3);
  bind(verify_continue);
  pop(c_rarg2);
  pop(c_rarg3);
  pop(rbx);
  pop(rax);
#endif // ASSERT
}
void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in,
                                                int constant,
                                                Register value) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  Address data(mdp_in, constant);
  movptr(data, value);
}
void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
                                                      int constant,
                                                      bool decrement) {
  Address data(mdp_in, constant);
  increment_mdp_data_at(data, decrement);
}
void InterpreterMacroAssembler::increment_mdp_data_at(Address data,
                                                      bool decrement) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  if (decrement) {
    addptr(data, (int32_t) -DataLayout::counter_increment);
    Label L;
    jcc(Assembler::negative, L);
    addptr(data, (int32_t) DataLayout::counter_increment);
    bind(L);
  } else {
    assert(DataLayout::counter_increment == 1,
           "flow-free idiom only works with 1");
    addptr(data, DataLayout::counter_increment);
    sbbptr(data, (int32_t)0);
  }
}
void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
                                                      Register reg,
                                                      int constant,
                                                      bool decrement) {
  Address data(mdp_in, reg, Address::times_1, constant);
  increment_mdp_data_at(data, decrement);
}
void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in,
                                                int flag_byte_constant) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  int header_offset = in_bytes(DataLayout::header_offset());
  int header_bits = DataLayout::flag_mask_to_header_mask(flag_byte_constant);
  orl(Address(mdp_in, header_offset), header_bits);
}
void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in,
                                                 int offset,
                                                 Register value,
                                                 Register test_value_out,
                                                 Label& not_equal_continue) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  if (test_value_out == noreg) {
    cmpptr(value, Address(mdp_in, offset));
  } else {
    movptr(test_value_out, Address(mdp_in, offset));
    cmpptr(test_value_out, value);
  }
  jcc(Assembler::notEqual, not_equal_continue);
}
void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in,
                                                     int offset_of_disp) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  Address disp_address(mdp_in, offset_of_disp);
  addptr(mdp_in, disp_address);
  movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), mdp_in);
}
void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in,
                                                     Register reg,
                                                     int offset_of_disp) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  Address disp_address(mdp_in, reg, Address::times_1, offset_of_disp);
  addptr(mdp_in, disp_address);
  movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), mdp_in);
}
void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in,
                                                       int constant) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  addptr(mdp_in, constant);
  movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), mdp_in);
}
void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) {
  assert(ProfileInterpreter, "must be profiling interpreter");
  push(return_bci); // save/restore across call_VM
  call_VM(noreg,
          CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret),
          return_bci);
  pop(return_bci);
}
void InterpreterMacroAssembler::profile_taken_branch(Register mdp,
                                                     Register bumped_count) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    Address data(mdp, in_bytes(JumpData::taken_offset()));
    movptr(bumped_count, data);
    assert(DataLayout::counter_increment == 1,
            "flow-free idiom only works with 1");
    addptr(bumped_count, DataLayout::counter_increment);
    sbbptr(bumped_count, 0);
    movptr(data, bumped_count); // Store back out
    update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
    update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::profile_call(Register mdp) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
    update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::profile_final_call(Register mdp) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
    update_mdp_by_constant(mdp,
                           in_bytes(VirtualCallData::
                                    virtual_call_data_size()));
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
                                                     Register mdp,
                                                     Register reg2,
                                                     bool receiver_can_be_null) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    Label skip_receiver_profile;
    if (receiver_can_be_null) {
      Label not_null;
      testptr(receiver, receiver);
      jccb(Assembler::notZero, not_null);
      increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
      jmp(skip_receiver_profile);
      bind(not_null);
    }
    record_klass_in_profile(receiver, mdp, reg2, true);
    bind(skip_receiver_profile);
    update_mdp_by_constant(mdp,
                           in_bytes(VirtualCallData::
                                    virtual_call_data_size()));
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::record_klass_in_profile_helper(
                                        Register receiver, Register mdp,
                                        Register reg2, int start_row,
                                        Label& done, bool is_virtual_call) {
  if (TypeProfileWidth == 0) {
    if (is_virtual_call) {
      increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
    }
    return;
  }
  int last_row = VirtualCallData::row_limit() - 1;
  assert(start_row <= last_row, "must be work left to do");
  for (int row = start_row; row <= last_row; row++) {
    Label next_test;
    bool test_for_null_also = (row == start_row);
    int recvr_offset = in_bytes(VirtualCallData::receiver_offset(row));
    test_mdp_data_at(mdp, recvr_offset, receiver,
                     (test_for_null_also ? reg2 : noreg),
                     next_test);
    int count_offset = in_bytes(VirtualCallData::receiver_count_offset(row));
    increment_mdp_data_at(mdp, count_offset);
    jmp(done);
    bind(next_test);
    if (test_for_null_also) {
      Label found_null;
      testptr(reg2, reg2);
      if (start_row == last_row) {
        if (is_virtual_call) {
          jccb(Assembler::zero, found_null);
          increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
          jmp(done);
          bind(found_null);
        } else {
          jcc(Assembler::notZero, done);
        }
        break;
      }
      jcc(Assembler::zero, found_null);
      record_klass_in_profile_helper(receiver, mdp, reg2, start_row + 1, done, is_virtual_call);
      bind(found_null);
    }
  }
  int recvr_offset = in_bytes(VirtualCallData::receiver_offset(start_row));
  set_mdp_data_at(mdp, recvr_offset, receiver);
  int count_offset = in_bytes(VirtualCallData::receiver_count_offset(start_row));
  movl(reg2, DataLayout::counter_increment);
  set_mdp_data_at(mdp, count_offset, reg2);
  if (start_row > 0) {
    jmp(done);
  }
}
void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
                                                        Register mdp, Register reg2,
                                                        bool is_virtual_call) {
  assert(ProfileInterpreter, "must be profiling");
  Label done;
  record_klass_in_profile_helper(receiver, mdp, reg2, 0, done, is_virtual_call);
  bind (done);
}
void InterpreterMacroAssembler::profile_ret(Register return_bci,
                                            Register mdp) {
  if (ProfileInterpreter) {
    Label profile_continue;
    uint row;
    test_method_data_pointer(mdp, profile_continue);
    increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
    for (row = 0; row < RetData::row_limit(); row++) {
      Label next_test;
      test_mdp_data_at(mdp,
                       in_bytes(RetData::bci_offset(row)),
                       return_bci, noreg,
                       next_test);
      increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)));
      update_mdp_by_offset(mdp,
                           in_bytes(RetData::bci_displacement_offset(row)));
      jmp(profile_continue);
      bind(next_test);
    }
    update_mdp_for_ret(return_bci);
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::profile_null_seen(Register mdp) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
    int mdp_delta = in_bytes(BitData::bit_data_size());
    if (TypeProfileCasts) {
      mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
    }
    update_mdp_by_constant(mdp, mdp_delta);
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) {
  if (ProfileInterpreter && TypeProfileCasts) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    int count_offset = in_bytes(CounterData::count_offset());
    count_offset -= in_bytes(VirtualCallData::virtual_call_data_size());
    increment_mdp_data_at(mdp, count_offset, true);
    bind (profile_continue);
  }
}
void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    int mdp_delta = in_bytes(BitData::bit_data_size());
    if (TypeProfileCasts) {
      mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
      record_klass_in_profile(klass, mdp, reg2, false);
    }
    update_mdp_by_constant(mdp, mdp_delta);
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::profile_switch_default(Register mdp) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    increment_mdp_data_at(mdp,
                          in_bytes(MultiBranchData::default_count_offset()));
    update_mdp_by_offset(mdp,
                         in_bytes(MultiBranchData::
                                  default_displacement_offset()));
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::profile_switch_case(Register index,
                                                    Register mdp,
                                                    Register reg2) {
  if (ProfileInterpreter) {
    Label profile_continue;
    test_method_data_pointer(mdp, profile_continue);
    movl(reg2, in_bytes(MultiBranchData::per_case_size()));
    imulptr(index, reg2); // XXX l ?
    addptr(index, in_bytes(MultiBranchData::case_array_offset())); // XXX l ?
    increment_mdp_data_at(mdp,
                          index,
                          in_bytes(MultiBranchData::relative_count_offset()));
    update_mdp_by_offset(mdp,
                         index,
                         in_bytes(MultiBranchData::
                                  relative_displacement_offset()));
    bind(profile_continue);
  }
}
void InterpreterMacroAssembler::verify_oop(Register reg, TosState state) {
  if (state == atos) {
    MacroAssembler::verify_oop(reg);
  }
}
void InterpreterMacroAssembler::verify_FPU(int stack_depth, TosState state) {
}
#endif // !CC_INTERP
void InterpreterMacroAssembler::notify_method_entry() {
  if (JvmtiExport::can_post_interpreter_events()) {
    Label L;
    movl(rdx, Address(r15_thread, JavaThread::interp_only_mode_offset()));
    testl(rdx, rdx);
    jcc(Assembler::zero, L);
    call_VM(noreg, CAST_FROM_FN_PTR(address,
                                    InterpreterRuntime::post_method_entry));
    bind(L);
  }
  {
    SkipIfEqual skip(this, &DTraceMethodProbes, false);
    get_method(c_rarg1);
    call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
                 r15_thread, c_rarg1);
  }
  if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) {
    get_method(c_rarg1);
    call_VM_leaf(
      CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
      r15_thread, c_rarg1);
  }
}
void InterpreterMacroAssembler::notify_method_exit(
    TosState state, NotifyMethodExitMode mode) {
  if (mode == NotifyJVMTI && JvmtiExport::can_post_interpreter_events()) {
    Label L;
    NOT_CC_INTERP(push(state);)
    movl(rdx, Address(r15_thread, JavaThread::interp_only_mode_offset()));
    testl(rdx, rdx);
    jcc(Assembler::zero, L);
    call_VM(noreg,
            CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
    bind(L);
    NOT_CC_INTERP(pop(state));
  }
  {
    SkipIfEqual skip(this, &DTraceMethodProbes, false);
    NOT_CC_INTERP(push(state));
    get_method(c_rarg1);
    call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
                 r15_thread, c_rarg1);
    NOT_CC_INTERP(pop(state));
  }
}
void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr,
                                                        int increment, int mask,
                                                        Register scratch, bool preloaded,
                                                        Condition cond, Label* where) {
  if (!preloaded) {
    movl(scratch, counter_addr);
  }
  incrementl(scratch, increment);
  movl(counter_addr, scratch);
  andl(scratch, mask);
  if (where != NULL) {
    jcc(cond, *where);
  }
}
C:\hotspot-69087d08d473\src\cpu\x86\vm/interp_masm_x86_64.hpp
#ifndef CC_INTERP
 protected:
  virtual void call_VM_leaf_base(address entry_point,
                                 int number_of_arguments);
  virtual void call_VM_base(Register oop_result,
                            Register java_thread,
                            Register last_java_sp,
                            address  entry_point,
                            int number_of_arguments,
                            bool check_exceptions);
  virtual void check_and_handle_popframe(Register java_thread);
  virtual void check_and_handle_earlyret(Register java_thread);
  void dispatch_base(TosState state, address* table, bool verifyoop = true);
#endif // CC_INTERP
 public:
  InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code), _locals_register(r14), _bcp_register(r13) {}
  void load_earlyret_value(TosState state);
#ifdef CC_INTERP
  void save_bcp()                                          { /*  not needed in c++ interpreter and harmless */ }
  void restore_bcp()                                       { /*  not needed in c++ interpreter and harmless */ }
  void get_method(Register reg);
#else
  void save_bcp() {
    movptr(Address(rbp, frame::interpreter_frame_bcx_offset * wordSize), r13);
  }
  void restore_bcp() {
    movptr(r13, Address(rbp, frame::interpreter_frame_bcx_offset * wordSize));
  }
  void restore_locals() {
    movptr(r14, Address(rbp, frame::interpreter_frame_locals_offset * wordSize));
  }
  void get_method(Register reg) {
    movptr(reg, Address(rbp, frame::interpreter_frame_method_offset * wordSize));
  }
  void get_const(Register reg) {
    get_method(reg);
    movptr(reg, Address(reg, Method::const_offset()));
  }
  void get_constant_pool(Register reg) {
    get_const(reg);
    movptr(reg, Address(reg, ConstMethod::constants_offset()));
  }
  void get_constant_pool_cache(Register reg) {
    get_constant_pool(reg);
    movptr(reg, Address(reg, ConstantPool::cache_offset_in_bytes()));
  }
  void get_cpool_and_tags(Register cpool, Register tags) {
    get_constant_pool(cpool);
    movptr(tags, Address(cpool, ConstantPool::tags_offset_in_bytes()));
  }
  void get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset);
  void get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size = sizeof(u2));
  void get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size = sizeof(u2));
  void get_cache_entry_pointer_at_bcp(Register cache, Register tmp, int bcp_offset, size_t index_size = sizeof(u2));
  void get_cache_index_at_bcp(Register index, int bcp_offset, size_t index_size = sizeof(u2));
  void get_method_counters(Register method, Register mcs, Label& skip);
  void load_resolved_reference_at_index(Register result, Register index);
  void pop_ptr(Register r = rax);
  void pop_i(Register r = rax);
  void pop_l(Register r = rax);
  void pop_f(XMMRegister r = xmm0);
  void pop_d(XMMRegister r = xmm0);
  void push_ptr(Register r = rax);
  void push_i(Register r = rax);
  void push_l(Register r = rax);
  void push_f(XMMRegister r = xmm0);
  void push_d(XMMRegister r = xmm0);
  void pop(Register r ) { ((MacroAssembler*)this)->pop(r); }
  void push(Register r ) { ((MacroAssembler*)this)->push(r); }
  void push(int32_t imm ) { ((MacroAssembler*)this)->push(imm); }
  void pop(TosState state); // transition vtos -> state
  void push(TosState state); // transition state -> vtos
  void empty_expression_stack() {
    movptr(rsp, Address(rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize));
    movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
  }
  void load_ptr(int n, Register val);
  void store_ptr(int n, Register val);
  void gen_subtype_check( Register sub_klass, Label &ok_is_subtype );
  void dispatch_prolog(TosState state, int step = 0);
  void dispatch_epilog(TosState state, int step = 0);
  void dispatch_only(TosState state);
  void dispatch_only_normal(TosState state);
  void dispatch_only_noverify(TosState state);
  void dispatch_next(TosState state, int step = 0);
  void dispatch_via (TosState state, address* table);
  void prepare_to_jump_from_interpreted();
  void jump_from_interpreted(Register method, Register temp);
  void remove_activation(TosState state, Register ret_addr,
                         bool throw_monitor_exception = true,
                         bool install_monitor_exception = true,
                         bool notify_jvmdi = true);
#endif // CC_INTERP
  void lock_object  (Register lock_reg);
  void unlock_object(Register lock_reg);
#ifndef CC_INTERP
  void set_method_data_pointer_for_bcp();
  void test_method_data_pointer(Register mdp, Label& zero_continue);
  void verify_method_data_pointer();
  void set_mdp_data_at(Register mdp_in, int constant, Register value);
  void increment_mdp_data_at(Address data, bool decrement = false);
  void increment_mdp_data_at(Register mdp_in, int constant,
                             bool decrement = false);
  void increment_mdp_data_at(Register mdp_in, Register reg, int constant,
                             bool decrement = false);
  void increment_mask_and_jump(Address counter_addr,
                               int increment, int mask,
                               Register scratch, bool preloaded,
                               Condition cond, Label* where);
  void set_mdp_flag_at(Register mdp_in, int flag_constant);
  void test_mdp_data_at(Register mdp_in, int offset, Register value,
                        Register test_value_out,
                        Label& not_equal_continue);
  void record_klass_in_profile(Register receiver, Register mdp,
                               Register reg2, bool is_virtual_call);
  void record_klass_in_profile_helper(Register receiver, Register mdp,
                                      Register reg2, int start_row,
                                      Label& done, bool is_virtual_call);
  void update_mdp_by_offset(Register mdp_in, int offset_of_offset);
  void update_mdp_by_offset(Register mdp_in, Register reg, int offset_of_disp);
  void update_mdp_by_constant(Register mdp_in, int constant);
  void update_mdp_for_ret(Register return_bci);
  void profile_taken_branch(Register mdp, Register bumped_count);
  void profile_not_taken_branch(Register mdp);
  void profile_call(Register mdp);
  void profile_final_call(Register mdp);
  void profile_virtual_call(Register receiver, Register mdp,
                            Register scratch2,
                            bool receiver_can_be_null = false);
  void profile_ret(Register return_bci, Register mdp);
  void profile_null_seen(Register mdp);
  void profile_typecheck(Register mdp, Register klass, Register scratch);
  void profile_typecheck_failed(Register mdp);
  void profile_switch_default(Register mdp);
  void profile_switch_case(Register index_in_scratch, Register mdp,
                           Register scratch2);
  void verify_oop(Register reg, TosState state = atos);
  void verify_FPU(int stack_depth, TosState state = ftos);
#endif // !CC_INTERP
  typedef enum { NotifyJVMTI, SkipNotifyJVMTI } NotifyMethodExitMode;
  void notify_method_entry();
  void notify_method_exit(TosState state, NotifyMethodExitMode mode);
C:\hotspot-69087d08d473\src\cpu\x86\vm/javaFrameAnchor_x86.hpp
#ifndef CPU_X86_VM_JAVAFRAMEANCHOR_X86_HPP
#define CPU_X86_VM_JAVAFRAMEANCHOR_X86_HPP
private:
  intptr_t* volatile        _last_Java_fp;           // pointer is volatile not what it points to
public:
  void clear(void) {
    _last_Java_sp = NULL;
    _last_Java_fp = NULL;
    _last_Java_pc = NULL;
  }
  void copy(JavaFrameAnchor* src) {
    if (_last_Java_sp != src->_last_Java_sp)
      _last_Java_sp = NULL;
    _last_Java_fp = src->_last_Java_fp;
    _last_Java_pc = src->_last_Java_pc;
    _last_Java_sp = src->_last_Java_sp;
  }
  bool walkable(void)                            { return _last_Java_sp != NULL && _last_Java_pc != NULL; }
  void make_walkable(JavaThread* thread);
  void capture_last_Java_pc(void);
  intptr_t* last_Java_sp(void) const             { return _last_Java_sp; }
  address last_Java_pc(void)                     { return _last_Java_pc; }
private:
  static ByteSize last_Java_fp_offset()          { return byte_offset_of(JavaFrameAnchor, _last_Java_fp); }
public:
  void set_last_Java_sp(intptr_t* sp)            { _last_Java_sp = sp; }
  intptr_t*   last_Java_fp(void)                     { return _last_Java_fp; }
  void set_last_Java_fp(intptr_t* fp)                { _last_Java_fp = fp; }
#endif // CPU_X86_VM_JAVAFRAMEANCHOR_X86_HPP
C:\hotspot-69087d08d473\src\cpu\x86\vm/jniFastGetField_x86_32.cpp
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
#include "memory/resourceArea.hpp"
#include "prims/jniFastGetField.hpp"
#include "prims/jvm_misc.hpp"
#include "runtime/safepoint.hpp"
#define __ masm->
#define BUFFER_SIZE 30
#ifdef _WINDOWS
GetBooleanField_t JNI_FastGetField::jni_fast_GetBooleanField_fp;
GetByteField_t    JNI_FastGetField::jni_fast_GetByteField_fp;
GetCharField_t    JNI_FastGetField::jni_fast_GetCharField_fp;
GetShortField_t   JNI_FastGetField::jni_fast_GetShortField_fp;
GetIntField_t     JNI_FastGetField::jni_fast_GetIntField_fp;
GetLongField_t    JNI_FastGetField::jni_fast_GetLongField_fp;
GetFloatField_t   JNI_FastGetField::jni_fast_GetFloatField_fp;
GetDoubleField_t  JNI_FastGetField::jni_fast_GetDoubleField_fp;
#endif
address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
  const char *name = NULL;
  switch (type) {
    case T_BOOLEAN: name = "jni_fast_GetBooleanField"; break;
    case T_BYTE:    name = "jni_fast_GetByteField";    break;
    case T_CHAR:    name = "jni_fast_GetCharField";    break;
    case T_SHORT:   name = "jni_fast_GetShortField";   break;
    case T_INT:     name = "jni_fast_GetIntField";     break;
    default:        ShouldNotReachHere();
  }
  ResourceMark rm;
  BufferBlob* blob = BufferBlob::create(name, BUFFER_SIZE*wordSize);
  CodeBuffer cbuf(blob);
  MacroAssembler* masm = new MacroAssembler(&cbuf);
  address fast_entry = __ pc();
  Label slow;
  ExternalAddress counter(SafepointSynchronize::safepoint_counter_addr());
  __ mov32 (rcx, counter);
  __ testb (rcx, 1);
  __ jcc (Assembler::notZero, slow);
  if (os::is_MP()) {
    __ mov(rax, rcx);
    __ andptr(rax, 1);                         // rax, must end up 0
    __ movptr(rdx, Address(rsp, rax, Address::times_1, 2*wordSize));
  } else {
    __ movptr (rdx, Address(rsp, 2*wordSize)); // obj
  }
  __ movptr(rax, Address(rsp, 3*wordSize));  // jfieldID
  __ clear_jweak_tag(rdx);
  __ movptr(rdx, Address(rdx, 0));           // *obj
  __ shrptr (rax, 2);                        // offset
  assert(count < LIST_CAPACITY, "LIST_CAPACITY too small");
  speculative_load_pclist[count] = __ pc();
  switch (type) {
    case T_BOOLEAN: __ movzbl (rax, Address(rdx, rax, Address::times_1)); break;
    case T_BYTE:    __ movsbl (rax, Address(rdx, rax, Address::times_1)); break;
    case T_CHAR:    __ movzwl (rax, Address(rdx, rax, Address::times_1)); break;
    case T_SHORT:   __ movswl (rax, Address(rdx, rax, Address::times_1)); break;
    case T_INT:     __ movl   (rax, Address(rdx, rax, Address::times_1)); break;
    default:        ShouldNotReachHere();
  }
  Address ca1;
  if (os::is_MP()) {
    __ lea(rdx, counter);
    __ xorptr(rdx, rax);
    __ xorptr(rdx, rax);
    __ cmp32(rcx, Address(rdx, 0));
  } else {
    __ cmp32(rcx, counter);
  }
  __ jcc (Assembler::notEqual, slow);
#ifndef _WINDOWS
  __ ret (0);
#else
  __ ret (3*wordSize);
#endif
  slowcase_entry_pclist[count++] = __ pc();
  __ bind (slow);
  address slow_case_addr = NULL;
  switch (type) {
    case T_BOOLEAN: slow_case_addr = jni_GetBooleanField_addr(); break;
    case T_BYTE:    slow_case_addr = jni_GetByteField_addr();    break;
    case T_CHAR:    slow_case_addr = jni_GetCharField_addr();    break;
    case T_SHORT:   slow_case_addr = jni_GetShortField_addr();   break;
    case T_INT:     slow_case_addr = jni_GetIntField_addr();
  }
  __ jump (ExternalAddress(slow_case_addr));
  __ flush ();
#ifndef _WINDOWS
  return fast_entry;
#else
  switch (type) {
  case T_BOOLEAN: jni_fast_GetBooleanField_fp = (GetBooleanField_t) fast_entry; break;
  case T_BYTE:    jni_fast_GetByteField_fp    = (GetByteField_t)    fast_entry; break;
  case T_CHAR:    jni_fast_GetCharField_fp    = (GetCharField_t)    fast_entry; break;
  case T_SHORT:   jni_fast_GetShortField_fp   = (GetShortField_t)   fast_entry; break;
  case T_INT:     jni_fast_GetIntField_fp     = (GetIntField_t)     fast_entry; break;
  }
  return os::win32::fast_jni_accessor_wrapper(type);
#endif
}
address JNI_FastGetField::generate_fast_get_boolean_field() {
  return generate_fast_get_int_field0(T_BOOLEAN);
}
address JNI_FastGetField::generate_fast_get_byte_field() {
  return generate_fast_get_int_field0(T_BYTE);
}
address JNI_FastGetField::generate_fast_get_char_field() {
  return generate_fast_get_int_field0(T_CHAR);
}
address JNI_FastGetField::generate_fast_get_short_field() {
  return generate_fast_get_int_field0(T_SHORT);
}
address JNI_FastGetField::generate_fast_get_int_field() {
  return generate_fast_get_int_field0(T_INT);
}
address JNI_FastGetField::generate_fast_get_long_field() {
  const char *name = "jni_fast_GetLongField";
  ResourceMark rm;
  BufferBlob* blob = BufferBlob::create(name, BUFFER_SIZE*wordSize);
  CodeBuffer cbuf(blob);
  MacroAssembler* masm = new MacroAssembler(&cbuf);
  address fast_entry = __ pc();
  Label slow;
  ExternalAddress counter(SafepointSynchronize::safepoint_counter_addr());
  __ push  (rsi);
  __ mov32 (rcx, counter);
  __ testb (rcx, 1);
  __ jcc (Assembler::notZero, slow);
  if (os::is_MP()) {
    __ mov(rax, rcx);
    __ andptr(rax, 1);                        // rax, must end up 0
    __ movptr(rdx, Address(rsp, rax, Address::times_1, 3*wordSize));
  } else {
    __ movptr(rdx, Address(rsp, 3*wordSize)); // obj
  }
  __ movptr(rsi, Address(rsp, 4*wordSize));  // jfieldID
  __ clear_jweak_tag(rdx);
  __ movptr(rdx, Address(rdx, 0));           // *obj
  __ shrptr(rsi, 2);                         // offset
  assert(count < LIST_CAPACITY-1, "LIST_CAPACITY too small");
  speculative_load_pclist[count++] = __ pc();
  __ movptr(rax, Address(rdx, rsi, Address::times_1));
#ifndef _LP64
  speculative_load_pclist[count] = __ pc();
  __ movl(rdx, Address(rdx, rsi, Address::times_1, 4));
#endif // _LP64
  if (os::is_MP()) {
    __ lea(rsi, counter);
    __ xorptr(rsi, rdx);
    __ xorptr(rsi, rax);
    __ xorptr(rsi, rdx);
    __ xorptr(rsi, rax);
    __ cmp32(rcx, Address(rsi, 0));
  } else {
    __ cmp32(rcx, counter);
  }
  __ jcc (Assembler::notEqual, slow);
  __ pop (rsi);
#ifndef _WINDOWS
  __ ret (0);
#else
  __ ret (3*wordSize);
#endif
  slowcase_entry_pclist[count-1] = __ pc();
  slowcase_entry_pclist[count++] = __ pc();
  __ bind (slow);
  __ pop  (rsi);
  address slow_case_addr = jni_GetLongField_addr();;
  __ jump (ExternalAddress(slow_case_addr));
  __ flush ();
#ifndef _WINDOWS
  return fast_entry;
#else
  jni_fast_GetLongField_fp = (GetLongField_t) fast_entry;
  return os::win32::fast_jni_accessor_wrapper(T_LONG);
#endif
}
address JNI_FastGetField::generate_fast_get_float_field0(BasicType type) {
  const char *name = NULL;
  switch (type) {
    case T_FLOAT:  name = "jni_fast_GetFloatField";  break;
    case T_DOUBLE: name = "jni_fast_GetDoubleField"; break;
    default:       ShouldNotReachHere();
  }
  ResourceMark rm;
  BufferBlob* blob = BufferBlob::create(name, BUFFER_SIZE*wordSize);
  CodeBuffer cbuf(blob);
  MacroAssembler* masm = new MacroAssembler(&cbuf);
  address fast_entry = __ pc();
  Label slow_with_pop, slow;
  ExternalAddress counter(SafepointSynchronize::safepoint_counter_addr());
  __ mov32 (rcx, counter);
  __ testb (rcx, 1);
  __ jcc (Assembler::notZero, slow);
  if (os::is_MP()) {
    __ mov(rax, rcx);
    __ andptr(rax, 1);                        // rax, must end up 0
    __ movptr(rdx, Address(rsp, rax, Address::times_1, 2*wordSize));
  } else {
    __ movptr(rdx, Address(rsp, 2*wordSize)); // obj
  }
  __ movptr(rax, Address(rsp, 3*wordSize));  // jfieldID
  __ clear_jweak_tag(rdx);
  __ movptr(rdx, Address(rdx, 0));           // *obj
  __ shrptr(rax, 2);                         // offset
  assert(count < LIST_CAPACITY, "LIST_CAPACITY too small");
  speculative_load_pclist[count] = __ pc();
  switch (type) {
#ifndef _LP64
    case T_FLOAT:  __ fld_s (Address(rdx, rax, Address::times_1)); break;
    case T_DOUBLE: __ fld_d (Address(rdx, rax, Address::times_1)); break;
#else
    case T_FLOAT:  __ movflt (xmm0, Address(robj, roffset, Address::times_1)); break;
    case T_DOUBLE: __ movdbl (xmm0, Address(robj, roffset, Address::times_1)); break;
#endif // _LP64
    default:       ShouldNotReachHere();
  }
  Address ca1;
  if (os::is_MP()) {
    __ fst_s (Address(rsp, -4));
    __ lea(rdx, counter);
    __ movl (rax, Address(rsp, -4));
    __ xorptr(rdx, rax);
    __ xorptr(rdx, rax);
    __ cmp32(rcx, Address(rdx, 0));
  } else {
    __ cmp32(rcx, counter);
  }
  __ jcc (Assembler::notEqual, slow_with_pop);
#ifndef _WINDOWS
  __ ret (0);
#else
  __ ret (3*wordSize);
#endif
  __ bind (slow_with_pop);
  __ fstp_d (0);
  slowcase_entry_pclist[count++] = __ pc();
  __ bind (slow);
  address slow_case_addr = NULL;
  switch (type) {
    case T_FLOAT:  slow_case_addr = jni_GetFloatField_addr();  break;
    case T_DOUBLE: slow_case_addr = jni_GetDoubleField_addr(); break;
    default:       ShouldNotReachHere();
  }
  __ jump (ExternalAddress(slow_case_addr));
  __ flush ();
#ifndef _WINDOWS
  return fast_entry;
#else
  switch (type) {
  case T_FLOAT:  jni_fast_GetFloatField_fp  = (GetFloatField_t)  fast_entry; break;
  case T_DOUBLE: jni_fast_GetDoubleField_fp = (GetDoubleField_t) fast_entry; break;
  }
  return os::win32::fast_jni_accessor_wrapper(type);
#endif
}
address JNI_FastGetField::generate_fast_get_float_field() {
  return generate_fast_get_float_field0(T_FLOAT);
}
address JNI_FastGetField::generate_fast_get_double_field() {
  return generate_fast_get_float_field0(T_DOUBLE);
}
C:\hotspot-69087d08d473\src\cpu\x86\vm/jniFastGetField_x86_64.cpp
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
#include "memory/resourceArea.hpp"
#include "prims/jniFastGetField.hpp"
#include "prims/jvm_misc.hpp"
#include "runtime/safepoint.hpp"
#define __ masm->
#define BUFFER_SIZE 30*wordSize
static const Register robj          = r9;
static const Register rcounter      = r10;
static const Register roffset       = r11;
static const Register rcounter_addr = r11;
address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
  const char *name = NULL;
  switch (type) {
    case T_BOOLEAN: name = "jni_fast_GetBooleanField"; break;
    case T_BYTE:    name = "jni_fast_GetByteField";    break;
    case T_CHAR:    name = "jni_fast_GetCharField";    break;
    case T_SHORT:   name = "jni_fast_GetShortField";   break;
    case T_INT:     name = "jni_fast_GetIntField";     break;
    case T_LONG:    name = "jni_fast_GetLongField";    break;
    default:        ShouldNotReachHere();
  }
  ResourceMark rm;
  BufferBlob* blob = BufferBlob::create(name, BUFFER_SIZE);
  CodeBuffer cbuf(blob);
  MacroAssembler* masm = new MacroAssembler(&cbuf);
  address fast_entry = __ pc();
  Label slow;
  ExternalAddress counter(SafepointSynchronize::safepoint_counter_addr());
  __ mov32 (rcounter, counter);
  __ mov   (robj, c_rarg1);
  __ testb (rcounter, 1);
  __ jcc (Assembler::notZero, slow);
  if (os::is_MP()) {
    __ xorptr(robj, rcounter);
    __ xorptr(robj, rcounter);                  // obj, since
  }
  __ clear_jweak_tag(robj);
  __ movptr(robj, Address(robj, 0));            // *obj
  __ mov   (roffset, c_rarg2);
  __ shrptr(roffset, 2);                        // offset
  assert(count < LIST_CAPACITY, "LIST_CAPACITY too small");
  speculative_load_pclist[count] = __ pc();
  switch (type) {
    case T_BOOLEAN: __ movzbl (rax, Address(robj, roffset, Address::times_1)); break;
    case T_BYTE:    __ movsbl (rax, Address(robj, roffset, Address::times_1)); break;
    case T_CHAR:    __ movzwl (rax, Address(robj, roffset, Address::times_1)); break;
    case T_SHORT:   __ movswl (rax, Address(robj, roffset, Address::times_1)); break;
    case T_INT:     __ movl   (rax, Address(robj, roffset, Address::times_1)); break;
    case T_LONG:    __ movq   (rax, Address(robj, roffset, Address::times_1)); break;
    default:        ShouldNotReachHere();
  }
  if (os::is_MP()) {
    __ lea(rcounter_addr, counter);
    __ xorptr(rcounter_addr, rax);
    __ xorptr(rcounter_addr, rax);
    __ cmpl (rcounter, Address(rcounter_addr, 0));
  } else {
    __ cmp32 (rcounter, counter);
  }
  __ jcc (Assembler::notEqual, slow);
  __ ret (0);
  slowcase_entry_pclist[count++] = __ pc();
  __ bind (slow);
  address slow_case_addr = NULL;
  switch (type) {
    case T_BOOLEAN: slow_case_addr = jni_GetBooleanField_addr(); break;
    case T_BYTE:    slow_case_addr = jni_GetByteField_addr();    break;
    case T_CHAR:    slow_case_addr = jni_GetCharField_addr();    break;
    case T_SHORT:   slow_case_addr = jni_GetShortField_addr();   break;
    case T_INT:     slow_case_addr = jni_GetIntField_addr();     break;
    case T_LONG:    slow_case_addr = jni_GetLongField_addr();
  }
  __ jump (ExternalAddress(slow_case_addr));
  __ flush ();
  return fast_entry;
}
address JNI_FastGetField::generate_fast_get_boolean_field() {
  return generate_fast_get_int_field0(T_BOOLEAN);
}
address JNI_FastGetField::generate_fast_get_byte_field() {
  return generate_fast_get_int_field0(T_BYTE);
}
address JNI_FastGetField::generate_fast_get_char_field() {
  return generate_fast_get_int_field0(T_CHAR);
}
address JNI_FastGetField::generate_fast_get_short_field() {
  return generate_fast_get_int_field0(T_SHORT);
}
address JNI_FastGetField::generate_fast_get_int_field() {
  return generate_fast_get_int_field0(T_INT);
}
address JNI_FastGetField::generate_fast_get_long_field() {
  return generate_fast_get_int_field0(T_LONG);
}
address JNI_FastGetField::generate_fast_get_float_field0(BasicType type) {
  const char *name = NULL;
  switch (type) {
    case T_FLOAT:     name = "jni_fast_GetFloatField";     break;
    case T_DOUBLE:    name = "jni_fast_GetDoubleField";    break;
    default:          ShouldNotReachHere();
  }
  ResourceMark rm;
  BufferBlob* blob = BufferBlob::create(name, BUFFER_SIZE);
  CodeBuffer cbuf(blob);
  MacroAssembler* masm = new MacroAssembler(&cbuf);
  address fast_entry = __ pc();
  Label slow;
  ExternalAddress counter(SafepointSynchronize::safepoint_counter_addr());
  __ mov32 (rcounter, counter);
  __ mov   (robj, c_rarg1);
  __ testb (rcounter, 1);
  __ jcc (Assembler::notZero, slow);
  if (os::is_MP()) {
    __ xorptr(robj, rcounter);
    __ xorptr(robj, rcounter);                  // obj, since
  }
  __ clear_jweak_tag(robj);
  __ movptr(robj, Address(robj, 0));            // *obj
  __ mov   (roffset, c_rarg2);
  __ shrptr(roffset, 2);                        // offset
  assert(count < LIST_CAPACITY, "LIST_CAPACITY too small");
  speculative_load_pclist[count] = __ pc();
  switch (type) {
    case T_FLOAT:  __ movflt (xmm0, Address(robj, roffset, Address::times_1)); break;
    case T_DOUBLE: __ movdbl (xmm0, Address(robj, roffset, Address::times_1)); break;
    default:        ShouldNotReachHere();
  }
  if (os::is_MP()) {
    __ lea(rcounter_addr, counter);
    __ movdq (rax, xmm0);
    __ xorptr(rcounter_addr, rax);
    __ xorptr(rcounter_addr, rax);
    __ cmpl (rcounter, Address(rcounter_addr, 0));
  } else {
    __ cmp32 (rcounter, counter);
  }
  __ jcc (Assembler::notEqual, slow);
  __ ret (0);
  slowcase_entry_pclist[count++] = __ pc();
  __ bind (slow);
  address slow_case_addr = NULL;
  switch (type) {
    case T_FLOAT:     slow_case_addr = jni_GetFloatField_addr();  break;
    case T_DOUBLE:    slow_case_addr = jni_GetDoubleField_addr();
  }
  __ jump (ExternalAddress(slow_case_addr));
  __ flush ();
  return fast_entry;
}
address JNI_FastGetField::generate_fast_get_float_field() {
  return generate_fast_get_float_field0(T_FLOAT);
}
address JNI_FastGetField::generate_fast_get_double_field() {
  return generate_fast_get_float_field0(T_DOUBLE);
}
C:\hotspot-69087d08d473\src\cpu\x86\vm/jniTypes_x86.hpp
#ifndef CPU_X86_VM_JNITYPES_X86_HPP
#define CPU_X86_VM_JNITYPES_X86_HPP
#include "memory/allocation.hpp"
#include "oops/oop.hpp"
#include "prims/jni.h"
class JNITypes : AllStatic {
private:
#ifndef AMD64
  static inline void    put_int2r(jint *from, intptr_t *to)           { *(jint *)(to++) = from[1];
  static inline void    put_int2r(jint *from, intptr_t *to, int& pos) { put_int2r(from, to + pos); pos += 2; }
#endif // AMD64
public:
  static inline void    put_int(jint  from, intptr_t *to)           { *(jint *)(to +   0  ) =  from; }
  static inline void    put_int(jint  from, intptr_t *to, int& pos) { *(jint *)(to + pos++) =  from; }
  static inline void    put_int(jint *from, intptr_t *to, int& pos) { *(jint *)(to + pos++) = *from; }
#ifdef AMD64
  static inline void put_long(jlong  from, intptr_t *to) {
  }
  static inline void put_long(jlong  from, intptr_t *to, int& pos) {
    pos += 2;
  }
  static inline void put_long(jlong *from, intptr_t *to, int& pos) {
    pos += 2;
  }
#else
  static inline void    put_long(jlong  from, intptr_t *to)           { put_int2r((jint *)&from, to); }
  static inline void    put_long(jlong  from, intptr_t *to, int& pos) { put_int2r((jint *)&from, to, pos); }
  static inline void    put_long(jlong *from, intptr_t *to, int& pos) { put_int2r((jint *) from, to, pos); }
#endif // AMD64
  static inline void    put_obj(oop  from, intptr_t *to)           { *(oop *)(to +   0  ) =  from; }
  static inline void    put_obj(oop  from, intptr_t *to, int& pos) { *(oop *)(to + pos++) =  from; }
  static inline void    put_obj(oop *from, intptr_t *to, int& pos) { *(oop *)(to + pos++) = *from; }
  static inline void    put_float(jfloat  from, intptr_t *to)           { *(jfloat *)(to +   0  ) =  from;  }
  static inline void    put_float(jfloat  from, intptr_t *to, int& pos) { *(jfloat *)(to + pos++) =  from; }
  static inline void    put_float(jfloat *from, intptr_t *to, int& pos) { *(jfloat *)(to + pos++) = *from; }
#undef _JNI_SLOT_OFFSET
#ifdef AMD64
#define _JNI_SLOT_OFFSET 1
  static inline void put_double(jdouble  from, intptr_t *to) {
  }
  static inline void put_double(jdouble  from, intptr_t *to, int& pos) {
    pos += 2;
  }
  static inline void put_double(jdouble *from, intptr_t *to, int& pos) {
    pos += 2;
  }
#else
#define _JNI_SLOT_OFFSET 0
  static inline void    put_double(jdouble  from, intptr_t *to)           { put_int2r((jint *)&from, to); }
  static inline void    put_double(jdouble  from, intptr_t *to, int& pos) { put_int2r((jint *)&from, to, pos); }
  static inline void    put_double(jdouble *from, intptr_t *to, int& pos) { put_int2r((jint *) from, to, pos); }
#endif // AMD64
  static inline jint    get_int   (intptr_t *from) { return *(jint *)   from; }
  static inline jlong   get_long  (intptr_t *from) { return *(jlong *)  (from + _JNI_SLOT_OFFSET); }
  static inline oop     get_obj   (intptr_t *from) { return *(oop *)    from; }
  static inline jfloat  get_float (intptr_t *from) { return *(jfloat *) from; }
  static inline jdouble get_double(intptr_t *from) { return *(jdouble *)(from + _JNI_SLOT_OFFSET); }
#undef _JNI_SLOT_OFFSET
};
#endif // CPU_X86_VM_JNITYPES_X86_HPP
C:\hotspot-69087d08d473\src\cpu\x86\vm/jni_x86.h
#ifndef _JAVASOFT_JNI_MD_H_
#define _JAVASOFT_JNI_MD_H_
#if defined(SOLARIS) || defined(LINUX) || defined(_ALLBSD_SOURCE)
#ifndef __has_attribute
  #define __has_attribute(x) 0
#endif
#if (defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4) && (__GNUC_MINOR__ > 2))) || __has_attribute(visibility)
  #define JNIEXPORT     __attribute__((visibility("default")))
  #define JNIIMPORT     __attribute__((visibility("default")))
#else
  #define JNIEXPORT
  #define JNIIMPORT
#endif
  #define JNICALL
  typedef int jint;
#if defined(_LP64)
  typedef long jlong;
#else
  typedef long long jlong;
#endif
#else
  #define JNIEXPORT __declspec(dllexport)
  #define JNIIMPORT __declspec(dllimport)
  #define JNICALL __stdcall
  typedef int jint;
  typedef __int64 jlong;
#endif
typedef signed char jbyte;
#endif /* !_JAVASOFT_JNI_MD_H_ */
C:\hotspot-69087d08d473\src\cpu\x86\vm/macroAssembler_x86.cpp
#include "precompiled.hpp"
#include "asm/assembler.hpp"
#include "asm/assembler.inline.hpp"
#include "compiler/disassembler.hpp"
#include "gc_interface/collectedHeap.inline.hpp"
#include "interpreter/interpreter.hpp"
#include "memory/cardTableModRefBS.hpp"
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
#include "prims/methodHandles.hpp"
#include "runtime/biasedLocking.hpp"
#include "runtime/interfaceSupport.hpp"
#include "runtime/objectMonitor.hpp"
#include "runtime/os.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/macros.hpp"
#if INCLUDE_ALL_GCS
#include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
#include "gc_implementation/g1/heapRegion.hpp"
#endif // INCLUDE_ALL_GCS
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
#define STOP(error) stop(error)
#else
#define BLOCK_COMMENT(str) block_comment(str)
#define STOP(error) block_comment(error); stop(error)
#endif
#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
#ifdef ASSERT
bool AbstractAssembler::pd_check_instruction_mark() { return true; }
#endif
static Assembler::Condition reverse[] = {
    Assembler::noOverflow     /* overflow      = 0x0 */ ,
    Assembler::overflow       /* noOverflow    = 0x1 */ ,
    Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
    Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
    Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
    Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
    Assembler::above          /* belowEqual    = 0x6 */ ,
    Assembler::belowEqual     /* above         = 0x7 */ ,
    Assembler::positive       /* negative      = 0x8 */ ,
    Assembler::negative       /* positive      = 0x9 */ ,
    Assembler::noParity       /* parity        = 0xa */ ,
    Assembler::parity         /* noParity      = 0xb */ ,
    Assembler::greaterEqual   /* less          = 0xc */ ,
    Assembler::less           /* greaterEqual  = 0xd */ ,
    Assembler::greater        /* lessEqual     = 0xe */ ,
    Assembler::lessEqual      /* greater       = 0xf, */
};
#ifndef _LP64
Address MacroAssembler::as_Address(AddressLiteral adr) {
  return Address(adr.target(), adr.rspec());
}
Address MacroAssembler::as_Address(ArrayAddress adr) {
  return Address::make_array(adr);
}
void MacroAssembler::call_VM_leaf_base(address entry_point,
                                       int number_of_arguments) {
  call(RuntimeAddress(entry_point));
  increment(rsp, number_of_arguments * wordSize);
}
void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
  cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
}
void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
  cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
}
void MacroAssembler::cmpoop(Address src1, jobject obj) {
  cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
}
void MacroAssembler::cmpoop(Register src1, jobject obj) {
  cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
}
void MacroAssembler::extend_sign(Register hi, Register lo) {
  if (VM_Version::is_P6() && hi == rdx && lo == rax) {
    cdql();
  } else {
    movl(hi, lo);
    sarl(hi, 31);
  }
}
void MacroAssembler::jC2(Register tmp, Label& L) {
  save_rax(tmp);
  fwait(); fnstsw_ax();
  sahf();
  restore_rax(tmp);
  jcc(Assembler::parity, L);
}
void MacroAssembler::jnC2(Register tmp, Label& L) {
  save_rax(tmp);
  fwait(); fnstsw_ax();
  sahf();
  restore_rax(tmp);
  jcc(Assembler::noParity, L);
}
void MacroAssembler::jump(ArrayAddress entry) {
  jmp(as_Address(entry));
}
void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  Label high, low, done;
  cmpl(x_hi, y_hi);
  jcc(Assembler::less, low);
  jcc(Assembler::greater, high);
  xorl(x_hi, x_hi);
  cmpl(x_lo, y_lo);
  jcc(Assembler::below, low);
  jcc(Assembler::equal, done);
  bind(high);
  xorl(x_hi, x_hi);
  increment(x_hi);
  jmp(done);
  bind(low);
  xorl(x_hi, x_hi);
  decrementl(x_hi);
  bind(done);
}
void MacroAssembler::lea(Register dst, AddressLiteral src) {
    mov_literal32(dst, (int32_t)src.target(), src.rspec());
}
void MacroAssembler::lea(Address dst, AddressLiteral adr) {
  mov_literal32(dst, (int32_t) adr.target(), adr.rspec());
}
void MacroAssembler::leave() {
  mov(rsp, rbp);
  pop(rbp);
}
void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
  Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
  Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
  Label quick;
  movl(rbx, x_hi);
  movl(rcx, y_hi);
  movl(rax, rbx);
  orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
  jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
  mull(y_lo);                                    // x_hi * y_lo
  movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
  movl(rax, x_lo);
  mull(rcx);                                     // x_lo * y_hi
  addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
  bind(quick);                                   // note: rbx, = 0 if quick multiply!
  movl(rax, x_lo);
  mull(y_lo);                                    // x_lo * y_lo
  addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
}
void MacroAssembler::lneg(Register hi, Register lo) {
  negl(lo);
  adcl(hi, 0);
  negl(hi);
}
void MacroAssembler::lshl(Register hi, Register lo) {
  assert(hi != rcx, "must not use rcx");
  assert(lo != rcx, "must not use rcx");
  const Register s = rcx;                        // shift count
  const int      n = BitsPerWord;
  Label L;
  andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  cmpl(s, n);                                    // if (s < n)
  jcc(Assembler::less, L);                       // else (s >= n)
  movl(hi, lo);                                  // x := x << n
  xorl(lo, lo);
  bind(L);                                       // s (mod n) < n
  shldl(hi, lo);                                 // x := x << s
  shll(lo);
}
void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
  assert(hi != rcx, "must not use rcx");
  assert(lo != rcx, "must not use rcx");
  const Register s = rcx;                        // shift count
  const int      n = BitsPerWord;
  Label L;
  andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  cmpl(s, n);                                    // if (s < n)
  jcc(Assembler::less, L);                       // else (s >= n)
  movl(lo, hi);                                  // x := x >> n
  if (sign_extension) sarl(hi, 31);
  else                xorl(hi, hi);
  bind(L);                                       // s (mod n) < n
  shrdl(lo, hi);                                 // x := x >> s
  if (sign_extension) sarl(hi);
  else                shrl(hi);
}
void MacroAssembler::movoop(Register dst, jobject obj) {
  mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
}
void MacroAssembler::movoop(Address dst, jobject obj) {
  mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
}
void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
}
void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
  mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
}
void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
  if (src.is_lval()) {
    mov_literal32(dst, (intptr_t)src.target(), src.rspec());
  } else {
    movl(dst, as_Address(src));
  }
}
void MacroAssembler::movptr(ArrayAddress dst, Register src) {
  movl(as_Address(dst), src);
}
void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  movl(dst, as_Address(src));
}
void MacroAssembler::movptr(Address dst, intptr_t src) {
  movl(dst, src);
}
void MacroAssembler::pop_callee_saved_registers() {
  pop(rcx);
  pop(rdx);
  pop(rdi);
  pop(rsi);
}
void MacroAssembler::pop_fTOS() {
  fld_d(Address(rsp, 0));
  addl(rsp, 2 * wordSize);
}
void MacroAssembler::push_callee_saved_registers() {
  push(rsi);
  push(rdi);
  push(rdx);
  push(rcx);
}
void MacroAssembler::push_fTOS() {
  subl(rsp, 2 * wordSize);
  fstp_d(Address(rsp, 0));
}
void MacroAssembler::pushoop(jobject obj) {
  push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
}
void MacroAssembler::pushklass(Metadata* obj) {
  push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
}
void MacroAssembler::pushptr(AddressLiteral src) {
  if (src.is_lval()) {
    push_literal32((int32_t)src.target(), src.rspec());
  } else {
    pushl(as_Address(src));
  }
}
void MacroAssembler::set_word_if_not_zero(Register dst) {
  xorl(dst, dst);
  set_byte_if_not_zero(dst);
}
static void pass_arg0(MacroAssembler* masm, Register arg) {
  masm->push(arg);
}
static void pass_arg1(MacroAssembler* masm, Register arg) {
  masm->push(arg);
}
static void pass_arg2(MacroAssembler* masm, Register arg) {
  masm->push(arg);
}
static void pass_arg3(MacroAssembler* masm, Register arg) {
  masm->push(arg);
}
#ifndef PRODUCT
extern "C" void findpc(intptr_t x);
#endif
void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
  JavaThread* thread = JavaThread::current();
  JavaThreadState saved_state = thread->thread_state();
  thread->set_thread_state(_thread_in_vm);
  if (ShowMessageBoxOnError) {
    JavaThread* thread = JavaThread::current();
    JavaThreadState saved_state = thread->thread_state();
    thread->set_thread_state(_thread_in_vm);
    if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
      ttyLocker ttyl;
      BytecodeCounter::print();
    }
    if (os::message_box(msg, "Execution stopped, print registers?")) {
      print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
      BREAKPOINT;
    }
  } else {
    ttyLocker ttyl;
    ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
  }
    assert(false, err_msg("DEBUG MESSAGE: %s", msg));
  ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
}
void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
  ttyLocker ttyl;
  FlagSetting fs(Debugging, true);
  tty->print_cr("eip = 0x%08x", eip);
#ifndef PRODUCT
  if ((WizardMode || Verbose) && PrintMiscellaneous) {
    tty->cr();
    findpc(eip);
    tty->cr();
  }
#endif
#define PRINT_REG(rax) \
  { tty->print("%s = ", #rax); os::print_location(tty, rax); }
  PRINT_REG(rax);
  PRINT_REG(rbx);
  PRINT_REG(rcx);
  PRINT_REG(rdx);
  PRINT_REG(rdi);
  PRINT_REG(rsi);
  PRINT_REG(rbp);
  PRINT_REG(rsp);
#undef PRINT_REG
  int* dump_sp = (int*) rsp;
  for (int col1 = 0; col1 < 8; col1++) {
    tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
    os::print_location(tty, *dump_sp++);
  }
  for (int row = 0; row < 16; row++) {
    tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
    for (int col = 0; col < 8; col++) {
      tty->print(" 0x%08x", *dump_sp++);
    }
    tty->cr();
  }
  Disassembler::decode((address)eip-64, (address)eip);
  tty->print_cr("--------");
  Disassembler::decode((address)eip, (address)eip+32);
}
void MacroAssembler::stop(const char* msg) {
  ExternalAddress message((address)msg);
  pushptr(message.addr());
  { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  pusha();                                            // push registers
  call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
  hlt();
}
void MacroAssembler::warn(const char* msg) {
  push_CPU_state();
  ExternalAddress message((address) msg);
  pushptr(message.addr());
  call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  addl(rsp, wordSize);       // discard argument
  pop_CPU_state();
}
void MacroAssembler::print_state() {
  { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  pusha();                                            // push registers
  push_CPU_state();
  call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
  pop_CPU_state();
  popa();
  addl(rsp, wordSize);
}
#else // _LP64
Address MacroAssembler::as_Address(AddressLiteral adr) {
  assert(!adr.is_lval(), "must be rval");
  assert(reachable(adr), "must be");
  return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());
}
Address MacroAssembler::as_Address(ArrayAddress adr) {
  AddressLiteral base = adr.base();
  lea(rscratch1, base);
  Address index = adr.index();
  assert(index._disp == 0, "must not have disp"); // maybe it can?
  Address array(rscratch1, index._index, index._scale, index._disp);
  return array;
}
void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
  Label L, E;
#ifdef _WIN64
  assert(num_args <= 4, "only register arguments supported");
  subq(rsp,  frame::arg_reg_save_area_bytes);
#endif
  testl(rsp, 15);
  jcc(Assembler::zero, L);
  subq(rsp, 8);
  {
    call(RuntimeAddress(entry_point));
  }
  addq(rsp, 8);
  jmp(E);
  bind(L);
  {
    call(RuntimeAddress(entry_point));
  }
  bind(E);
#ifdef _WIN64
  addq(rsp, frame::arg_reg_save_area_bytes);
#endif
}
void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {
  assert(!src2.is_lval(), "should use cmpptr");
  if (reachable(src2)) {
    cmpq(src1, as_Address(src2));
  } else {
    lea(rscratch1, src2);
    Assembler::cmpq(src1, Address(rscratch1, 0));
  }
}
int MacroAssembler::corrected_idivq(Register reg) {
  assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
  static const int64_t min_long = 0x8000000000000000;
  Label normal_case, special_case;
  cmp64(rax, ExternalAddress((address) &min_long));
  jcc(Assembler::notEqual, normal_case);
  xorl(rdx, rdx); // prepare rdx for possible special case (where
  cmpq(reg, -1);
  jcc(Assembler::equal, special_case);
  bind(normal_case);
  cdqq();
  int idivq_offset = offset();
  idivq(reg);
  bind(special_case);
  return idivq_offset;
}
void MacroAssembler::decrementq(Register reg, int value) {
  if (value == min_jint) { subq(reg, value); return; }
  if (value <  0) { incrementq(reg, -value); return; }
  if (value == 0) {                        ; return; }
  if (value == 1 && UseIncDec) { decq(reg) ; return; }
}
void MacroAssembler::decrementq(Address dst, int value) {
  if (value == min_jint) { subq(dst, value); return; }
  if (value <  0) { incrementq(dst, -value); return; }
  if (value == 0) {                        ; return; }
  if (value == 1 && UseIncDec) { decq(dst) ; return; }
}
void MacroAssembler::incrementq(AddressLiteral dst) {
  if (reachable(dst)) {
    incrementq(as_Address(dst));
  } else {
    lea(rscratch1, dst);
    incrementq(Address(rscratch1, 0));
  }
}
void MacroAssembler::incrementq(Register reg, int value) {
  if (value == min_jint) { addq(reg, value); return; }
  if (value <  0) { decrementq(reg, -value); return; }
  if (value == 0) {                        ; return; }
  if (value == 1 && UseIncDec) { incq(reg) ; return; }
}
void MacroAssembler::incrementq(Address dst, int value) {
  if (value == min_jint) { addq(dst, value); return; }
  if (value <  0) { decrementq(dst, -value); return; }
  if (value == 0) {                        ; return; }
  if (value == 1 && UseIncDec) { incq(dst) ; return; }
}
void MacroAssembler::jump(ArrayAddress entry) {
  lea(rscratch1, entry.base());
  Address dispatch = entry.index();
  assert(dispatch._base == noreg, "must be");
  dispatch._base = rscratch1;
  jmp(dispatch);
}
void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  ShouldNotReachHere(); // 64bit doesn't use two regs
  cmpq(x_lo, y_lo);
}
void MacroAssembler::lea(Register dst, AddressLiteral src) {
    mov_literal64(dst, (intptr_t)src.target(), src.rspec());
}
void MacroAssembler::lea(Address dst, AddressLiteral adr) {
  mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec());
  movptr(dst, rscratch1);
}
void MacroAssembler::leave() {
  emit_int8((unsigned char)0xC9); // LEAVE
}
void MacroAssembler::lneg(Register hi, Register lo) {
  ShouldNotReachHere(); // 64bit doesn't use two regs
  negq(lo);
}
void MacroAssembler::movoop(Register dst, jobject obj) {
  mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
}
void MacroAssembler::movoop(Address dst, jobject obj) {
  mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  movq(dst, rscratch1);
}
void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
}
void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
  mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  movq(dst, rscratch1);
}
void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
  if (src.is_lval()) {
    mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  } else {
    if (reachable(src)) {
      movq(dst, as_Address(src));
    } else {
      lea(scratch, src);
      movq(dst, Address(scratch, 0));
    }
  }
}
void MacroAssembler::movptr(ArrayAddress dst, Register src) {
  movq(as_Address(dst), src);
}
void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  movq(dst, as_Address(src));
}
void MacroAssembler::movptr(Address dst, intptr_t src) {
  mov64(rscratch1, src);
  movq(dst, rscratch1);
}
void MacroAssembler::movptr(Address dst, int32_t src) {
  movslq(dst, src);
}
void MacroAssembler::movptr(Register dst, int32_t src) {
  mov64(dst, (intptr_t)src);
}
void MacroAssembler::pushoop(jobject obj) {
  movoop(rscratch1, obj);
  push(rscratch1);
}
void MacroAssembler::pushklass(Metadata* obj) {
  mov_metadata(rscratch1, obj);
  push(rscratch1);
}
void MacroAssembler::pushptr(AddressLiteral src) {
  lea(rscratch1, src);
  if (src.is_lval()) {
    push(rscratch1);
  } else {
    pushq(Address(rscratch1, 0));
  }
}
void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
  movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
  if (clear_fp) {
    movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
  }
  movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
}
void MacroAssembler::set_last_Java_frame(Register last_java_sp,
                                         Register last_java_fp,
                                         address  last_java_pc) {
  if (!last_java_sp->is_valid()) {
    last_java_sp = rsp;
  }
  if (last_java_fp->is_valid()) {
    movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()),
           last_java_fp);
  }
  if (last_java_pc != NULL) {
    Address java_pc(r15_thread,
                    JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
    lea(rscratch1, InternalAddress(last_java_pc));
    movptr(java_pc, rscratch1);
  }
  movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
}
static void pass_arg0(MacroAssembler* masm, Register arg) {
  if (c_rarg0 != arg ) {
    masm->mov(c_rarg0, arg);
  }
}
static void pass_arg1(MacroAssembler* masm, Register arg) {
  if (c_rarg1 != arg ) {
    masm->mov(c_rarg1, arg);
  }
}
static void pass_arg2(MacroAssembler* masm, Register arg) {
  if (c_rarg2 != arg ) {
    masm->mov(c_rarg2, arg);
  }
}
static void pass_arg3(MacroAssembler* masm, Register arg) {
  if (c_rarg3 != arg ) {
    masm->mov(c_rarg3, arg);
  }
}
void MacroAssembler::stop(const char* msg) {
  address rip = pc();
  pusha(); // get regs on stack
  lea(c_rarg0, ExternalAddress((address) msg));
  lea(c_rarg1, InternalAddress(rip));
  movq(c_rarg2, rsp); // pass pointer to regs array
  andq(rsp, -16); // align stack as required by ABI
  call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
  hlt();
}
void MacroAssembler::warn(const char* msg) {
  push(rbp);
  movq(rbp, rsp);
  andq(rsp, -16);     // align stack as required by push_CPU_state and call
  push_CPU_state();   // keeps alignment at 16 bytes
  lea(c_rarg0, ExternalAddress((address) msg));
  call_VM_leaf(CAST_FROM_FN_PTR(address, warning), c_rarg0);
  pop_CPU_state();
  mov(rsp, rbp);
  pop(rbp);
}
void MacroAssembler::print_state() {
  address rip = pc();
  pusha();            // get regs on stack
  push(rbp);
  movq(rbp, rsp);
  andq(rsp, -16);     // align stack as required by push_CPU_state and call
  push_CPU_state();   // keeps alignment at 16 bytes
  lea(c_rarg0, InternalAddress(rip));
  lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
  call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
  pop_CPU_state();
  mov(rsp, rbp);
  pop(rbp);
  popa();
}
#ifndef PRODUCT
extern "C" void findpc(intptr_t x);
#endif
void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
  if (ShowMessageBoxOnError) {
    JavaThread* thread = JavaThread::current();
    JavaThreadState saved_state = thread->thread_state();
    thread->set_thread_state(_thread_in_vm);
#ifndef PRODUCT
    if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
      ttyLocker ttyl;
      BytecodeCounter::print();
    }
#endif
    if (os::message_box(msg, "Execution stopped, print registers?")) {
      print_state64(pc, regs);
      BREAKPOINT;
      assert(false, "start up GDB");
    }
    ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
  } else {
    ttyLocker ttyl;
    ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
                    msg);
    assert(false, err_msg("DEBUG MESSAGE: %s", msg));
  }
}
void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
  ttyLocker ttyl;
  FlagSetting fs(Debugging, true);
  tty->print_cr("rip = 0x%016lx", pc);
#ifndef PRODUCT
  tty->cr();
  findpc(pc);
  tty->cr();
#endif
#define PRINT_REG(rax, value) \
  { tty->print("%s = ", #rax); os::print_location(tty, value); }
  PRINT_REG(rax, regs[15]);
  PRINT_REG(rbx, regs[12]);
  PRINT_REG(rcx, regs[14]);
  PRINT_REG(rdx, regs[13]);
  PRINT_REG(rdi, regs[8]);
  PRINT_REG(rsi, regs[9]);
  PRINT_REG(rbp, regs[10]);
  PRINT_REG(rsp, regs[11]);
  PRINT_REG(r8 , regs[7]);
  PRINT_REG(r9 , regs[6]);
  PRINT_REG(r10, regs[5]);
  PRINT_REG(r11, regs[4]);
  PRINT_REG(r12, regs[3]);
  PRINT_REG(r13, regs[2]);
  PRINT_REG(r14, regs[1]);
  PRINT_REG(r15, regs[0]);
#undef PRINT_REG
  int64_t* rsp = (int64_t*) regs[11];
  int64_t* dump_sp = rsp;
  for (int col1 = 0; col1 < 8; col1++) {
    tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (int64_t)dump_sp);
    os::print_location(tty, *dump_sp++);
  }
  for (int row = 0; row < 25; row++) {
    tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (int64_t)dump_sp);
    for (int col = 0; col < 4; col++) {
      tty->print(" 0x%016lx", *dump_sp++);
    }
    tty->cr();
  }
  Disassembler::decode((address)pc-64, (address)pc);
  tty->print_cr("--------");
  Disassembler::decode((address)pc, (address)pc+32);
}
#endif // _LP64
void MacroAssembler::addptr(Register dst, int32_t imm32) {
  LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
}
void MacroAssembler::addptr(Register dst, Register src) {
  LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
}
void MacroAssembler::addptr(Address dst, Register src) {
  LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
}
void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) {
  if (reachable(src)) {
    Assembler::addsd(dst, as_Address(src));
  } else {
    lea(rscratch1, src);
    Assembler::addsd(dst, Address(rscratch1, 0));
  }
}
void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) {
  if (reachable(src)) {
    addss(dst, as_Address(src));
  } else {
    lea(rscratch1, src);
    addss(dst, Address(rscratch1, 0));
  }
}
void MacroAssembler::align(int modulus) {
  if (offset() % modulus != 0) {
    nop(modulus - (offset() % modulus));
  }
}
void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) {
  assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
  if (reachable(src)) {
    Assembler::andpd(dst, as_Address(src));
  } else {
    lea(rscratch1, src);
    Assembler::andpd(dst, Address(rscratch1, 0));
  }
}
void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) {
  assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
  if (reachable(src)) {
    Assembler::andps(dst, as_Address(src));
  } else {
    lea(rscratch1, src);
    Assembler::andps(dst, Address(rscratch1, 0));
  }
}
void MacroAssembler::andptr(Register dst, int32_t imm32) {
  LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
}
void MacroAssembler::atomic_incl(Address counter_addr) {
  if (os::is_MP())
    lock();
  incrementl(counter_addr);
}
void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) {
  if (reachable(counter_addr)) {
    atomic_incl(as_Address(counter_addr));
  } else {
    lea(scr, counter_addr);
    atomic_incl(Address(scr, 0));
  }
}
#ifdef _LP64
void MacroAssembler::atomic_incq(Address counter_addr) {
  if (os::is_MP())
    lock();
  incrementq(counter_addr);
}
void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) {
  if (reachable(counter_addr)) {
    atomic_incq(as_Address(counter_addr));
  } else {
    lea(scr, counter_addr);
    atomic_incq(Address(scr, 0));
  }
}
#endif
void MacroAssembler::bang_stack_size(Register size, Register tmp) {
  movptr(tmp, rsp);
  Label loop;
  bind(loop);
  movl(Address(tmp, (-os::vm_page_size())), size );
  subptr(tmp, os::vm_page_size());
  subl(size, os::vm_page_size());
  jcc(Assembler::greater, loop);
  for (int i = 1; i < StackShadowPages; i++) {
    movptr(Address(tmp, (-i*os::vm_page_size())), size );
  }
}
int MacroAssembler::biased_locking_enter(Register lock_reg,
                                         Register obj_reg,
                                         Register swap_reg,
                                         Register tmp_reg,
                                         bool swap_reg_contains_mark,
                                         Label& done,
                                         Label* slow_case,
                                         BiasedLockingCounters* counters) {
  assert(UseBiasedLocking, "why call this otherwise?");
  assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");
  LP64_ONLY( assert(tmp_reg != noreg, "tmp_reg must be supplied"); )
  bool need_tmp_reg = false;
  if (tmp_reg == noreg) {
    need_tmp_reg = true;
    tmp_reg = lock_reg;
    assert_different_registers(lock_reg, obj_reg, swap_reg);
  } else {
    assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
  }
  assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
  Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());
  Address saved_mark_addr(lock_reg, 0);
  if (PrintBiasedLockingStatistics && counters == NULL) {
    counters = BiasedLocking::counters();
  }
  Label cas_label;
  int null_check_offset = -1;
  if (!swap_reg_contains_mark) {
    null_check_offset = offset();
    movptr(swap_reg, mark_addr);
  }
  if (need_tmp_reg) {
    push(tmp_reg);
  }
  movptr(tmp_reg, swap_reg);
  andptr(tmp_reg, markOopDesc::biased_lock_mask_in_place);
  cmpptr(tmp_reg, markOopDesc::biased_lock_pattern);
  if (need_tmp_reg) {
    pop(tmp_reg);
  }
  jcc(Assembler::notEqual, cas_label);
#ifndef _LP64
  movptr(saved_mark_addr, swap_reg);
#endif
  if (need_tmp_reg) {
    push(tmp_reg);
  }
  if (swap_reg_contains_mark) {
    null_check_offset = offset();
  }
  load_prototype_header(tmp_reg, obj_reg);
#ifdef _LP64
  orptr(tmp_reg, r15_thread);
  xorptr(tmp_reg, swap_reg);
  Register header_reg = tmp_reg;
#else
  xorptr(tmp_reg, swap_reg);
  get_thread(swap_reg);
  xorptr(swap_reg, tmp_reg);
  Register header_reg = swap_reg;
#endif
  andptr(header_reg, ~((int) markOopDesc::age_mask_in_place));
  if (need_tmp_reg) {
    pop(tmp_reg);
  }
  if (counters != NULL) {
    cond_inc32(Assembler::zero,
               ExternalAddress((address) counters->biased_lock_entry_count_addr()));
  }
  jcc(Assembler::equal, done);
  Label try_revoke_bias;
  Label try_rebias;
  testptr(header_reg, markOopDesc::biased_lock_mask_in_place);
  jccb(Assembler::notZero, try_revoke_bias);
  testptr(header_reg, markOopDesc::epoch_mask_in_place);
  jccb(Assembler::notZero, try_rebias);
  NOT_LP64( movptr(swap_reg, saved_mark_addr); )
  andptr(swap_reg,
         markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
  if (need_tmp_reg) {
    push(tmp_reg);
  }
#ifdef _LP64
  movptr(tmp_reg, swap_reg);
  orptr(tmp_reg, r15_thread);
#else
  get_thread(tmp_reg);
  orptr(tmp_reg, swap_reg);
#endif
  if (os::is_MP()) {
    lock();
  }
  cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
  if (need_tmp_reg) {
    pop(tmp_reg);
  }
  if (counters != NULL) {
    cond_inc32(Assembler::zero,
               ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
  }
  if (slow_case != NULL) {
    jcc(Assembler::notZero, *slow_case);
  }
  jmp(done);
  bind(try_rebias);
  if (need_tmp_reg) {
    push(tmp_reg);
  }
  load_prototype_header(tmp_reg, obj_reg);
#ifdef _LP64
  orptr(tmp_reg, r15_thread);
#else
  get_thread(swap_reg);
  orptr(tmp_reg, swap_reg);
  movptr(swap_reg, saved_mark_addr);
#endif
  if (os::is_MP()) {
    lock();
  }
  cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
  if (need_tmp_reg) {
    pop(tmp_reg);
  }
  if (counters != NULL) {
    cond_inc32(Assembler::zero,
               ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));
  }
  if (slow_case != NULL) {
    jcc(Assembler::notZero, *slow_case);
  }
  jmp(done);
  bind(try_revoke_bias);
  NOT_LP64( movptr(swap_reg, saved_mark_addr); )
  if (need_tmp_reg) {
    push(tmp_reg);
  }
  load_prototype_header(tmp_reg, obj_reg);
  if (os::is_MP()) {
    lock();
  }
  cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
  if (need_tmp_reg) {
    pop(tmp_reg);
  }
  if (counters != NULL) {
    cond_inc32(Assembler::zero,
               ExternalAddress((address) counters->revoked_lock_entry_count_addr()));
  }
  bind(cas_label);
  return null_check_offset;
}
void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
  assert(UseBiasedLocking, "why call this otherwise?");
  movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
  andptr(temp_reg, markOopDesc::biased_lock_mask_in_place);
  cmpptr(temp_reg, markOopDesc::biased_lock_pattern);
  jcc(Assembler::equal, done);
}
#ifdef COMPILER2
#if INCLUDE_RTM_OPT
void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters) {
  atomic_incptr(Address(rtm_counters, RTMLockingCounters::abort_count_offset()));
  if (PrintPreciseRTMLockingStatistics) {
    for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) {
      Label check_abort;
      testl(abort_status, (1<<i));
      jccb(Assembler::equal, check_abort);
      atomic_incptr(Address(rtm_counters, RTMLockingCounters::abortX_count_offset() + (i * sizeof(uintx))));
      bind(check_abort);
    }
  }
}
void MacroAssembler::branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel) {
  assert(tmp == rax, "");
  assert(scr == rdx, "");
  rdtsc(); // modifies EDX:EAX
  andptr(tmp, count-1);
  jccb(Assembler::notZero, brLabel);
}
void MacroAssembler::rtm_abort_ratio_calculation(Register tmpReg,
                                                 Register rtm_counters_Reg,
                                                 RTMLockingCounters* rtm_counters,
                                                 Metadata* method_data) {
  Label L_done, L_check_always_rtm1, L_check_always_rtm2;
  if (RTMLockingCalculationDelay > 0) {
    movptr(tmpReg, ExternalAddress((address) RTMLockingCounters::rtm_calculation_flag_addr()), tmpReg);
    testptr(tmpReg, tmpReg);
    jccb(Assembler::equal, L_done);
  }
  movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::abort_count_offset()));
  cmpptr(tmpReg, RTMAbortThreshold);
  jccb(Assembler::below, L_check_always_rtm2);
  imulptr(tmpReg, tmpReg, 100);
  Register scrReg = rtm_counters_Reg;
  movptr(scrReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
  imulptr(scrReg, scrReg, RTMTotalCountIncrRate);
  imulptr(scrReg, scrReg, RTMAbortRatio);
  cmpptr(tmpReg, scrReg);
  jccb(Assembler::below, L_check_always_rtm1);
  if (method_data != NULL) {
    mov_metadata(tmpReg, method_data);
    if (os::is_MP()) {
      lock();
    }
    orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), NoRTM);
  }
  jmpb(L_done);
  bind(L_check_always_rtm1);
  lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
  bind(L_check_always_rtm2);
  movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
  cmpptr(tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate);
  jccb(Assembler::below, L_done);
  if (method_data != NULL) {
    mov_metadata(tmpReg, method_data);
    if (os::is_MP()) {
      lock();
    }
    orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), UseRTM);
  }
  bind(L_done);
}
void MacroAssembler::rtm_profiling(Register abort_status_Reg,
                                   Register rtm_counters_Reg,
                                   RTMLockingCounters* rtm_counters,
                                   Metadata* method_data,
                                   bool profile_rtm) {
  assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
  lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
  rtm_counters_update(abort_status_Reg, rtm_counters_Reg);
  if (profile_rtm) {
    if (RTMRetryCount > 0) {
      push(abort_status_Reg);
    }
    assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
    rtm_abort_ratio_calculation(abort_status_Reg, rtm_counters_Reg, rtm_counters, method_data);
    if (RTMRetryCount > 0) {
      pop(abort_status_Reg);
    }
  }
}
void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg, Label& retryLabel) {
  Label doneRetry;
  assert(abort_status_Reg == rax, "");
  andptr(abort_status_Reg, 0x6);
  jccb(Assembler::zero, doneRetry);
  testl(retry_count_Reg, retry_count_Reg);
  jccb(Assembler::zero, doneRetry);
  pause();
  decrementl(retry_count_Reg);
  jmp(retryLabel);
  bind(doneRetry);
}
void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register box_Reg,
                                            Register tmp_Reg, Register scr_Reg, Label& retryLabel) {
  Label SpinLoop, SpinExit, doneRetry;
  int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value;
  testl(retry_count_Reg, retry_count_Reg);
  jccb(Assembler::zero, doneRetry);
  decrementl(retry_count_Reg);
  movptr(scr_Reg, RTMSpinLoopCount);
  bind(SpinLoop);
  pause();
  decrementl(scr_Reg);
  jccb(Assembler::lessEqual, SpinExit);
  movptr(tmp_Reg, Address(box_Reg, owner_offset));
  testptr(tmp_Reg, tmp_Reg);
  jccb(Assembler::notZero, SpinLoop);
  bind(SpinExit);
  jmp(retryLabel);
  bind(doneRetry);
  incrementl(retry_count_Reg); // clear z flag
}
void MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Register scrReg,
                                       Register retry_on_abort_count_Reg,
                                       RTMLockingCounters* stack_rtm_counters,
                                       Metadata* method_data, bool profile_rtm,
                                       Label& DONE_LABEL, Label& IsInflated) {
  assert(UseRTMForStackLocks, "why call this otherwise?");
  assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
  assert(tmpReg == rax, "");
  assert(scrReg == rdx, "");
  Label L_rtm_retry, L_decrement_retry, L_on_abort;
  if (RTMRetryCount > 0) {
    movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
    bind(L_rtm_retry);
  }
  movptr(tmpReg, Address(objReg, 0));
  testptr(tmpReg, markOopDesc::monitor_value);  // inflated vs stack-locked|neutral|biased
  jcc(Assembler::notZero, IsInflated);
  if (PrintPreciseRTMLockingStatistics || profile_rtm) {
    Label L_noincrement;
    if (RTMTotalCountIncrRate > 1) {
      branch_on_random_using_rdtsc(tmpReg, scrReg, (int)RTMTotalCountIncrRate, L_noincrement);
    }
    assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM");
    atomic_incptr(ExternalAddress((address)stack_rtm_counters->total_count_addr()), scrReg);
    bind(L_noincrement);
  }
  xbegin(L_on_abort);
  movptr(tmpReg, Address(objReg, 0));       // fetch markword
  andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
  cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
  jcc(Assembler::equal, DONE_LABEL);        // all done if unlocked
  Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
  if (UseRTMXendForLockBusy) {
    xend();
    movptr(abort_status_Reg, 0x2);   // Set the abort status to 2 (so we can retry)
    jmp(L_decrement_retry);
  }
  else {
    xabort(0);
  }
  bind(L_on_abort);
  if (PrintPreciseRTMLockingStatistics || profile_rtm) {
    rtm_profiling(abort_status_Reg, scrReg, stack_rtm_counters, method_data, profile_rtm);
  }
  bind(L_decrement_retry);
  if (RTMRetryCount > 0) {
    rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
  }
}
void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg,
                                          Register scrReg, Register retry_on_busy_count_Reg,
                                          Register retry_on_abort_count_Reg,
                                          RTMLockingCounters* rtm_counters,
                                          Metadata* method_data, bool profile_rtm,
                                          Label& DONE_LABEL) {
  assert(UseRTMLocking, "why call this otherwise?");
  assert(tmpReg == rax, "");
  assert(scrReg == rdx, "");
  Label L_rtm_retry, L_decrement_retry, L_on_abort;
  int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value;
  movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
  movptr(boxReg, tmpReg); // Save ObjectMonitor address
  if (RTMRetryCount > 0) {
    movl(retry_on_busy_count_Reg, RTMRetryCount);  // Retry on lock busy
    movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
    bind(L_rtm_retry);
  }
  if (PrintPreciseRTMLockingStatistics || profile_rtm) {
    Label L_noincrement;
    if (RTMTotalCountIncrRate > 1) {
      branch_on_random_using_rdtsc(tmpReg, scrReg, (int)RTMTotalCountIncrRate, L_noincrement);
    }
    assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
    atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg);
    bind(L_noincrement);
  }
  xbegin(L_on_abort);
  movptr(tmpReg, Address(objReg, 0));
  movptr(tmpReg, Address(tmpReg, owner_offset));
  testptr(tmpReg, tmpReg);
  jcc(Assembler::zero, DONE_LABEL);
  if (UseRTMXendForLockBusy) {
    xend();
    jmp(L_decrement_retry);
  }
  else {
    xabort(0);
  }
  bind(L_on_abort);
  Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
  if (PrintPreciseRTMLockingStatistics || profile_rtm) {
    rtm_profiling(abort_status_Reg, scrReg, rtm_counters, method_data, profile_rtm);
  }
  if (RTMRetryCount > 0) {
    rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
  }
  movptr(tmpReg, Address(boxReg, owner_offset)) ;
  testptr(tmpReg, tmpReg) ;
  jccb(Assembler::notZero, L_decrement_retry) ;
#ifdef _LP64
  Register threadReg = r15_thread;
#else
  get_thread(scrReg);
  Register threadReg = scrReg;
#endif
  if (os::is_MP()) {
    lock();
  }
  cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg
  if (RTMRetryCount > 0) {
    jccb(Assembler::equal, DONE_LABEL) ;
    bind(L_decrement_retry);
    rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry);
  }
  else {
    bind(L_decrement_retry);
  }
}
#endif //  INCLUDE_RTM_OPT
void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
                               Register scrReg, Register cx1Reg, Register cx2Reg,
                               BiasedLockingCounters* counters,
                               RTMLockingCounters* rtm_counters,
                               RTMLockingCounters* stack_rtm_counters,
                               Metadata* method_data,
                               bool use_rtm, bool profile_rtm) {
  assert(tmpReg == rax, "");
  if (use_rtm) {
    assert_different_registers(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg);
  } else {
    assert(cx1Reg == noreg, "");
    assert(cx2Reg == noreg, "");
    assert_different_registers(objReg, boxReg, tmpReg, scrReg);
  }
  if (counters != NULL) {
    atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg);
  }
  if (EmitSync & 1) {
      movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
      cmpptr (rsp, (int32_t)NULL_WORD);
  } else
  if (EmitSync & 2) {
      Label DONE_LABEL ;
      if (UseBiasedLocking) {
         biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, counters);
      }
      movptr(tmpReg, Address(objReg, 0));           // fetch markword
      orptr (tmpReg, 0x1);
      movptr(Address(boxReg, 0), tmpReg);           // Anticipate successful CAS
      if (os::is_MP()) {
        lock();
      }
      cmpxchgptr(boxReg, Address(objReg, 0));       // Updates tmpReg
      jccb(Assembler::equal, DONE_LABEL);
      subptr(tmpReg, rsp);
      andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
      movptr(Address(boxReg, 0), tmpReg);
      bind(DONE_LABEL);
  } else {
    Label IsInflated, DONE_LABEL;
    if (UseBiasedLocking && !UseOptoBiasInlining) {
      biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, counters);
    }
#if INCLUDE_RTM_OPT
    if (UseRTMForStackLocks && use_rtm) {
      rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,
                        stack_rtm_counters, method_data, profile_rtm,
                        DONE_LABEL, IsInflated);
    }
#endif // INCLUDE_RTM_OPT
    movptr(tmpReg, Address(objReg, 0));          // [FETCH]
    testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
    jccb(Assembler::notZero, IsInflated);
    orptr (tmpReg, markOopDesc::unlocked_value);
    movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
    if (os::is_MP()) {
      lock();
    }
    cmpxchgptr(boxReg, Address(objReg, 0));      // Updates tmpReg
    if (counters != NULL) {
      cond_inc32(Assembler::equal,
                 ExternalAddress((address)counters->fast_path_entry_count_addr()));
    }
    jcc(Assembler::equal, DONE_LABEL);           // Success
    subptr(tmpReg, rsp);
    andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
    movptr(Address(boxReg, 0), tmpReg);
    if (counters != NULL) {
      cond_inc32(Assembler::equal,
                 ExternalAddress((address)counters->fast_path_entry_count_addr()));
    }
    jmp(DONE_LABEL);
    bind(IsInflated);
#if INCLUDE_RTM_OPT
    if (use_rtm) {
      rtm_inflated_locking(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg,
                           rtm_counters, method_data, profile_rtm, DONE_LABEL);
    } else {
#endif // INCLUDE_RTM_OPT
#ifndef _LP64
    #define OFFSET_SKEWED(f) ((ObjectMonitor::f ## _offset_in_bytes())-2)
    if (EmitSync & 8192) {
       movptr(Address(boxReg, 0), 3);            // results in ST-before-CAS penalty
       get_thread (scrReg);
       movptr(boxReg, tmpReg);                    // consider: LEA box, [tmp-2]
       movptr(tmpReg, NULL_WORD);                 // consider: xor vs mov
       if (os::is_MP()) {
         lock();
       }
       cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
    } else
    if ((EmitSync & 128) == 0) {                      // avoid ST-before-CAS
       movptr(scrReg, boxReg);
       movptr(boxReg, tmpReg);                   // consider: LEA box, [tmp-2]
       if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
          prefetchw(Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
       }
       if ((EmitSync & 64) == 0) {
         movptr(tmpReg, NULL_WORD);
       } else {
         movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));   // rax, = m->_owner
         testptr(tmpReg, tmpReg);                   // Locked ?
         jccb  (Assembler::notZero, DONE_LABEL);
       }
       if (os::is_MP()) {
         lock();
       }
       cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
       movptr(Address(scrReg, 0), 3);          // box->_displaced_header = 3
       jccb  (Assembler::notZero, DONE_LABEL);
       get_thread (scrReg);                    // beware: clobbers ICCs
       movptr(Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2), scrReg);
       xorptr(boxReg, boxReg);                 // set icc.ZFlag = 1 to indicate success
    } else {
       movptr(Address(boxReg, 0), intptr_t(markOopDesc::unused_mark()));  // results in ST-before-CAS penalty
       movptr(boxReg, tmpReg);
       if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
          prefetchw(Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
       }
       if ((EmitSync & 64) == 0) {
         xorptr  (tmpReg, tmpReg);
       } else {
         movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));   // rax, = m->_owner
         testptr(tmpReg, tmpReg);                   // Locked ?
         jccb  (Assembler::notZero, DONE_LABEL);
       }
       get_thread (scrReg);
       if (os::is_MP()) {
         lock();
       }
       cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
    }
#else // _LP64
    movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
    movptr (boxReg, tmpReg);
    movptr (tmpReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
    testptr(tmpReg, tmpReg);
    jccb   (Assembler::notZero, DONE_LABEL);
    if (os::is_MP()) {
      lock();
    }
    cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
#endif // _LP64
#if INCLUDE_RTM_OPT
    } // use_rtm()
#endif
    bind(DONE_LABEL);
  }
}
void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {
  assert(boxReg == rax, "");
  assert_different_registers(objReg, boxReg, tmpReg);
  if (EmitSync & 4) {
    cmpptr (rsp, 0);
  } else
  if (EmitSync & 8) {
    Label DONE_LABEL;
    if (UseBiasedLocking) {
       biased_locking_exit(objReg, tmpReg, DONE_LABEL);
    }
    movptr(tmpReg, Address(boxReg, 0));
    testptr(tmpReg, tmpReg);
    jccb(Assembler::zero, DONE_LABEL);
    if (os::is_MP()) {
      lock();
    }
    cmpxchgptr(tmpReg, Address(objReg, 0));   // Uses RAX which is box
    bind(DONE_LABEL);
  } else {
    Label DONE_LABEL, Stacked, CheckSucc;
    if (UseBiasedLocking && !UseOptoBiasInlining) {
       biased_locking_exit(objReg, tmpReg, DONE_LABEL);
    }
#if INCLUDE_RTM_OPT
    if (UseRTMForStackLocks && use_rtm) {
      assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
      Label L_regular_unlock;
      movptr(tmpReg, Address(objReg, 0));           // fetch markword
      andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
      cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
      jccb(Assembler::notEqual, L_regular_unlock);  // if !HLE RegularLock
      xend();                                       // otherwise end...
      jmp(DONE_LABEL);                              // ... and we're done
      bind(L_regular_unlock);
    }
#endif
    cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
    jcc   (Assembler::zero, DONE_LABEL);            // 0 indicates recursive stack-lock
    movptr(tmpReg, Address(objReg, 0));             // Examine the object's markword
    testptr(tmpReg, markOopDesc::monitor_value);    // Inflated?
    jccb  (Assembler::zero, Stacked);
#if INCLUDE_RTM_OPT
    if (use_rtm) {
      Label L_regular_inflated_unlock;
      int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value;
      movptr(boxReg, Address(tmpReg, owner_offset));
      testptr(boxReg, boxReg);
      jccb(Assembler::notZero, L_regular_inflated_unlock);
      xend();
      jmpb(DONE_LABEL);
      bind(L_regular_inflated_unlock);
    }
#endif
#ifndef _LP64
    get_thread (boxReg);
    if ((EmitSync & 4096) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
      prefetchw(Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
    }
    if ((EmitSync & 65536) == 0 && (EmitSync & 256)) {
       xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
       orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2));
       orptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2));
       orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2));
       jccb  (Assembler::notZero, DONE_LABEL);
       movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD);
       jmpb  (DONE_LABEL);
    } else {
       xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
       orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2));
       jccb  (Assembler::notZero, DONE_LABEL);
       movptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2));
       orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2));
       jccb  (Assembler::notZero, CheckSucc);
       movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD);
       jmpb  (DONE_LABEL);
    }
    if ((EmitSync & 65536) != 0) {
       Label LSuccess, LGoSlowPath ;
       bind  (CheckSucc);
       if ((EmitSync & 16) == 0) {
          cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD);
          jccb  (Assembler::zero, LGoSlowPath);
       }
       movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD);
       if (os::is_MP()) {
          if (VM_Version::supports_sse2() && 1 == FenceInstruction) {
            mfence();
          } else {
            lock (); addptr(Address(rsp, 0), 0);
          }
       }
       cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0);
       jccb  (Assembler::notZero, LSuccess);
       xorptr(boxReg, boxReg);                  // box is really EAX
       if (os::is_MP()) { lock(); }
       cmpxchgptr(rsp, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
       jccb  (Assembler::notEqual, LSuccess);
       get_thread (boxReg);
       movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), boxReg);
       bind  (LGoSlowPath);
       orptr(boxReg, 1);                      // set ICC.ZF=0 to indicate failure
       jmpb  (DONE_LABEL);
       bind  (LSuccess);
       xorptr(boxReg, boxReg);                 // set ICC.ZF=1 to indicate success
       jmpb  (DONE_LABEL);
    }
    bind (Stacked);
    movptr(tmpReg, Address(boxReg, 0));
    if (os::is_MP()) {
      lock();
    }
    cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
    if ((EmitSync & 65536) == 0) {
       bind (CheckSucc);
    }
#else // _LP64
    movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
    xorptr(boxReg, r15_thread);
    orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2));
    jccb  (Assembler::notZero, DONE_LABEL);
    movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2));
    orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2));
    jccb  (Assembler::notZero, CheckSucc);
    movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD);
    jmpb  (DONE_LABEL);
    if ((EmitSync & 65536) == 0) {
      Label LSuccess, LGoSlowPath ;
      bind  (CheckSucc);
      cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD);
      jccb  (Assembler::zero, LGoSlowPath);
      movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD);
      if (os::is_MP()) {
         lock (); addl (Address(rsp, 0), 0);
      }
      cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD);
      jccb  (Assembler::notZero, LSuccess);
      movptr (boxReg, (int32_t)NULL_WORD);                   // box is really EAX
      if (os::is_MP()) { lock(); }
      cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
      jccb  (Assembler::notEqual, LSuccess);
      bind  (LGoSlowPath);
      orl   (boxReg, 1);                      // set ICC.ZF=0 to indicate failure
      jmpb  (DONE_LABEL);
      bind  (LSuccess);
      testl (boxReg, 0);                      // set ICC.ZF=1 to indicate success
      jmpb  (DONE_LABEL);
    }
    bind  (Stacked);
    movptr(tmpReg, Address (boxReg, 0));      // re-fetch
    if (os::is_MP()) { lock(); }
    cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
    if (EmitSync & 65536) {
       bind (CheckSucc);
    }
#endif
    bind(DONE_LABEL);
    if (EmitSync & 32768) {
       nop();
    }
  }
}
#endif // COMPILER2
void MacroAssembler::c2bool(Register x) {
  andl(x, 0xFF);
  setb(Assembler::notZero, x);
}
void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
  Assembler::call(L, rtype);
}
void MacroAssembler::call(Register entry) {
  Assembler::call(entry);
}
void MacroAssembler::call(AddressLiteral entry) {
  if (reachable(entry)) {
    Assembler::call_literal(entry.target(), entry.rspec());
  } else {
    lea(rscratch1, entry);
    Assembler::call(rscratch1);
  }
}
void MacroAssembler::ic_call(address entry) {
  RelocationHolder rh = virtual_call_Relocation::spec(pc());
  movptr(rax, (intptr_t)Universe::non_oop_word());
  call(AddressLiteral(entry, rh));
}
void MacroAssembler::call_VM(Register oop_result,
                             address entry_point,
                             bool check_exceptions) {
  Label C, E;
  call(C, relocInfo::none);
  jmp(E);
  bind(C);
  call_VM_helper(oop_result, entry_point, 0, check_exceptions);
  ret(0);
  bind(E);
}
void MacroAssembler::call_VM(Register oop_result,
                             address entry_point,
                             Register arg_1,
                             bool check_exceptions) {
  Label C, E;
  call(C, relocInfo::none);
  jmp(E);
  bind(C);
  pass_arg1(this, arg_1);
  call_VM_helper(oop_result, entry_point, 1, check_exceptions);
  ret(0);
  bind(E);
}
void MacroAssembler::call_VM(Register oop_result,
                             address entry_point,
                             Register arg_1,
                             Register arg_2,
                             bool check_exceptions) {
  Label C, E;
  call(C, relocInfo::none);
  jmp(E);
  bind(C);
  LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
  pass_arg2(this, arg_2);
  pass_arg1(this, arg_1);
  call_VM_helper(oop_result, entry_point, 2, check_exceptions);
  ret(0);
  bind(E);
}
void MacroAssembler::call_VM(Register oop_result,
                             address entry_point,
                             Register arg_1,
                             Register arg_2,
                             Register arg_3,
                             bool check_exceptions) {
  Label C, E;
  call(C, relocInfo::none);
  jmp(E);
  bind(C);
  LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
  LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
  pass_arg3(this, arg_3);
  LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
  pass_arg2(this, arg_2);
  pass_arg1(this, arg_1);
  call_VM_helper(oop_result, entry_point, 3, check_exceptions);
  ret(0);
  bind(E);
}
void MacroAssembler::call_VM(Register oop_result,
                             Register last_java_sp,
                             address entry_point,
                             int number_of_arguments,
                             bool check_exceptions) {
  Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
  call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
}
void MacroAssembler::call_VM(Register oop_result,
                             Register last_java_sp,
                             address entry_point,
                             Register arg_1,
                             bool check_exceptions) {
  pass_arg1(this, arg_1);
  call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
}
void MacroAssembler::call_VM(Register oop_result,
                             Register last_java_sp,
                             address entry_point,
                             Register arg_1,
                             Register arg_2,
                             bool check_exceptions) {
  LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
  pass_arg2(this, arg_2);
  pass_arg1(this, arg_1);
  call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
}
void MacroAssembler::call_VM(Register oop_result,
                             Register last_java_sp,
                             address entry_point,
                             Register arg_1,
                             Register arg_2,
                             Register arg_3,
                             bool check_exceptions) {
  LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
  LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
  pass_arg3(this, arg_3);
  LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
  pass_arg2(this, arg_2);
  pass_arg1(this, arg_1);
  call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
}
void MacroAssembler::super_call_VM(Register oop_result,
                                   Register last_java_sp,
                                   address entry_point,
                                   int number_of_arguments,
                                   bool check_exceptions) {
  Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
  MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
}
void MacroAssembler::super_call_VM(Register oop_result,
                                   Register last_java_sp,
                                   address entry_point,
                                   Register arg_1,
                                   bool check_exceptions) {
  pass_arg1(this, arg_1);
  super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
}
void MacroAssembler::super_call_VM(Register oop_result,
                                   Register last_java_sp,
                                   address entry_point,
                                   Register arg_1,
                                   Register arg_2,
                                   bool check_exceptions) {
  LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
  pass_arg2(this, arg_2);
  pass_arg1(this, arg_1);
  super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
}
void MacroAssembler::super_call_VM(Register oop_result,
                                   Register last_java_sp,
                                   address entry_point,
                                   Register arg_1,
                                   Register arg_2,
                                   Register arg_3,
                                   bool check_exceptions) {
  LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
  LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
  pass_arg3(this, arg_3);
  LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
  pass_arg2(this, arg_2);
  pass_arg1(this, arg_1);
  super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
}
void MacroAssembler::call_VM_base(Register oop_result,
                                  Register java_thread,
                                  Register last_java_sp,
                                  address  entry_point,
                                  int      number_of_arguments,
                                  bool     check_exceptions) {
  if (!java_thread->is_valid()) {
#ifdef _LP64
    java_thread = r15_thread;
#else
    java_thread = rdi;
    get_thread(java_thread);
#endif // LP64
  }
  if (!last_java_sp->is_valid()) {
    last_java_sp = rsp;
  }
  assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
  LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
#ifdef ASSERT
  LP64_ONLY(if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
#endif // ASSERT
  assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
  assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
  NOT_LP64(push(java_thread); number_of_arguments++);
  LP64_ONLY(mov(c_rarg0, r15_thread));
  assert(last_java_sp != rbp, "can't use ebp/rbp");
  set_last_Java_frame(java_thread, last_java_sp, rbp, NULL);
  MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
  if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
#ifdef ASSERT
    guarantee(java_thread != rax, "change this code");
    push(rax);
    { Label L;
      get_thread(rax);
      cmpptr(java_thread, rax);
      jcc(Assembler::equal, L);
      STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
      bind(L);
    }
    pop(rax);
#endif
  } else {
    get_thread(java_thread);
  }
  reset_last_Java_frame(java_thread, true);
#ifndef CC_INTERP
  check_and_handle_popframe(java_thread);
  check_and_handle_earlyret(java_thread);
#endif /* CC_INTERP */
  if (check_exceptions) {
    cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
#ifndef _LP64
    jump_cc(Assembler::notEqual,
            RuntimeAddress(StubRoutines::forward_exception_entry()));
#else
    Label ok;
    jcc(Assembler::equal, ok);
    jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
    bind(ok);
#endif // LP64
  }
  if (oop_result->is_valid()) {
    get_vm_result(oop_result, java_thread);
  }
}
void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
#ifdef _LP64
  lea(rax, Address(rsp, wordSize));
#else
  lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
#endif // LP64
  call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
}
void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
  call_VM_leaf_base(entry_point, number_of_arguments);
}
void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
  pass_arg0(this, arg_0);
  call_VM_leaf(entry_point, 1);
}
void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
  LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
  pass_arg1(this, arg_1);
  pass_arg0(this, arg_0);
  call_VM_leaf(entry_point, 2);
}
void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
  LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
  LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
  pass_arg2(this, arg_2);
  LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
  pass_arg1(this, arg_1);
  pass_arg0(this, arg_0);
  call_VM_leaf(entry_point, 3);
}
void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
  pass_arg0(this, arg_0);
  MacroAssembler::call_VM_leaf_base(entry_point, 1);
}
void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
  LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
  pass_arg1(this, arg_1);
  pass_arg0(this, arg_0);
  MacroAssembler::call_VM_leaf_base(entry_point, 2);
}
void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
  LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
  LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
  pass_arg2(this, arg_2);
  LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
  pass_arg1(this, arg_1);
  pass_arg0(this, arg_0);
  MacroAssembler::call_VM_leaf_base(entry_point, 3);
}
void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
  LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg"));
  LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
  LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
  pass_arg3(this, arg_3);
  LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
  LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
  pass_arg2(this, arg_2);
  LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
  pass_arg1(this, arg_1);
  pass_arg0(this, arg_0);
  MacroAssembler::call_VM_leaf_base(entry_point, 4);
}
void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
  movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
  movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
  verify_oop(oop_result, "broken oop in call_VM_base");
}
void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
  movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
  movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
}
void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
}
void MacroAssembler::check_and_handle_popframe(Register java_thread) {
}
void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) {
  if (reachable(src1)) {
    cmpl(as_Address(src1), imm);
  } else {
    lea(rscratch1, src1);
    cmpl(Address(rscratch1, 0), imm);
  }
}
void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {
  assert(!src2.is_lval(), "use cmpptr");
  if (reachable(src2)) {
    cmpl(src1, as_Address(src2));
  } else {
    lea(rscratch1, src2);
    cmpl(src1, Address(rscratch1, 0));
  }
}
void MacroAssembler::cmp32(Register src1, int32_t imm) {
  Assembler::cmpl(src1, imm);
}
void MacroAssembler::cmp32(Register src1, Address src2) {
  Assembler::cmpl(src1, src2);
}
void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
  ucomisd(opr1, opr2);
  Label L;
  if (unordered_is_less) {
    movl(dst, -1);
    jcc(Assembler::parity, L);
    jcc(Assembler::below , L);
    movl(dst, 0);
    jcc(Assembler::equal , L);
    increment(dst);
  } else { // unordered is greater
    movl(dst, 1);
    jcc(Assembler::parity, L);
    jcc(Assembler::above , L);
    movl(dst, 0);
    jcc(Assembler::equal , L);
    decrementl(dst);
  }
  bind(L);
}
void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
  ucomiss(opr1, opr2);
  Label L;
  if (unordered_is_less) {
    movl(dst, -1);
    jcc(Assembler::parity, L);
    jcc(Assembler::below , L);
    movl(dst, 0);
    jcc(Assembler::equal , L);
    increment(dst);
  } else { // unordered is greater
    movl(dst, 1);
    jcc(Assembler::parity, L);
    jcc(Assembler::above , L);
    movl(dst, 0);
    jcc(Assembler::equal , L);
    decrementl(dst);
  }
  bind(L);
}
void MacroAssembler::cmp8(AddressLiteral src1, int imm) {
  if (reachable(src1)) {
    cmpb(as_Address(src1), imm);
  } else {
    lea(rscratch1, src1);
    cmpb(Address(rscratch1, 0), imm);
  }
}
void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {
#ifdef _LP64
  if (src2.is_lval()) {
    movptr(rscratch1, src2);
    Assembler::cmpq(src1, rscratch1);
  } else if (reachable(src2)) {
    cmpq(src1, as_Address(src2));
  } else {
    lea(rscratch1, src2);
    Assembler::cmpq(src1, Address(rscratch1, 0));
  }
#else
  if (src2.is_lval()) {
    cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
  } else {
    cmpl(src1, as_Address(src2));
  }
#endif // _LP64
}
void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {
  assert(src2.is_lval(), "not a mem-mem compare");
#ifdef _LP64
  movptr(rscratch1, src2);
  Assembler::cmpq(src1, rscratch1);
#else
  cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
#endif // _LP64
}
void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) {
  if (reachable(adr)) {
    if (os::is_MP())
      lock();
    cmpxchgptr(reg, as_Address(adr));
  } else {
    lea(rscratch1, adr);
    if (os::is_MP())
      lock();
    cmpxchgptr(reg, Address(rscratch1, 0));
  }
}
void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
  LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
}
void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) {
  if (reachable(src)) {
    Assembler::comisd(dst, as_Address(src));
  } else {
    lea(rscratch1, src);
    Assembler::comisd(dst, Address(rscratch1, 0));
  }
}
void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) {
  if (reachable(src)) {
    Assembler::comiss(dst, as_Address(src));
  } else {
    lea(rscratch1, src);
    Assembler::comiss(dst, Address(rscratch1, 0));
  }
}
void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
  Condition negated_cond = negate_condition(cond);
  Label L;
  jcc(negated_cond, L);
  pushf(); // Preserve flags
  atomic_incl(counter_addr);
  popf();
  bind(L);
}
int MacroAssembler::corrected_idivl(Register reg) {
  assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
  const int min_int = 0x80000000;
  Label normal_case, special_case;
  cmpl(rax, min_int);
  jcc(Assembler::notEqual, normal_case);
  xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
  cmpl(reg, -1);
  jcc(Assembler::equal, special_case);
  bind(normal_case);
  cdql();
  int idivl_offset = offset();
  idivl(reg);
  bind(special_case);
  return idivl_offset;
}
void MacroAssembler::decrementl(Register reg, int value) {
  if (value == min_jint) {subl(reg, value) ; return; }
  if (value <  0) { incrementl(reg, -value); return; }
  if (value == 0) {                        ; return; }
  if (value == 1 && UseIncDec) { decl(reg) ; return; }
}
void MacroAssembler::decrementl(Address dst, int value) {
  if (value == min_jint) {subl(dst, value) ; return; }
  if (value <  0) { incrementl(dst, -value); return; }
  if (value == 0) {                        ; return; }
  if (value == 1 && UseIncDec) { decl(dst) ; return; }
}
void MacroAssembler::division_with_shift (Register reg, int shift_value) {
  assert (shift_value > 0, "illegal shift value");
  Label _is_positive;
  testl (reg, reg);
  jcc (Assembler::positive, _is_positive);
  int offset = (1 << shift_value) - 1 ;
  if (offset == 1) {
    incrementl(reg);
  } else {
    addl(reg, offset);
  }
  bind (_is_positive);
  sarl(reg, shift_value);
}
void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) {
  if (reachable(src)) {
    Assembler::divsd(dst, as_Address(src));
  } else {
    lea(rscratch1, src);
    Assembler::divsd(dst, Address(rscratch1, 0));
  }
}
void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) {
  if (reachable(src)) {
    Assembler::divss(dst, as_Address(src));
  } else {
    lea(rscratch1, src);
    Assembler::divss(dst, Address(rscratch1, 0));
  }
}
#if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2)
void MacroAssembler::empty_FPU_stack() {
  if (VM_Version::supports_mmx()) {
    emms();
  } else {
    for (int i = 8; i-- > 0; ) ffree(i);
  }
}
#endif // !LP64 || C1 || !C2
void MacroAssembler::eden_allocate(Register obj,
                                   Register var_size_in_bytes,
                                   int con_size_in_bytes,
                                   Register t1,
                                   Label& slow_case) {
  assert(obj == rax, "obj must be in rax, for cmpxchg");
  assert_different_registers(obj, var_size_in_bytes, t1);
  if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {
    jmp(slow_case);
  } else {
    Register end = t1;
    Label retry;
    bind(retry);
    ExternalAddress heap_top((address) Universe::heap()->top_addr());
    movptr(obj, heap_top);
    if (var_size_in_bytes == noreg) {
      lea(end, Address(obj, con_size_in_bytes));
    } else {
      lea(end, Address(obj, var_size_in_bytes, Address::times_1));
    }
    cmpptr(end, obj);
    jcc(Assembler::below, slow_case);
    cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
    jcc(Assembler::above, slow_case);
    locked_cmpxchgptr(end, heap_top);
    jcc(Assembler::notEqual, retry);
  }
}
void MacroAssembler::enter() {
  push(rbp);
  mov(rbp, rsp);
}
void MacroAssembler::fat_nop() {
  if (UseAddressNop) {
    addr_nop_5();
  } else {
    emit_int8(0x26); // es:
    emit_int8(0x2e); // cs:
    emit_int8(0x64); // fs:
    emit_int8(0x65); // gs:
    emit_int8((unsigned char)0x90);
  }
}