ssssssssss54


Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_array, jint targetOffset_i,
                                     jint cache_i, jint md2_i) {
  Node* no_ctrl  = NULL;
  float likely   = PROB_LIKELY(0.9);
  float unlikely = PROB_UNLIKELY(0.9);
  const int nargs = 0; // no arguments to push back for uncommon trap in predicate
  Node* source        = load_String_value(no_ctrl, string_object);
  Node* sourceOffset  = load_String_offset(no_ctrl, string_object);
  Node* sourceCount   = load_String_length(no_ctrl, string_object);
  Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true)));
  jint target_length = target_array->length();
  const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));
  const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);
  if (UseImplicitStableValues) {
    target = cast_array_to_stable(target, target_type);
  }
  IdealKit kit(this, false, true);
#define __ kit.
  Node* zero             = __ ConI(0);
  Node* one              = __ ConI(1);
  Node* cache            = __ ConI(cache_i);
  Node* md2              = __ ConI(md2_i);
  Node* lastChar         = __ ConI(target_array->char_at(target_length - 1));
  Node* targetCount      = __ ConI(target_length);
  Node* targetCountLess1 = __ ConI(target_length - 1);
  Node* targetOffset     = __ ConI(targetOffset_i);
  Node* sourceEnd        = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1);
  IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done();
  Node* outer_loop = __ make_label(2 /* goto */);
  Node* return_    = __ make_label(1);
  __ set(rtn,__ ConI(-1));
  __ loop(this, nargs, i, sourceOffset, BoolTest::lt, sourceEnd); {
       Node* i2  = __ AddI(__ value(i), targetCountLess1);
       Node* src = load_array_element(__ ctrl(), source, i2, TypeAryPtr::CHARS);
       __ if_then(src, BoolTest::eq, lastChar, unlikely); {
         __ loop(this, nargs, j, zero, BoolTest::lt, targetCountLess1); {
              Node* tpj = __ AddI(targetOffset, __ value(j));
              Node* targ = load_array_element(no_ctrl, target, tpj, target_type);
              Node* ipj  = __ AddI(__ value(i), __ value(j));
              Node* src2 = load_array_element(no_ctrl, source, ipj, TypeAryPtr::CHARS);
              __ if_then(targ, BoolTest::ne, src2); {
                __ if_then(__ AndI(cache, __ LShiftI(one, src2)), BoolTest::eq, zero); {
                  __ if_then(md2, BoolTest::lt, __ AddI(__ value(j), one)); {
                    __ increment(i, __ AddI(__ value(j), one));
                    __ goto_(outer_loop);
                  } __ end_if(); __ dead(j);
                }__ end_if(); __ dead(j);
                __ increment(i, md2);
                __ goto_(outer_loop);
              }__ end_if();
              __ increment(j, one);
         }__ end_loop(); __ dead(j);
         __ set(rtn, __ SubI(__ value(i), sourceOffset)); __ dead(i);
         __ goto_(return_);
       }__ end_if();
       __ if_then(__ AndI(cache, __ LShiftI(one, src)), BoolTest::eq, zero, likely); {
         __ increment(i, targetCountLess1);
       }__ end_if();
       __ increment(i, one);
       __ bind(outer_loop);
  }__ end_loop(); __ dead(i);
  __ bind(return_);
  final_sync(kit);
  Node* result = __ value(rtn);
#undef __
  C->set_has_loops(true);
  return result;
}
bool LibraryCallKit::inline_string_indexOf() {
  Node* receiver = argument(0);
  Node* arg      = argument(1);
  Node* result;
  if (Matcher::has_match_rule(Op_StrIndexOf) &&
      UseSSE42Intrinsics) {
    receiver = null_check(receiver);
    arg      = null_check(arg);
    if (stopped()) {
      return true;
    }
    ciInstanceKlass* str_klass = env()->String_klass();
    const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(str_klass);
    RegionNode* result_rgn = new (C) RegionNode(4);
    Node*       result_phi = new (C) PhiNode(result_rgn, TypeInt::INT);
    Node* no_ctrl  = NULL;
    Node* source = load_String_value(no_ctrl, receiver);
    Node* source_offset = load_String_offset(no_ctrl, receiver);
    Node* source_start = array_element_address(source, source_offset, T_CHAR);
    Node* source_cnt  = load_String_length(no_ctrl, receiver);
    Node* substr = load_String_value(no_ctrl, arg);
    Node* substr_offset = load_String_offset(no_ctrl, arg);
    Node* substr_start = array_element_address(substr, substr_offset, T_CHAR);
    Node* substr_cnt  = load_String_length(no_ctrl, arg);
    Node* cmp = _gvn.transform(new(C) CmpINode(substr_cnt, source_cnt));
    Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::gt));
    Node* if_gt = generate_slow_guard(bol, NULL);
    if (if_gt != NULL) {
      result_phi->init_req(2, intcon(-1));
      result_rgn->init_req(2, if_gt);
    }
    if (!stopped()) {
      cmp = _gvn.transform(new(C) CmpINode(substr_cnt, intcon(0)));
      bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::eq));
      Node* if_zero = generate_slow_guard(bol, NULL);
      if (if_zero != NULL) {
        result_phi->init_req(3, intcon(0));
        result_rgn->init_req(3, if_zero);
      }
    }
    if (!stopped()) {
      result = make_string_method_node(Op_StrIndexOf, source_start, source_cnt, substr_start, substr_cnt);
      result_phi->init_req(1, result);
      result_rgn->init_req(1, control());
    }
    set_control(_gvn.transform(result_rgn));
    record_for_igvn(result_rgn);
    result = _gvn.transform(result_phi);
  } else { // Use LibraryCallKit::string_indexOf
    if (!arg->is_Con()) {
     return false;
    }
    const TypeOopPtr* str_type = _gvn.type(arg)->isa_oopptr();
    if (str_type == NULL) {
      return false;
    }
    ciInstanceKlass* klass = env()->String_klass();
    ciObject* str_const = str_type->const_oop();
    if (str_const == NULL || str_const->klass() != klass) {
      return false;
    }
    ciInstance* str = str_const->as_instance();
    assert(str != NULL, "must be instance");
    ciObject* v = str->field_value_by_offset(java_lang_String::value_offset_in_bytes()).as_object();
    ciTypeArray* pat = v->as_type_array(); // pattern (argument) character array
    int o;
    int c;
    if (java_lang_String::has_offset_field()) {
      o = str->field_value_by_offset(java_lang_String::offset_offset_in_bytes()).as_int();
      c = str->field_value_by_offset(java_lang_String::count_offset_in_bytes()).as_int();
    } else {
      o = 0;
      c = pat->length();
    }
    if (o != 0 || c != pat->length()) {
     return false;
    }
    receiver = null_check(receiver, T_OBJECT);
    if (stopped()) {
      return true;
    }
    if (c == 0) {
      set_result(intcon(0));
      return true;
    }
    jchar lastChar = pat->char_at(o + (c - 1));
    int cache = 0;
    int i;
    for (i = 0; i < c - 1; i++) {
      assert(i < pat->length(), "out of range");
      cache |= (1 << (pat->char_at(o + i) & (sizeof(cache) * BitsPerByte - 1)));
    }
    int md2 = c;
    for (i = 0; i < c - 1; i++) {
      assert(i < pat->length(), "out of range");
      if (pat->char_at(o + i) == lastChar) {
        md2 = (c - 1) - i;
      }
    }
    result = string_indexOf(receiver, pat, o, cache, md2);
  }
  set_result(result);
  return true;
}
Node* LibraryCallKit::round_double_node(Node* n) {
  if (Matcher::strict_fp_requires_explicit_rounding && UseSSE <= 1)
    n = _gvn.transform(new (C) RoundDoubleNode(0, n));
  return n;
}
bool LibraryCallKit::inline_math(vmIntrinsics::ID id) {
  Node* arg = round_double_node(argument(0));
  Node* n = NULL;
  switch (id) {
  case vmIntrinsics::_dabs:   n = new (C) AbsDNode(                arg);  break;
  case vmIntrinsics::_dsqrt:  n = new (C) SqrtDNode(C, control(),  arg);  break;
  case vmIntrinsics::_dlog:   n = new (C) LogDNode(C, control(),   arg);  break;
  case vmIntrinsics::_dlog10: n = new (C) Log10DNode(C, control(), arg);  break;
  default:  fatal_unexpected_iid(id);  break;
  }
  set_result(_gvn.transform(n));
  return true;
}
bool LibraryCallKit::inline_trig(vmIntrinsics::ID id) {
  Node* arg = round_double_node(argument(0));
  Node* n = NULL;
  switch (id) {
  case vmIntrinsics::_dsin:  n = new (C) SinDNode(C, control(), arg);  break;
  case vmIntrinsics::_dcos:  n = new (C) CosDNode(C, control(), arg);  break;
  case vmIntrinsics::_dtan:  n = new (C) TanDNode(C, control(), arg);  break;
  default:  fatal_unexpected_iid(id);  break;
  }
  n = _gvn.transform(n);
  if (Matcher::strict_fp_requires_explicit_rounding) {
    static const double     pi_4 =  0.7853981633974483;
    static const double neg_pi_4 = -0.7853981633974483;
    RegionNode* r = new (C) RegionNode(3);
    Node* phi = new (C) PhiNode(r, Type::DOUBLE);
    Node *abs = _gvn.transform(new (C) AbsDNode(arg));
    Node *pi4 = makecon(TypeD::make(pi_4));
    Node *cmp = _gvn.transform(new (C) CmpDNode(pi4,abs));
    Node *bol = _gvn.transform(new (C) BoolNode( cmp, BoolTest::lt ));
    IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
    set_control(opt_iff(r,iff));
    phi->init_req(2, n);
    Node* call = NULL;
    switch (id) {
    case vmIntrinsics::_dsin:
      call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
                               CAST_FROM_FN_PTR(address, SharedRuntime::dsin),
                               "Sin", NULL, arg, top());
      break;
    case vmIntrinsics::_dcos:
      call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
                               CAST_FROM_FN_PTR(address, SharedRuntime::dcos),
                               "Cos", NULL, arg, top());
      break;
    case vmIntrinsics::_dtan:
      call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
                               CAST_FROM_FN_PTR(address, SharedRuntime::dtan),
                               "Tan", NULL, arg, top());
      break;
    }
    assert(control()->in(0) == call, "");
    Node* slow_result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
    r->init_req(1, control());
    phi->init_req(1, slow_result);
    set_control(_gvn.transform(r));
    record_for_igvn(r);
    n = _gvn.transform(phi);
    C->set_has_split_ifs(true); // Has chance for split-if optimization
  }
  set_result(n);
  return true;
}
Node* LibraryCallKit::finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName) {
  Node* cmpisnan = _gvn.transform(new (C) CmpDNode(result, result));
  Node* bolisnum = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::eq));
  if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
    { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
      uncommon_trap(Deoptimization::Reason_intrinsic,
                    Deoptimization::Action_make_not_entrant);
    }
    return result;
  } else {
    IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
    Node* if_slow = _gvn.transform(new (C) IfFalseNode(iff));
    Node* if_fast = _gvn.transform(new (C) IfTrueNode(iff));
    if (!if_slow->is_top()) {
      RegionNode* result_region = new (C) RegionNode(3);
      PhiNode*    result_val = new (C) PhiNode(result_region, Type::DOUBLE);
      result_region->init_req(1, if_fast);
      result_val->init_req(1, result);
      set_control(if_slow);
      const TypePtr* no_memory_effects = NULL;
      Node* rt = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
                                   no_memory_effects,
                                   x, top(), y, y ? top() : NULL);
      Node* value = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+0));
#ifdef ASSERT
      Node* value_top = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+1));
      assert(value_top == top(), "second value must be top");
#endif
      result_region->init_req(2, control());
      result_val->init_req(2, value);
      set_control(_gvn.transform(result_region));
      return _gvn.transform(result_val);
    } else {
      return result;
    }
  }
}
bool LibraryCallKit::inline_exp() {
  Node* arg = round_double_node(argument(0));
  Node* n   = _gvn.transform(new (C) ExpDNode(C, control(), arg));
  n = finish_pow_exp(n, arg, NULL, OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dexp), "EXP");
  set_result(n);
  C->set_has_split_ifs(true); // Has chance for split-if optimization
  return true;
}
bool LibraryCallKit::inline_pow() {
  Node* x = round_double_node(argument(0));
  Node* y = round_double_node(argument(2));
  Node* result = NULL;
  Node*   const_two_node = makecon(TypeD::make(2.0));
  Node*   cmp_node       = _gvn.transform(new (C) CmpDNode(y, const_two_node));
  Node*   bool_node      = _gvn.transform(new (C) BoolNode(cmp_node, BoolTest::eq));
  IfNode* if_node        = create_and_xform_if(control(), bool_node, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
  Node*   if_true        = _gvn.transform(new (C) IfTrueNode(if_node));
  Node*   if_false       = _gvn.transform(new (C) IfFalseNode(if_node));
  RegionNode* region_node = new (C) RegionNode(3);
  region_node->init_req(1, if_true);
  Node* phi_node = new (C) PhiNode(region_node, Type::DOUBLE);
  phi_node->init_req(1, _gvn.transform(new (C) MulDNode(x, x)));
  set_control(if_false);
  if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
    result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
  } else {
    RegionNode *r = new (C) RegionNode(4);
    Node *phi = new (C) PhiNode(r, Type::DOUBLE);
    Node *zeronode = makecon(TypeD::ZERO);
    Node *cmp = _gvn.transform(new (C) CmpDNode(x, zeronode));
    Node *bol1 = _gvn.transform(new (C) BoolNode( cmp, BoolTest::le ));
    IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
    Node *fast_taken = _gvn.transform(new (C) IfFalseNode(if1));
    r->init_req(3,fast_taken); // Capture fast-control
    Node *complex_path = _gvn.transform(new (C) IfTrueNode(if1));
    Node *fast_result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
    phi->init_req(3, fast_result);
    Node *longy = _gvn.transform(new (C) ConvD2LNode(y));
    Node *doublelongy= _gvn.transform(new (C) ConvL2DNode(longy));
    Node *cmplongy= _gvn.transform(new (C) CmpDNode(doublelongy, y));
    Node *bol2 = _gvn.transform(new (C) BoolNode( cmplongy, BoolTest::ne ));
    IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
    Node* ylong_path = _gvn.transform(new (C) IfFalseNode(if2));
    Node *slow_path = _gvn.transform(new (C) IfTrueNode(if2));
    Node *conone = longcon(1);
    Node *signnode= _gvn.transform(new (C) AndLNode(conone, longy));
    Node* yplus1 = _gvn.transform(new (C) AddDNode(y, makecon(TypeD::make(1))));
    Node *cmpyplus1= _gvn.transform(new (C) CmpDNode(yplus1, y));
    Node *bolyplus1 = _gvn.transform(new (C) BoolNode( cmpyplus1, BoolTest::eq ));
    Node* correctedsign = NULL;
    if (ConditionalMoveLimit != 0) {
      correctedsign = _gvn.transform( CMoveNode::make(C, NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG));
    } else {
      IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN);
      RegionNode *r = new (C) RegionNode(3);
      Node *phi = new (C) PhiNode(r, TypeLong::LONG);
      r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyplus1)));
      r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyplus1)));
      phi->init_req(1, signnode);
      phi->init_req(2, longcon(0));
      correctedsign = _gvn.transform(phi);
      ylong_path = _gvn.transform(r);
      record_for_igvn(r);
    }
    Node *conzero = longcon(0);
    Node *cmpeq1 = _gvn.transform(new (C) CmpLNode(correctedsign, conzero));
    Node *bol3 = _gvn.transform(new (C) BoolNode( cmpeq1, BoolTest::ne ));
    Node *absx=_gvn.transform(new (C) AbsDNode(x));
    Node *absxpowy = _gvn.transform(new (C) PowDNode(C, control(), absx, y));
    Node *negabsxpowy = _gvn.transform(new (C) NegDNode (absxpowy));
    Node *signresult = NULL;
    if (ConditionalMoveLimit != 0) {
      signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));
    } else {
      IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN);
      RegionNode *r = new (C) RegionNode(3);
      Node *phi = new (C) PhiNode(r, Type::DOUBLE);
      r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyeven)));
      r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyeven)));
      phi->init_req(1, absxpowy);
      phi->init_req(2, negabsxpowy);
      signresult = _gvn.transform(phi);
      ylong_path = _gvn.transform(r);
      record_for_igvn(r);
    }
    r->init_req(2, ylong_path);
    phi->init_req(2, signresult);
    static const jlong nan_bits = CONST64(0x7ff8000000000000);
    Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN
    r->init_req(1,slow_path);
    phi->init_req(1,slow_result);
    set_control(_gvn.transform(r));
    record_for_igvn(r);
    result = _gvn.transform(phi);
  }
  result = finish_pow_exp(result, x, y, OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW");
  region_node->set_req(2, control());
  phi_node->init_req(2, result);
  set_control(_gvn.transform(region_node));
  record_for_igvn(region_node);
  set_result(_gvn.transform(phi_node));
  C->set_has_split_ifs(true); // Has chance for split-if optimization
  return true;
}
bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName) {
  assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_D_Type(),
         "must be (DD)D or (D)D type");
  Node* a = round_double_node(argument(0));
  Node* b = (call_type == OptoRuntime::Math_DD_D_Type()) ? round_double_node(argument(2)) : NULL;
  const TypePtr* no_memory_effects = NULL;
  Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
                                 no_memory_effects,
                                 a, top(), b, b ? top() : NULL);
  Node* value = _gvn.transform(new (C) ProjNode(trig, TypeFunc::Parms+0));
#ifdef ASSERT
  Node* value_top = _gvn.transform(new (C) ProjNode(trig, TypeFunc::Parms+1));
  assert(value_top == top(), "second value must be top");
#endif
  set_result(value);
  return true;
}
bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) {
#define FN_PTR(f) CAST_FROM_FN_PTR(address, f)
  switch (id) {
  case vmIntrinsics::_dcos:   return Matcher::has_match_rule(Op_CosD)   ? inline_trig(id) :
    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dcos),   "COS");
  case vmIntrinsics::_dsin:   return Matcher::has_match_rule(Op_SinD)   ? inline_trig(id) :
    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dsin),   "SIN");
  case vmIntrinsics::_dtan:   return Matcher::has_match_rule(Op_TanD)   ? inline_trig(id) :
    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dtan),   "TAN");
  case vmIntrinsics::_dlog:   return Matcher::has_match_rule(Op_LogD)   ? inline_math(id) :
    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog),   "LOG");
  case vmIntrinsics::_dlog10: return Matcher::has_match_rule(Op_Log10D) ? inline_math(id) :
    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog10), "LOG10");
  case vmIntrinsics::_dsqrt:  return Matcher::match_rule_supported(Op_SqrtD) ? inline_math(id) : false;
  case vmIntrinsics::_dabs:   return Matcher::has_match_rule(Op_AbsD)   ? inline_math(id) : false;
  case vmIntrinsics::_dexp:   return Matcher::has_match_rule(Op_ExpD)   ? inline_exp()    :
    runtime_math(OptoRuntime::Math_D_D_Type(),  FN_PTR(SharedRuntime::dexp),  "EXP");
  case vmIntrinsics::_dpow:   return Matcher::has_match_rule(Op_PowD)   ? inline_pow()    :
    runtime_math(OptoRuntime::Math_DD_D_Type(), FN_PTR(SharedRuntime::dpow),  "POW");
#undef FN_PTR
  case vmIntrinsics::_datan2:
    return false;
  default:
    fatal_unexpected_iid(id);
    return false;
  }
}
static bool is_simple_name(Node* n) {
  return (n->req() == 1         // constant
          || (n->is_Type() && n->as_Type()->type()->singleton())
          || n->is_Proj()       // parameter or return value
          || n->is_Phi()        // local of some sort
          );
}
bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) {
  set_result(generate_min_max(id, argument(0), argument(1)));
  return true;
}
void LibraryCallKit::inline_math_mathExact(Node* math, Node *test) {
  Node* bol = _gvn.transform( new (C) BoolNode(test, BoolTest::overflow) );
  IfNode* check = create_and_map_if(control(), bol, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
  Node* fast_path = _gvn.transform( new (C) IfFalseNode(check));
  Node* slow_path = _gvn.transform( new (C) IfTrueNode(check) );
  {
    PreserveJVMState pjvms(this);
    PreserveReexecuteState preexecs(this);
    jvms()->set_should_reexecute(true);
    set_control(slow_path);
    set_i_o(i_o());
    uncommon_trap(Deoptimization::Reason_intrinsic,
                  Deoptimization::Action_none);
  }
  set_control(fast_path);
  set_result(math);
}
template <typename OverflowOp>
bool LibraryCallKit::inline_math_overflow(Node* arg1, Node* arg2) {
  typedef typename OverflowOp::MathOp MathOp;
  MathOp* mathOp = new(C) MathOp(arg1, arg2);
  Node* operation = _gvn.transform( mathOp );
  Node* ofcheck = _gvn.transform( new(C) OverflowOp(arg1, arg2) );
  inline_math_mathExact(operation, ofcheck);
  return true;
}
bool LibraryCallKit::inline_math_addExactI(bool is_increment) {
  return inline_math_overflow<OverflowAddINode>(argument(0), is_increment ? intcon(1) : argument(1));
}
bool LibraryCallKit::inline_math_addExactL(bool is_increment) {
  return inline_math_overflow<OverflowAddLNode>(argument(0), is_increment ? longcon(1) : argument(2));
}
bool LibraryCallKit::inline_math_subtractExactI(bool is_decrement) {
  return inline_math_overflow<OverflowSubINode>(argument(0), is_decrement ? intcon(1) : argument(1));
}
bool LibraryCallKit::inline_math_subtractExactL(bool is_decrement) {
  return inline_math_overflow<OverflowSubLNode>(argument(0), is_decrement ? longcon(1) : argument(2));
}
bool LibraryCallKit::inline_math_negateExactI() {
  return inline_math_overflow<OverflowSubINode>(intcon(0), argument(0));
}
bool LibraryCallKit::inline_math_negateExactL() {
  return inline_math_overflow<OverflowSubLNode>(longcon(0), argument(0));
}
bool LibraryCallKit::inline_math_multiplyExactI() {
  return inline_math_overflow<OverflowMulINode>(argument(0), argument(1));
}
bool LibraryCallKit::inline_math_multiplyExactL() {
  return inline_math_overflow<OverflowMulLNode>(argument(0), argument(2));
}
Node*
LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) {
  Node* xvalue = x0;
  Node* yvalue = y0;
  if (xvalue == yvalue) {
    return xvalue;
  }
  bool want_max = (id == vmIntrinsics::_max);
  const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
  const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
  if (txvalue == NULL || tyvalue == NULL)  return top();
  int widen = MAX2(txvalue->_widen, tyvalue->_widen);
  int   cmp_op = Op_CmpI;
  Node* xkey = xvalue;
  Node* ykey = yvalue;
  Node* ideal_cmpxy = _gvn.transform(new(C) CmpINode(xkey, ykey));
  if (ideal_cmpxy->is_Cmp()) {
    cmp_op = ideal_cmpxy->Opcode();
    xkey = ideal_cmpxy->in(1);
    ykey = ideal_cmpxy->in(2);
  }
  Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
  Node* cmpxy = NULL;
  Node* cmpyx = NULL;
  for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
    Node* cmp = start_from->fast_out(k);
    if (cmp->outcnt() > 0 &&            // must have prior uses
        cmp->in(0) == NULL &&           // must be context-independent
        cmp->Opcode() == cmp_op) {      // right kind of compare
      if (cmp->in(1) == xkey && cmp->in(2) == ykey)  cmpxy = cmp;
      if (cmp->in(1) == ykey && cmp->in(2) == xkey)  cmpyx = cmp;
    }
  }
  const int NCMPS = 2;
  Node* cmps[NCMPS] = { cmpxy, cmpyx };
  int cmpn;
  for (cmpn = 0; cmpn < NCMPS; cmpn++) {
    if (cmps[cmpn] != NULL)  break;     // find a result
  }
  if (cmpn < NCMPS) {
    int depth = 0;                // Limit search depth for speed
    Node* dom = control();
    for (; dom != NULL; dom = IfNode::up_one_dom(dom, true)) {
      if (++depth >= 100)  break;
      Node* ifproj = dom;
      if (!ifproj->is_Proj())  continue;
      Node* iff = ifproj->in(0);
      if (!iff->is_If())  continue;
      Node* bol = iff->in(1);
      if (!bol->is_Bool())  continue;
      Node* cmp = bol->in(1);
      if (cmp == NULL)  continue;
      for (cmpn = 0; cmpn < NCMPS; cmpn++)
        if (cmps[cmpn] == cmp)  break;
      if (cmpn == NCMPS)  continue;
      BoolTest::mask btest = bol->as_Bool()->_test._test;
      if (ifproj->is_IfFalse())  btest = BoolTest(btest).negate();
      if (cmp->in(1) == ykey)    btest = BoolTest(btest).commute();
      switch (btest) {
      case BoolTest::eq:
        if (is_simple_name(yvalue) && !is_simple_name(xvalue))
          return yvalue;
        return xvalue;
      case BoolTest::lt:          // x < y
      case BoolTest::le:          // x <= y
        return (want_max ? yvalue : xvalue);
      case BoolTest::gt:          // x > y
      case BoolTest::ge:          // x >= y
        return (want_max ? xvalue : yvalue);
      }
    }
  }
  Node*          best_bol   = NULL;
  BoolTest::mask best_btest = BoolTest::illegal;
  for (cmpn = 0; cmpn < NCMPS; cmpn++) {
    Node* cmp = cmps[cmpn];
    if (cmp == NULL)  continue;
    for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) {
      Node* bol = cmp->fast_out(j);
      if (!bol->is_Bool())  continue;
      BoolTest::mask btest = bol->as_Bool()->_test._test;
      if (btest == BoolTest::eq || btest == BoolTest::ne)  continue;
      if (cmp->in(1) == ykey)   btest = BoolTest(btest).commute();
      if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) {
        best_bol   = bol->as_Bool();
        best_btest = btest;
      }
    }
  }
  Node* answer_if_true  = NULL;
  Node* answer_if_false = NULL;
  switch (best_btest) {
  default:
    if (cmpxy == NULL)
      cmpxy = ideal_cmpxy;
    best_bol = _gvn.transform(new(C) BoolNode(cmpxy, BoolTest::lt));
  case BoolTest::lt:          // x < y
  case BoolTest::le:          // x <= y
    answer_if_true  = (want_max ? yvalue : xvalue);
    answer_if_false = (want_max ? xvalue : yvalue);
    break;
  case BoolTest::gt:          // x > y
  case BoolTest::ge:          // x >= y
    answer_if_true  = (want_max ? xvalue : yvalue);
    answer_if_false = (want_max ? yvalue : xvalue);
    break;
  }
  jint hi, lo;
  if (want_max) {
    hi = MAX2(txvalue->_hi, tyvalue->_hi);
    lo = MAX2(txvalue->_lo, tyvalue->_lo);
  } else {
    hi = MIN2(txvalue->_hi, tyvalue->_hi);
    lo = MIN2(txvalue->_lo, tyvalue->_lo);
  }
  Node* cmov = CMoveNode::make(C, NULL, best_bol,
                               answer_if_false, answer_if_true,
                               TypeInt::make(lo, hi, widen));
  return _gvn.transform(cmov);
  switch (id) {
  case vmIntrinsics::_min:
    result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break;
  case vmIntrinsics::_max:
    result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break;
  default:
    ShouldNotReachHere();
  }
}
inline int
LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) {
  const TypePtr* base_type = TypePtr::NULL_PTR;
  if (base != NULL)  base_type = _gvn.type(base)->isa_ptr();
  if (base_type == NULL) {
    return Type::AnyPtr;
  } else if (base_type == TypePtr::NULL_PTR) {
    base   = _gvn.transform(new (C) CastX2PNode(offset));
    offset = MakeConX(0);
    return Type::RawPtr;
  } else if (base_type->base() == Type::RawPtr) {
    return Type::RawPtr;
  } else if (base_type->isa_oopptr()) {
    if (base_type->ptr() == TypePtr::NotNull) {
      return Type::OopPtr;
    }
    const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t();
    if (offset_type != NULL &&
        base_type->offset() == 0 &&     // (should always be?)
        offset_type->_lo >= 0 &&
        !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) {
      return Type::OopPtr;
    }
    return Type::AnyPtr;
  } else {
    return Type::AnyPtr;
  }
}
inline Node* LibraryCallKit::make_unsafe_address(Node* base, Node* offset) {
  int kind = classify_unsafe_addr(base, offset);
  if (kind == Type::RawPtr) {
    return basic_plus_adr(top(), base, offset);
  } else {
    return basic_plus_adr(base, offset);
  }
}
bool LibraryCallKit::inline_number_methods(vmIntrinsics::ID id) {
  Node* arg = argument(0);
  Node* n = NULL;
  switch (id) {
  case vmIntrinsics::_numberOfLeadingZeros_i:   n = new (C) CountLeadingZerosINode( arg);  break;
  case vmIntrinsics::_numberOfLeadingZeros_l:   n = new (C) CountLeadingZerosLNode( arg);  break;
  case vmIntrinsics::_numberOfTrailingZeros_i:  n = new (C) CountTrailingZerosINode(arg);  break;
  case vmIntrinsics::_numberOfTrailingZeros_l:  n = new (C) CountTrailingZerosLNode(arg);  break;
  case vmIntrinsics::_bitCount_i:               n = new (C) PopCountINode(          arg);  break;
  case vmIntrinsics::_bitCount_l:               n = new (C) PopCountLNode(          arg);  break;
  case vmIntrinsics::_reverseBytes_c:           n = new (C) ReverseBytesUSNode(0,   arg);  break;
  case vmIntrinsics::_reverseBytes_s:           n = new (C) ReverseBytesSNode( 0,   arg);  break;
  case vmIntrinsics::_reverseBytes_i:           n = new (C) ReverseBytesINode( 0,   arg);  break;
  case vmIntrinsics::_reverseBytes_l:           n = new (C) ReverseBytesLNode( 0,   arg);  break;
  default:  fatal_unexpected_iid(id);  break;
  }
  set_result(_gvn.transform(n));
  return true;
}
const static BasicType T_ADDRESS_HOLDER = T_LONG;
void LibraryCallKit::insert_pre_barrier(Node* base_oop, Node* offset,
                                        Node* pre_val, bool need_mem_bar) {
  if (!UseG1GC && !need_mem_bar)
    return;
  const TypeX* otype = offset->find_intptr_t_type();
  if (otype != NULL && otype->is_con() &&
      otype->get_con() != java_lang_ref_Reference::referent_offset) {
    return;
  }
  const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
  if (btype != NULL) {
    if (btype->isa_aryptr()) {
      return;
    }
    const TypeInstPtr* itype = btype->isa_instptr();
    if (itype != NULL) {
      ciKlass* klass = itype->klass();
      if ( klass->is_loaded() &&
          !klass->is_subtype_of(env()->Reference_klass()) &&
          !env()->Object_klass()->is_subtype_of(klass)) {
        return;
      }
    }
  }
  float likely   = PROB_LIKELY(  0.999);
  float unlikely = PROB_UNLIKELY(0.999);
  IdealKit ideal(this);
#define __ ideal.
  Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);
  __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
      sync_kit(ideal);
      Node* ref_klass_con = makecon(TypeKlassPtr::make(env()->Reference_klass()));
      Node* is_instof = gen_instanceof(base_oop, ref_klass_con);
      __ sync_kit(this);
      Node* one = __ ConI(1);
      __ if_then(is_instof, BoolTest::eq, one, unlikely); {
        sync_kit(ideal);
        pre_barrier(false /* do_load */,
                    __ ctrl(),
                    NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
                    pre_val /* pre_val */,
                    T_OBJECT);
        if (need_mem_bar) {
          insert_mem_bar(Op_MemBarCPUOrder);
        }
        __ sync_kit(this);
      } __ end_if(); // _ref_type != ref_none
  } __ end_if(); // offset == referent_offset
  final_sync(ideal);
#undef __
}
extern jlong Unsafe_field_offset_to_byte_offset(jlong field_offset);
const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type, bool is_native_ptr) {
  ciKlass* sharpened_klass = NULL;
  if (alias_type->field() != NULL) {
    assert(!is_native_ptr, "native pointer op cannot use a java address");
    if (alias_type->field()->type()->is_klass()) {
      sharpened_klass = alias_type->field()->type()->as_klass();
    }
  }
  if (adr_type->isa_aryptr()) {
    if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) {
      const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr();
      if (elem_type != NULL) {
        sharpened_klass = elem_type->klass();
      }
    }
  }
  if (sharpened_klass != NULL && sharpened_klass->is_loaded()) {
    const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass);
#ifndef PRODUCT
    if (C->print_intrinsics() || C->print_inlining()) {
      tty->print("  from base type:  ");  adr_type->dump(); tty->cr();
      tty->print("  sharpened value: ");  tjp->dump();      tty->cr();
    }
#endif
    return tjp;
  }
  return NULL;
}
bool LibraryCallKit::inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile, bool unaligned) {
  if (callee()->is_static())  return false;  // caller must have the capability!
  assert(type != T_OBJECT || !unaligned, "unaligned access not supported with object type");
#ifndef PRODUCT
  {
    ResourceMark rm;
    ciSignature* sig = callee()->signature();
#ifdef ASSERT
    if (!is_store) {
      BasicType rtype = sig->return_type()->basic_type();
      if (rtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::getAddress_name())
          rtype = T_ADDRESS;  // it is really a C void*
      assert(rtype == type, "getter must return the expected value");
      if (!is_native_ptr) {
        assert(sig->count() == 2, "oop getter has 2 arguments");
        assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object");
        assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct");
      } else {
        assert(sig->count() == 1, "native getter has 1 argument");
        assert(sig->type_at(0)->basic_type() == T_LONG, "getter base is long");
      }
    } else {
      assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value");
      if (!is_native_ptr) {
        assert(sig->count() == 3, "oop putter has 3 arguments");
        assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object");
        assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct");
      } else {
        assert(sig->count() == 2, "native putter has 2 arguments");
        assert(sig->type_at(0)->basic_type() == T_LONG, "putter base is long");
      }
      BasicType vtype = sig->type_at(sig->count()-1)->basic_type();
      if (vtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::putAddress_name())
        vtype = T_ADDRESS;  // it is really a C void*
      assert(vtype == type, "putter must accept the expected value");
    }
#endif // ASSERT
 }
#endif //PRODUCT
  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
  Node* receiver = argument(0);  // type: oop
  Node* adr;
  Node* heap_base_oop = top();
  Node* offset = top();
  Node* val;
  Node* base = argument(1);  // type: oop
  if (!is_native_ptr) {
    offset = argument(2);  // type: long
    assert(Unsafe_field_offset_to_byte_offset(11) == 11,
           "fieldOffset must be byte-scaled");
    offset = ConvL2X(offset);
    adr = make_unsafe_address(base, offset);
    heap_base_oop = base;
    val = is_store ? argument(4) : NULL;
  } else {
    Node* ptr = argument(1);  // type: long
    ptr = ConvL2X(ptr);  // adjust Java long to machine word
    adr = make_unsafe_address(NULL, ptr);
    val = is_store ? argument(3) : NULL;
  }
  if ((_gvn.type(base)->isa_ptr() == TypePtr::NULL_PTR) && type == T_OBJECT) {
    return false; // off-heap oop accesses are not supported
  }
  const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
  Compile::AliasType* alias_type = C->alias_type(adr_type);
  assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
  if (alias_type->adr_type() == TypeInstPtr::KLASS ||
      alias_type->adr_type() == TypeAryPtr::RANGE) {
    return false; // not supported
  }
  bool mismatched = false;
  BasicType bt = alias_type->basic_type();
  if (bt != T_ILLEGAL) {
    assert(alias_type->adr_type()->is_oopptr(), "should be on-heap access");
    if (bt == T_BYTE && adr_type->isa_aryptr()) {
      bt = adr_type->is_aryptr()->elem()->array_element_basic_type();
    }
    if (bt == T_ARRAY || bt == T_NARROWOOP) {
      bt = T_OBJECT;
    }
    if ((bt == T_OBJECT) != (type == T_OBJECT)) {
      return false;
    }
    mismatched = (bt != type);
  }
  assert(!mismatched || alias_type->adr_type()->is_oopptr(), "off-heap access can't be mismatched");
  const Type *value_type = Type::get_const_basic_type(type);
  bool need_mem_bar = (alias_type->adr_type() == TypeOopPtr::BOTTOM);
  bool need_read_barrier = !is_native_ptr && !is_store &&
                           offset != top() && heap_base_oop != top();
  if (!is_store && type == T_OBJECT) {
    const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type, is_native_ptr);
    if (tjp != NULL) {
      value_type = tjp;
    }
  }
  receiver = null_check(receiver);
  if (stopped()) {
    return true;
  }
  Node* load = NULL;
  Node* store = NULL;
  Node* leading_membar = NULL;
  if (is_volatile) {
    need_mem_bar = true;
    if (is_store) {
      leading_membar = insert_mem_bar(Op_MemBarRelease);
    } else {
      if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
        leading_membar = insert_mem_bar(Op_MemBarVolatile);
      }
    }
  }
  if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
  if (!is_store) {
    MemNode::MemOrd mo = is_volatile ? MemNode::acquire : MemNode::unordered;
    load = make_load(control(), adr, value_type, type, adr_type, mo, LoadNode::Pinned, is_volatile, unaligned, mismatched);
    switch (type) {
    case T_BOOLEAN:
    case T_CHAR:
    case T_BYTE:
    case T_SHORT:
    case T_INT:
    case T_LONG:
    case T_FLOAT:
    case T_DOUBLE:
      break;
    case T_OBJECT:
      if (need_read_barrier) {
        insert_pre_barrier(heap_base_oop, offset, load, !(is_volatile || need_mem_bar));
      }
      break;
    case T_ADDRESS:
      load = _gvn.transform(new (C) CastP2XNode(NULL, load));
      load = ConvX2UL(load);
      break;
    default:
      fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
      break;
    }
    set_result(load);
  } else {
    switch (type) {
    case T_DOUBLE:
      val = dstore_rounding(val);
      break;
    case T_ADDRESS:
      val = ConvL2X(val);
      val = _gvn.transform(new (C) CastX2PNode(val));
      break;
    }
    MemNode::MemOrd mo = is_volatile ? MemNode::release : MemNode::unordered;
    if (type == T_OBJECT ) {
      store = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type, mo, mismatched);
    } else {
      store = store_to_memory(control(), adr, val, type, adr_type, mo, is_volatile, unaligned, mismatched);
    }
  }
  if (is_volatile) {
    if (!is_store) {
      Node* mb = insert_mem_bar(Op_MemBarAcquire, load);
      mb->as_MemBar()->set_trailing_load();
    } else {
      if (!support_IRIW_for_not_multiple_copy_atomic_cpu) {
        Node* mb = insert_mem_bar(Op_MemBarVolatile, store);
        MemBarNode::set_store_pair(leading_membar->as_MemBar(), mb->as_MemBar());
      }
    }
  }
  if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
  return true;
}
bool LibraryCallKit::inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static) {
#ifndef PRODUCT
  {
    ResourceMark rm;
    ciSignature* sig = callee()->signature();
#ifdef ASSERT
    BasicType rtype = sig->return_type()->basic_type();
    if (!is_native_ptr) {
      assert(sig->count() == 2, "oop prefetch has 2 arguments");
      assert(sig->type_at(0)->basic_type() == T_OBJECT, "prefetch base is object");
      assert(sig->type_at(1)->basic_type() == T_LONG, "prefetcha offset is correct");
    } else {
      assert(sig->count() == 1, "native prefetch has 1 argument");
      assert(sig->type_at(0)->basic_type() == T_LONG, "prefetch base is long");
    }
#endif // ASSERT
  }
#endif // !PRODUCT
  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
  const int idx = is_static ? 0 : 1;
  if (!is_static) {
    null_check_receiver();
    if (stopped()) {
      return true;
    }
  }
  Node *adr;
  if (!is_native_ptr) {
    Node* base   = argument(idx + 0);  // type: oop
    Node* offset = argument(idx + 1);  // type: long
    assert(Unsafe_field_offset_to_byte_offset(11) == 11,
           "fieldOffset must be byte-scaled");
    offset = ConvL2X(offset);
    adr = make_unsafe_address(base, offset);
  } else {
    Node* ptr = argument(idx + 0);  // type: long
    ptr = ConvL2X(ptr);  // adjust Java long to machine word
    adr = make_unsafe_address(NULL, ptr);
  }
  Node *prefetch;
  if (is_store) {
    prefetch = new (C) PrefetchWriteNode(i_o(), adr);
  } else {
    prefetch = new (C) PrefetchReadNode(i_o(), adr);
  }
  prefetch->init_req(0, control());
  set_i_o(_gvn.transform(prefetch));
  return true;
}
bool LibraryCallKit::inline_unsafe_load_store(BasicType type, LoadStoreKind kind) {
  if (callee()->is_static())  return false;  // caller must have the capability!
#ifndef PRODUCT
  BasicType rtype;
  {
    ResourceMark rm;
    ciSignature* sig = callee()->signature();
    rtype = sig->return_type()->basic_type();
    if (kind == LS_xadd || kind == LS_xchg) {
#ifdef ASSERT
      assert(rtype == type, "get and set must return the expected type");
      assert(sig->count() == 3, "get and set has 3 arguments");
      assert(sig->type_at(0)->basic_type() == T_OBJECT, "get and set base is object");
      assert(sig->type_at(1)->basic_type() == T_LONG, "get and set offset is long");
      assert(sig->type_at(2)->basic_type() == type, "get and set must take expected type as new value/delta");
#endif // ASSERT
    } else if (kind == LS_cmpxchg) {
#ifdef ASSERT
      assert(rtype == T_BOOLEAN, "CAS must return boolean");
      assert(sig->count() == 4, "CAS has 4 arguments");
      assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object");
      assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long");
#endif // ASSERT
    } else {
      ShouldNotReachHere();
    }
  }
#endif //PRODUCT
  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
  Node* receiver = NULL;
  Node* base     = NULL;
  Node* offset   = NULL;
  Node* oldval   = NULL;
  Node* newval   = NULL;
  if (kind == LS_cmpxchg) {
    const bool two_slot_type = type2size[type] == 2;
    receiver = argument(0);  // type: oop
    base     = argument(1);  // type: oop
    offset   = argument(2);  // type: long
    oldval   = argument(4);  // type: oop, int, or long
    newval   = argument(two_slot_type ? 6 : 5);  // type: oop, int, or long
  } else if (kind == LS_xadd || kind == LS_xchg){
    receiver = argument(0);  // type: oop
    base     = argument(1);  // type: oop
    offset   = argument(2);  // type: long
    oldval   = NULL;
    newval   = argument(4);  // type: oop, int, or long
  }
  assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
  offset = ConvL2X(offset);
  Node* adr = make_unsafe_address(base, offset);
  const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
  Compile::AliasType* alias_type = C->alias_type(adr_type);
  BasicType bt = alias_type->basic_type();
  if (bt != T_ILLEGAL &&
      ((bt == T_OBJECT || bt == T_ARRAY) != (type == T_OBJECT))) {
    return false;
  }
  assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
  const Type *value_type = Type::get_const_basic_type(type);
  if (kind == LS_xchg && type == T_OBJECT) {
    const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type);
    if (tjp != NULL) {
      value_type = tjp;
    }
  }
  receiver = null_check(receiver);
  if (stopped()) {
    return true;
  }
  int alias_idx = C->get_alias_index(adr_type);
  Node* leading_membar = insert_mem_bar(Op_MemBarRelease);
  insert_mem_bar(Op_MemBarCPUOrder);
  Node *mem = memory(alias_idx);
  Node* load_store = NULL;
  switch(type) {
  case T_INT:
    if (kind == LS_xadd) {
      load_store = _gvn.transform(new (C) GetAndAddINode(control(), mem, adr, newval, adr_type));
    } else if (kind == LS_xchg) {
      load_store = _gvn.transform(new (C) GetAndSetINode(control(), mem, adr, newval, adr_type));
    } else if (kind == LS_cmpxchg) {
      load_store = _gvn.transform(new (C) CompareAndSwapINode(control(), mem, adr, newval, oldval));
    } else {
      ShouldNotReachHere();
    }
    break;
  case T_LONG:
    if (kind == LS_xadd) {
      load_store = _gvn.transform(new (C) GetAndAddLNode(control(), mem, adr, newval, adr_type));
    } else if (kind == LS_xchg) {
      load_store = _gvn.transform(new (C) GetAndSetLNode(control(), mem, adr, newval, adr_type));
    } else if (kind == LS_cmpxchg) {
      load_store = _gvn.transform(new (C) CompareAndSwapLNode(control(), mem, adr, newval, oldval));
    } else {
      ShouldNotReachHere();
    }
    break;
  case T_OBJECT:
    if (_gvn.type(newval) == TypePtr::NULL_PTR)
      newval = _gvn.makecon(TypePtr::NULL_PTR);
    if (kind == LS_xchg) {
      if (!can_move_pre_barrier()) {
        pre_barrier(true /* do_load*/,
                    control(), base, adr, alias_idx, newval, value_type->make_oopptr(),
                    NULL /* pre_val*/,
                    T_OBJECT);
      } // Else move pre_barrier to use load_store value, see below.
    } else if (kind == LS_cmpxchg) {
      if (_gvn.type(oldval) == TypePtr::NULL_PTR) {
        oldval = _gvn.makecon(TypePtr::NULL_PTR);
      }
      pre_barrier(false /* do_load */,
                  control(), NULL, NULL, max_juint, NULL, NULL,
                  oldval /* pre_val */,
                  T_OBJECT);
    } else {
      ShouldNotReachHere();
    }
#ifdef _LP64
    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
      Node *newval_enc = _gvn.transform(new (C) EncodePNode(newval, newval->bottom_type()->make_narrowoop()));
      if (kind == LS_xchg) {
        load_store = _gvn.transform(new (C) GetAndSetNNode(control(), mem, adr,
                                                           newval_enc, adr_type, value_type->make_narrowoop()));
      } else {
        assert(kind == LS_cmpxchg, "wrong LoadStore operation");
        Node *oldval_enc = _gvn.transform(new (C) EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));
        load_store = _gvn.transform(new (C) CompareAndSwapNNode(control(), mem, adr,
                                                                newval_enc, oldval_enc));
      }
    } else
#endif
    {
      if (kind == LS_xchg) {
        load_store = _gvn.transform(new (C) GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr()));
      } else {
        assert(kind == LS_cmpxchg, "wrong LoadStore operation");
        load_store = _gvn.transform(new (C) CompareAndSwapPNode(control(), mem, adr, newval, oldval));
      }
    }
    post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
    break;
  default:
    fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
    break;
  }
  Node* proj = _gvn.transform(new (C) SCMemProjNode(load_store));
  set_memory(proj, alias_idx);
  Node* access = load_store;
  if (type == T_OBJECT && kind == LS_xchg) {
#ifdef _LP64
    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
      load_store = _gvn.transform(new (C) DecodeNNode(load_store, load_store->get_ptr_type()));
    }
#endif
    if (can_move_pre_barrier()) {
      pre_barrier(false /* do_load */,
                  control(), NULL, NULL, max_juint, NULL, NULL,
                  load_store /* pre_val */,
                  T_OBJECT);
    }
  }
  insert_mem_bar(Op_MemBarCPUOrder);
  Node* mb = insert_mem_bar(Op_MemBarAcquire, access);
  MemBarNode::set_load_store_pair(leading_membar->as_MemBar(), mb->as_MemBar());
  assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
  set_result(load_store);
  return true;
}
bool LibraryCallKit::inline_unsafe_ordered_store(BasicType type) {
  if (callee()->is_static())  return false;  // caller must have the capability!
#ifndef PRODUCT
  {
    ResourceMark rm;
    ciSignature* sig = callee()->signature();
#ifdef ASSERT
    BasicType rtype = sig->return_type()->basic_type();
    assert(rtype == T_VOID, "must return void");
    assert(sig->count() == 3, "has 3 arguments");
    assert(sig->type_at(0)->basic_type() == T_OBJECT, "base is object");
    assert(sig->type_at(1)->basic_type() == T_LONG, "offset is long");
#endif // ASSERT
  }
#endif //PRODUCT
  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
  Node* receiver = argument(0);  // type: oop
  Node* base     = argument(1);  // type: oop
  Node* offset   = argument(2);  // type: long
  Node* val      = argument(4);  // type: oop, int, or long
  receiver = null_check(receiver);
  if (stopped()) {
    return true;
  }
  assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
  offset = ConvL2X(offset);
  Node* adr = make_unsafe_address(base, offset);
  const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
  const Type *value_type = Type::get_const_basic_type(type);
  Compile::AliasType* alias_type = C->alias_type(adr_type);
  insert_mem_bar(Op_MemBarRelease);
  insert_mem_bar(Op_MemBarCPUOrder);
  const bool require_atomic_access = true;
  Node* store;
  if (type == T_OBJECT) // reference stores need a store barrier.
    store = store_oop_to_unknown(control(), base, adr, adr_type, val, type, MemNode::release);
  else {
    store = store_to_memory(control(), adr, val, type, adr_type, MemNode::release, require_atomic_access);
  }
  insert_mem_bar(Op_MemBarCPUOrder);
  return true;
}
bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) {
  insert_mem_bar(Op_MemBarCPUOrder);
  switch(id) {
    case vmIntrinsics::_loadFence:
      insert_mem_bar(Op_LoadFence);
      return true;
    case vmIntrinsics::_storeFence:
      insert_mem_bar(Op_StoreFence);
      return true;
    case vmIntrinsics::_fullFence:
      insert_mem_bar(Op_MemBarVolatile);
      return true;
    default:
      fatal_unexpected_iid(id);
      return false;
  }
}
bool LibraryCallKit::klass_needs_init_guard(Node* kls) {
  if (!kls->is_Con()) {
    return true;
  }
  const TypeKlassPtr* klsptr = kls->bottom_type()->isa_klassptr();
  if (klsptr == NULL) {
    return true;
  }
  ciInstanceKlass* ik = klsptr->klass()->as_instance_klass();
  return !ik->is_initialized();
}
bool LibraryCallKit::inline_unsafe_allocate() {
  if (callee()->is_static())  return false;  // caller must have the capability!
  null_check_receiver();  // null-check, then ignore
  Node* cls = null_check(argument(1));
  if (stopped())  return true;
  Node* kls = load_klass_from_mirror(cls, false, NULL, 0);
  kls = null_check(kls);
  if (stopped())  return true;  // argument was like int.class
  Node* test = NULL;
  if (LibraryCallKit::klass_needs_init_guard(kls)) {
    Node* insp = basic_plus_adr(kls, in_bytes(InstanceKlass::init_state_offset()));
    Node* inst = make_load(NULL, insp, TypeInt::UBYTE, T_BOOLEAN, MemNode::unordered);
    Node* bits = intcon(InstanceKlass::fully_initialized);
    test = _gvn.transform(new (C) SubINode(inst, bits));
  }
  Node* obj = new_instance(kls, test);
  set_result(obj);
  return true;
}
#ifdef JFR_HAVE_INTRINSICS
bool LibraryCallKit::inline_native_classID() {
  Node* cls = null_check(argument(0), T_OBJECT);
  Node* kls = load_klass_from_mirror(cls, false, NULL, 0);
  kls = null_check(kls, T_OBJECT);
  ByteSize offset = KLASS_TRACE_ID_OFFSET;
  Node* insp = basic_plus_adr(kls, in_bytes(offset));
  Node* tvalue = make_load(NULL, insp, TypeLong::LONG, T_LONG, MemNode::unordered);
  Node* clsused = longcon(0x01l); // set the class bit
  Node* orl = _gvn.transform(new (C) OrLNode(tvalue, clsused));
  const TypePtr *adr_type = _gvn.type(insp)->isa_ptr();
  store_to_memory(control(), insp, orl, T_LONG, adr_type, MemNode::unordered);
#ifdef TRACE_ID_META_BITS
  Node* mbits = longcon(~TRACE_ID_META_BITS);
  tvalue = _gvn.transform(new (C) AndLNode(tvalue, mbits));
#endif
#ifdef TRACE_ID_SHIFT
  Node* cbits = intcon(TRACE_ID_SHIFT);
  tvalue = _gvn.transform(new (C) URShiftLNode(tvalue, cbits));
#endif
  set_result(tvalue);
  return true;
}
bool LibraryCallKit::inline_native_getEventWriter() {
  Node* tls_ptr = _gvn.transform(new (C) ThreadLocalNode());
  Node* jobj_ptr = basic_plus_adr(top(), tls_ptr,
                                  in_bytes(THREAD_LOCAL_WRITER_OFFSET_JFR)
                                  );
  Node* jobj = make_load(control(), jobj_ptr, TypeRawPtr::BOTTOM, T_ADDRESS, MemNode::unordered);
  Node* jobj_cmp_null = _gvn.transform( new (C) CmpPNode(jobj, null()) );
  Node* test_jobj_eq_null  = _gvn.transform( new (C) BoolNode(jobj_cmp_null, BoolTest::eq) );
  IfNode* iff_jobj_null =
    create_and_map_if(control(), test_jobj_eq_null, PROB_MIN, COUNT_UNKNOWN);
  enum { _normal_path = 1,
         _null_path = 2,
         PATH_LIMIT };
  RegionNode* result_rgn = new (C) RegionNode(PATH_LIMIT);
  PhiNode*    result_val = new (C) PhiNode(result_rgn, TypePtr::BOTTOM);
  Node* jobj_is_null = _gvn.transform(new (C) IfTrueNode(iff_jobj_null));
  result_rgn->init_req(_null_path, jobj_is_null);
  result_val->init_req(_null_path, null());
  Node* jobj_is_not_null = _gvn.transform(new (C) IfFalseNode(iff_jobj_null));
  result_rgn->init_req(_normal_path, jobj_is_not_null);
  Node* res = make_load(jobj_is_not_null, jobj, TypeInstPtr::NOTNULL, T_OBJECT, MemNode::unordered);
  result_val->init_req(_normal_path, res);
  set_result(result_rgn, result_val);
  return true;
}
#endif // JFR_HAVE_INTRINSICS
bool LibraryCallKit::inline_native_time_funcs(address funcAddr, const char* funcName) {
  const TypeFunc* tf = OptoRuntime::void_long_Type();
  const TypePtr* no_memory_effects = NULL;
  Node* time = make_runtime_call(RC_LEAF, tf, funcAddr, funcName, no_memory_effects);
  Node* value = _gvn.transform(new (C) ProjNode(time, TypeFunc::Parms+0));
#ifdef ASSERT
  Node* value_top = _gvn.transform(new (C) ProjNode(time, TypeFunc::Parms+1));
  assert(value_top == top(), "second value must be top");
#endif
  set_result(value);
  return true;
}
bool LibraryCallKit::inline_native_currentThread() {
  Node* junk = NULL;
  set_result(generate_current_thread(junk));
  return true;
}
bool LibraryCallKit::inline_native_isInterrupted() {
  enum {
    no_int_result_path   = 1, // t == Thread.current() && !TLS._osthread._interrupted
    no_clear_result_path = 2, // t == Thread.current() &&  TLS._osthread._interrupted && !clear_int
    slow_result_path     = 3, // slow path: t.isInterrupted(clear_int)
    PATH_LIMIT
  };
  insert_mem_bar(Op_MemBarCPUOrder);
  RegionNode* result_rgn = new (C) RegionNode(PATH_LIMIT);
  PhiNode*    result_val = new (C) PhiNode(result_rgn, TypeInt::BOOL);
  RegionNode* slow_region = new (C) RegionNode(1);
  record_for_igvn(slow_region);
  Node* rec_thr = argument(0);
  Node* tls_ptr = NULL;
  Node* cur_thr = generate_current_thread(tls_ptr);
  Node* cmp_thr = _gvn.transform(new (C) CmpPNode(cur_thr, rec_thr));
  Node* bol_thr = _gvn.transform(new (C) BoolNode(cmp_thr, BoolTest::ne));
  generate_slow_guard(bol_thr, slow_region);
  Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
  Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
  p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
  Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT, MemNode::unordered);
  Node* cmp_bit = _gvn.transform(new (C) CmpINode(int_bit, intcon(0)));
  Node* bol_bit = _gvn.transform(new (C) BoolNode(cmp_bit, BoolTest::ne));
  IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
  Node* false_bit = _gvn.transform(new (C) IfFalseNode(iff_bit));
  result_rgn->init_req(no_int_result_path, false_bit);
  result_val->init_req(no_int_result_path, intcon(0));
  set_control( _gvn.transform(new (C) IfTrueNode(iff_bit)));
#ifndef TARGET_OS_FAMILY_windows
  Node* clr_arg = argument(1);
  Node* cmp_arg = _gvn.transform(new (C) CmpINode(clr_arg, intcon(0)));
  Node* bol_arg = _gvn.transform(new (C) BoolNode(cmp_arg, BoolTest::ne));
  IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN);
  Node* false_arg = _gvn.transform(new (C) IfFalseNode(iff_arg));
  result_rgn->init_req(no_clear_result_path, false_arg);
  result_val->init_req(no_clear_result_path, intcon(1));
  set_control( _gvn.transform(new (C) IfTrueNode(iff_arg)));
#else
#endif // TARGET_OS_FAMILY_windows
  slow_region->add_req(control());
  set_control( _gvn.transform(slow_region));
  if (stopped()) {
    result_rgn->init_req(slow_result_path, top());
    result_val->init_req(slow_result_path, top());
  } else {
    CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted);
    Node* slow_val = set_results_for_java_call(slow_call);
    Node* fast_io  = slow_call->in(TypeFunc::I_O);
    Node* fast_mem = slow_call->in(TypeFunc::Memory);
    PhiNode* result_mem  = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM);
    PhiNode* result_io   = PhiNode::make(result_rgn, fast_io,  Type::ABIO);
    result_rgn->init_req(slow_result_path, control());
    result_io ->init_req(slow_result_path, i_o());
    result_mem->init_req(slow_result_path, reset_memory());
    result_val->init_req(slow_result_path, slow_val);
    set_all_memory(_gvn.transform(result_mem));
    set_i_o(       _gvn.transform(result_io));
  }
  C->set_has_split_ifs(true); // Has chance for split-if optimization
  set_result(result_rgn, result_val);
  return true;
}
Node* LibraryCallKit::load_mirror_from_klass(Node* klass) {
  Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
  return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT, MemNode::unordered);
}
Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror,
                                                    bool never_see_null,
                                                    RegionNode* region,
                                                    int null_path,
                                                    int offset) {
  if (region == NULL)  never_see_null = true;
  Node* p = basic_plus_adr(mirror, offset);
  const TypeKlassPtr*  kls_type = TypeKlassPtr::OBJECT_OR_NULL;
  Node* kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type));
  Node* null_ctl = top();
  kls = null_check_oop(kls, &null_ctl, never_see_null);
  if (region != NULL) {
    region->init_req(null_path, null_ctl);
  } else {
    assert(null_ctl == top(), "no loose ends");
  }
  return kls;
}
Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) {
  Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
  Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT, MemNode::unordered);
  Node* mask = intcon(modifier_mask);
  Node* bits = intcon(modifier_bits);
  Node* mbit = _gvn.transform(new (C) AndINode(mods, mask));
  Node* cmp  = _gvn.transform(new (C) CmpINode(mbit, bits));
  Node* bol  = _gvn.transform(new (C) BoolNode(cmp, BoolTest::ne));
  return generate_fair_guard(bol, region);
}
Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) {
  return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region);
}
bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) {
  const Type* return_type = TypeInt::BOOL;
  Node* prim_return_value = top();  // what happens if it's a primitive class?
  bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
  bool expect_prim = false;     // most of these guys expect to work on refs
  enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT };
  Node* mirror = argument(0);
  Node* obj    = top();
  switch (id) {
  case vmIntrinsics::_isInstance:
    prim_return_value = intcon(0);
    obj = argument(1);
    break;
  case vmIntrinsics::_getModifiers:
    prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
    assert(is_power_of_2((int)JVM_ACC_WRITTEN_FLAGS+1), "change next line");
    return_type = TypeInt::make(0, JVM_ACC_WRITTEN_FLAGS, Type::WidenMin);
    break;
  case vmIntrinsics::_isInterface:
    prim_return_value = intcon(0);
    break;
  case vmIntrinsics::_isArray:
    prim_return_value = intcon(0);
    expect_prim = true;  // cf. ObjectStreamClass.getClassSignature
    break;
  case vmIntrinsics::_isPrimitive:
    prim_return_value = intcon(1);
    expect_prim = true;  // obviously
    break;
  case vmIntrinsics::_getSuperclass:
    prim_return_value = null();
    return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR);
    break;
  case vmIntrinsics::_getComponentType:
    prim_return_value = null();
    return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR);
    break;
  case vmIntrinsics::_getClassAccessFlags:
    prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
    return_type = TypeInt::INT;  // not bool!  6297094
    break;
  default:
    fatal_unexpected_iid(id);
    break;
  }
  const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr();
  if (mirror_con == NULL)  return false;  // cannot happen?
#ifndef PRODUCT
  if (C->print_intrinsics() || C->print_inlining()) {
    ciType* k = mirror_con->java_mirror_type();
    if (k) {
      tty->print("Inlining %s on constant Class ", vmIntrinsics::name_at(intrinsic_id()));
      k->print_name();
      tty->cr();
    }
  }
#endif
  RegionNode* region = new (C) RegionNode(PATH_LIMIT);
  record_for_igvn(region);
  PhiNode* phi = new (C) PhiNode(region, return_type);
  mirror = null_check(mirror);
  if (stopped())  return true;
  if (expect_prim)  never_see_null = false;  // expect nulls (meaning prims)
  Node* kls = load_klass_from_mirror(mirror, never_see_null, region, _prim_path);
  phi->init_req(_prim_path, prim_return_value);
  if (stopped()) { set_result(region, phi); return true; }
  bool safe_for_replace = (region->in(_prim_path) == top());
  Node* p;  // handy temp
  Node* null_ctl;
  Node* query_value = top();
  switch (id) {
  case vmIntrinsics::_isInstance:
    query_value = gen_instanceof(obj, kls, safe_for_replace);
    break;
  case vmIntrinsics::_getModifiers:
    p = basic_plus_adr(kls, in_bytes(Klass::modifier_flags_offset()));
    query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered);
    break;
  case vmIntrinsics::_isInterface:
    if (generate_interface_guard(kls, region) != NULL)
      phi->add_req(intcon(1));
    query_value = intcon(0);
    break;
  case vmIntrinsics::_isArray:
    if (generate_array_guard(kls, region) != NULL)
      phi->add_req(intcon(1));
    query_value = intcon(0);
    break;
  case vmIntrinsics::_isPrimitive:
    query_value = intcon(0); // "normal" path produces false
    break;
  case vmIntrinsics::_getSuperclass:
    if (generate_interface_guard(kls, region) != NULL)
      phi->add_req(null());
    if (generate_array_guard(kls, region) != NULL)
      phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror())));
    p = basic_plus_adr(kls, in_bytes(Klass::super_offset()));
    kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL));
    null_ctl = top();
    kls = null_check_oop(kls, &null_ctl);
    if (null_ctl != top()) {
      region->add_req(null_ctl);
      phi   ->add_req(null());
    }
    if (!stopped()) {
      query_value = load_mirror_from_klass(kls);
    }
    break;
  case vmIntrinsics::_getComponentType:
    if (generate_array_guard(kls, region) != NULL) {
      Node* is_array_ctrl = region->in(region->req()-1);
      Node* cma = basic_plus_adr(kls, in_bytes(ArrayKlass::component_mirror_offset()));
      Node* cmo = make_load(is_array_ctrl, cma, TypeInstPtr::MIRROR, T_OBJECT, MemNode::unordered);
      phi->add_req(cmo);
    }
    query_value = null();  // non-array case is null
    break;
  case vmIntrinsics::_getClassAccessFlags:
    p = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
    query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered);
    break;
  default:
    fatal_unexpected_iid(id);
    break;
  }
  phi->init_req(1, query_value);
  region->init_req(1, control());
  C->set_has_split_ifs(true); // Has chance for split-if optimization
  set_result(region, phi);
  return true;
}
bool LibraryCallKit::inline_native_subtype_check() {
  Node* args[2];                // two java.lang.Class mirrors: superc, subc
  args[0] = argument(0);
  args[1] = argument(1);
  Node* klasses[2];             // corresponding Klasses: superk, subk
  klasses[0] = klasses[1] = top();
  enum {
    _prim_0_path = 1,           // {P,N} => false
    _prim_same_path,            // {P,P} & superc==subc => true
    _prim_1_path,               // {N,P} => false
    _ref_subtype_path,          // {N,N} & subtype check wins => true
    _both_ref_path,             // {N,N} & subtype check loses => false
    PATH_LIMIT
  };
  RegionNode* region = new (C) RegionNode(PATH_LIMIT);
  Node*       phi    = new (C) PhiNode(region, TypeInt::BOOL);
  record_for_igvn(region);
  const TypePtr* adr_type = TypeRawPtr::BOTTOM;   // memory type of loads
  const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL;
  int class_klass_offset = java_lang_Class::klass_offset_in_bytes();
  int which_arg;
  for (which_arg = 0; which_arg <= 1; which_arg++) {
    Node* arg = args[which_arg];
    arg = null_check(arg);
    if (stopped())  break;
    args[which_arg] = arg;
    Node* p = basic_plus_adr(arg, class_klass_offset);
    Node* kls = LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, adr_type, kls_type);
    klasses[which_arg] = _gvn.transform(kls);
  }
  bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
  for (which_arg = 0; which_arg <= 1; which_arg++) {
    Node* kls = klasses[which_arg];
    Node* null_ctl = top();
    kls = null_check_oop(kls, &null_ctl, never_see_null);
    int prim_path = (which_arg == 0 ? _prim_0_path : _prim_1_path);
    region->init_req(prim_path, null_ctl);
    if (stopped())  break;
    klasses[which_arg] = kls;
  }
  if (!stopped()) {
    Node* subk   = klasses[1];  // the argument to isAssignableFrom
    Node* superk = klasses[0];  // the receiver
    region->set_req(_both_ref_path, gen_subtype_check(subk, superk));
    region->set_req(_ref_subtype_path, control());
  }
  set_control(region->in(_prim_0_path)); // go back to first null check
  if (!stopped()) {
    Node* cmp_eq = _gvn.transform(new (C) CmpPNode(args[0], args[1]));
    Node* bol_eq = _gvn.transform(new (C) BoolNode(cmp_eq, BoolTest::eq));
    generate_guard(bol_eq, region, PROB_FAIR);
    if (region->req() == PATH_LIMIT+1) {
      region->swap_edges(PATH_LIMIT, _prim_same_path);
      region->del_req(PATH_LIMIT);
    }
    region->set_req(_prim_0_path, control()); // Not equal after all.
  }
  phi->set_req(_prim_same_path,   intcon(1));
  phi->set_req(_ref_subtype_path, intcon(1));
  assert(region->req() == PATH_LIMIT, "sane region");
  for (uint i = 1; i < region->req(); i++) {
    Node* ctl = region->in(i);
    if (ctl == NULL || ctl == top()) {
      region->set_req(i, top());
      phi   ->set_req(i, top());
    } else if (phi->in(i) == NULL) {
      phi->set_req(i, intcon(0)); // all other paths produce 'false'
    }
  }
  set_control(_gvn.transform(region));
  set_result(_gvn.transform(phi));
  return true;
}
Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region,
                                                  bool obj_array, bool not_array) {
  jint  layout_con = 0;
  Node* layout_val = get_layout_helper(kls, layout_con);
  if (layout_val == NULL) {
    bool query = (obj_array
                  ? Klass::layout_helper_is_objArray(layout_con)
                  : Klass::layout_helper_is_array(layout_con));
    if (query == not_array) {
      return NULL;                       // never a branch
    } else {                             // always a branch
      Node* always_branch = control();
      if (region != NULL)
        region->add_req(always_branch);
      set_control(top());
      return always_branch;
    }
  }
  jint  nval = (obj_array
                ? (jint)(Klass::_lh_array_tag_type_value
                   <<    Klass::_lh_array_tag_shift)
                : Klass::_lh_neutral_value);
  Node* cmp = _gvn.transform(new(C) CmpINode(layout_val, intcon(nval)));
  BoolTest::mask btest = BoolTest::lt;  // correct for testing is_[obj]array
  if (not_array)  btest = BoolTest(btest).negate();
  Node* bol = _gvn.transform(new(C) BoolNode(cmp, btest));
  return generate_fair_guard(bol, region);
}
bool LibraryCallKit::inline_native_newArray() {
  Node* mirror    = argument(0);
  Node* count_val = argument(1);
  mirror = null_check(mirror);
  if (stopped())  return true;
  enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT };
  RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
  PhiNode*    result_val = new(C) PhiNode(result_reg,
                                          TypeInstPtr::NOTNULL);
  PhiNode*    result_io  = new(C) PhiNode(result_reg, Type::ABIO);
  PhiNode*    result_mem = new(C) PhiNode(result_reg, Type::MEMORY,
                                          TypePtr::BOTTOM);
  bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
  Node* klass_node = load_array_klass_from_mirror(mirror, never_see_null,
                                                  result_reg, _slow_path);
  Node* normal_ctl   = control();
  Node* no_array_ctl = result_reg->in(_slow_path);
  set_control(no_array_ctl);
  if (!stopped()) {
    PreserveJVMState pjvms(this);
    CallJavaNode* slow_call = generate_method_call_static(vmIntrinsics::_newArray);
    Node* slow_result = set_results_for_java_call(slow_call);
    result_reg->set_req(_slow_path, control());
    result_val->set_req(_slow_path, slow_result);
    result_io ->set_req(_slow_path, i_o());
    result_mem->set_req(_slow_path, reset_memory());
  }
  set_control(normal_ctl);
  if (!stopped()) {
    Node* obj = new_array(klass_node, count_val, 0);  // no arguments to push
    result_reg->init_req(_normal_path, control());
    result_val->init_req(_normal_path, obj);
    result_io ->init_req(_normal_path, i_o());
    result_mem->init_req(_normal_path, reset_memory());
  }
  set_i_o(        _gvn.transform(result_io)  );
  set_all_memory( _gvn.transform(result_mem));
  C->set_has_split_ifs(true); // Has chance for split-if optimization
  set_result(result_reg, result_val);
  return true;
}
bool LibraryCallKit::inline_native_getLength() {
  if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;
  Node* array = null_check(argument(0));
  if (stopped())  return true;
  Node* non_array = generate_non_array_guard(load_object_klass(array), NULL);
  if (non_array != NULL) {
    PreserveJVMState pjvms(this);
    set_control(non_array);
    uncommon_trap(Deoptimization::Reason_intrinsic,
                  Deoptimization::Action_maybe_recompile);
  }
  if (stopped())  return true;
  Node* result = load_array_length(array);
  C->set_has_split_ifs(true);  // Has chance for split-if optimization
  set_result(result);
  return true;
}
bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) {
  if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;
  Node* original          = argument(0);
  Node* start             = is_copyOfRange? argument(1): intcon(0);
  Node* end               = is_copyOfRange? argument(2): argument(1);
  Node* array_type_mirror = is_copyOfRange? argument(3): argument(2);
  Node* newcopy = NULL;
  { PreserveReexecuteState preexecs(this);
    jvms()->set_should_reexecute(true);
    array_type_mirror = null_check(array_type_mirror);
    original          = null_check(original);
    if (stopped())  return true;
    Node* orig_length = load_array_length(original);
    Node* klass_node = load_klass_from_mirror(array_type_mirror, false, NULL, 0);
    klass_node = null_check(klass_node);
    RegionNode* bailout = new (C) RegionNode(1);
    record_for_igvn(bailout);
    Node* not_objArray = generate_non_objArray_guard(klass_node, bailout);
    if (not_objArray != NULL) {
      ciKlass* ak = ciArrayKlass::make(env()->Object_klass());
      const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/);
      Node* cast = new (C) CastPPNode(klass_node, akls);
      cast->init_req(0, control());
      klass_node = _gvn.transform(cast);
    }
    generate_negative_guard(start, bailout, &start);
    generate_negative_guard(end,   bailout, &end);
    Node* length = end;
    if (_gvn.type(start) != TypeInt::ZERO) {
      length = _gvn.transform(new (C) SubINode(end, start));
    }
    generate_negative_guard(length, bailout, &length);
    if (bailout->req() > 1) {
      PreserveJVMState pjvms(this);
      set_control(_gvn.transform(bailout));
      uncommon_trap(Deoptimization::Reason_intrinsic,
                    Deoptimization::Action_maybe_recompile);
    }
    if (!stopped()) {
      Node* orig_tail = _gvn.transform(new (C) SubINode(orig_length, start));
      Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length);
      newcopy = new_array(klass_node, length, 0);  // no argments to push
      bool disjoint_bases = true;
      bool length_never_negative = !is_copyOfRange;
      generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
                         original, start, newcopy, intcon(0), moved,
                         disjoint_bases, length_never_negative);
    }
  } // original reexecute is set back here
  C->set_has_split_ifs(true); // Has chance for split-if optimization
  if (!stopped()) {
    set_result(newcopy);
  }
  return true;
}
Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
                                             RegionNode* slow_region) {
  ciMethod* method = callee();
  int vtable_index = method->vtable_index();
  assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index,
         err_msg_res("bad index %d", vtable_index));
  int entry_offset  = (InstanceKlass::vtable_start_offset() +
                     vtable_index*vtableEntry::size()) * wordSize +
                     vtableEntry::method_offset_in_bytes();
  Node* entry_addr  = basic_plus_adr(obj_klass, entry_offset);
  Node* target_call = make_load(NULL, entry_addr, TypePtr::NOTNULL, T_ADDRESS, MemNode::unordered);
  const TypePtr* native_call_addr = TypeMetadataPtr::make(method);
  Node* native_call = makecon(native_call_addr);
  Node* chk_native  = _gvn.transform(new(C) CmpPNode(target_call, native_call));
  Node* test_native = _gvn.transform(new(C) BoolNode(chk_native, BoolTest::ne));
  return generate_slow_guard(test_native, slow_region);
}
CallJavaNode*
LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) {
  guarantee(callee() != C->method(), "cannot make slow-call to self");
  ciMethod* method = callee();
  guarantee(method_id == method->intrinsic_id(), "must match");
  const TypeFunc* tf = TypeFunc::make(method);
  CallJavaNode* slow_call;
  if (is_static) {
    assert(!is_virtual, "");
    slow_call = new(C) CallStaticJavaNode(C, tf,
                           SharedRuntime::get_resolve_static_call_stub(),
                           method, bci());
  } else if (is_virtual) {
    null_check_receiver();
    int vtable_index = Method::invalid_vtable_index;
    if (UseInlineCaches) {
    } else {
       vtable_index = method->vtable_index();
       assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index,
              err_msg_res("bad index %d", vtable_index));
    }
    slow_call = new(C) CallDynamicJavaNode(tf,
                          SharedRuntime::get_resolve_virtual_call_stub(),
                          method, vtable_index, bci());
  } else {  // neither virtual nor static:  opt_virtual
    null_check_receiver();
    slow_call = new(C) CallStaticJavaNode(C, tf,
                                SharedRuntime::get_resolve_opt_virtual_call_stub(),
                                method, bci());
    slow_call->set_optimized_virtual(true);
  }
  set_arguments_for_java_call(slow_call);
  set_edges_for_java_call(slow_call);
  return slow_call;
}
bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) {
  assert(is_static == callee()->is_static(), "correct intrinsic selection");
  assert(!(is_virtual && is_static), "either virtual, special, or static");
  enum { _slow_path = 1, _fast_path, _null_path, PATH_LIMIT };
  RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
  PhiNode*    result_val = new(C) PhiNode(result_reg, TypeInt::INT);
  PhiNode*    result_io  = new(C) PhiNode(result_reg, Type::ABIO);
  PhiNode*    result_mem = new(C) PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
  Node* obj = NULL;
  if (!is_static) {
    obj = null_check_receiver();
    if (stopped())  return true;        // unconditionally null
    result_reg->init_req(_null_path, top());
    result_val->init_req(_null_path, top());
  } else {
    obj = argument(0);
    Node* null_ctl = top();
    obj = null_check_oop(obj, &null_ctl);
    result_reg->init_req(_null_path, null_ctl);
    result_val->init_req(_null_path, _gvn.intcon(0));
  }
  if (stopped()) {
    set_control( result_reg->in(_null_path));
    if (!stopped())
      set_result(result_val->in(_null_path));
    return true;
  }
  RegionNode* slow_region = new (C) RegionNode(1);
  record_for_igvn(slow_region);
  if (is_virtual) {
    Node* obj_klass = load_object_klass(obj);
    generate_virtual_guard(obj_klass, slow_region);
  }
  Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
  Node* no_ctrl = NULL;
  Node* header = make_load(no_ctrl, header_addr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered);
  Node* lock_mask      = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place);
  Node* lmasked_header = _gvn.transform(new (C) AndXNode(header, lock_mask));
  Node* unlocked_val   = _gvn.MakeConX(markOopDesc::unlocked_value);
  Node* chk_unlocked   = _gvn.transform(new (C) CmpXNode( lmasked_header, unlocked_val));
  Node* test_unlocked  = _gvn.transform(new (C) BoolNode( chk_unlocked, BoolTest::ne));
  generate_slow_guard(test_unlocked, slow_region);
  Node* hash_mask      = _gvn.intcon(markOopDesc::hash_mask);
  Node* hash_shift     = _gvn.intcon(markOopDesc::hash_shift);
  Node* hshifted_header= _gvn.transform(new (C) URShiftXNode(header, hash_shift));
  hshifted_header      = ConvX2I(hshifted_header);
  Node* hash_val       = _gvn.transform(new (C) AndINode(hshifted_header, hash_mask));
  Node* no_hash_val    = _gvn.intcon(markOopDesc::no_hash);
  Node* chk_assigned   = _gvn.transform(new (C) CmpINode( hash_val, no_hash_val));
  Node* test_assigned  = _gvn.transform(new (C) BoolNode( chk_assigned, BoolTest::eq));
  generate_slow_guard(test_assigned, slow_region);
  Node* init_mem = reset_memory();
  result_io ->init_req(_null_path, i_o());
  result_mem->init_req(_null_path, init_mem);
  result_val->init_req(_fast_path, hash_val);
  result_reg->init_req(_fast_path, control());
  result_io ->init_req(_fast_path, i_o());
  result_mem->init_req(_fast_path, init_mem);
  set_control(_gvn.transform(slow_region));
  if (!stopped()) {
    set_all_memory(init_mem);
    vmIntrinsics::ID hashCode_id = is_static ? vmIntrinsics::_identityHashCode : vmIntrinsics::_hashCode;
    CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static);
    Node* slow_result = set_results_for_java_call(slow_call);
    result_reg->init_req(_slow_path, control());
    result_val->init_req(_slow_path, slow_result);
    result_io  ->set_req(_slow_path, i_o());
    result_mem ->set_req(_slow_path, reset_memory());
  }
  set_i_o(        _gvn.transform(result_io)  );
  set_all_memory( _gvn.transform(result_mem));
  set_result(result_reg, result_val);
  return true;
}
bool LibraryCallKit::inline_native_getClass() {
  Node* obj = null_check_receiver();
  if (stopped())  return true;
  set_result(load_mirror_from_klass(load_object_klass(obj)));
  return true;
}
bool LibraryCallKit::inline_native_Reflection_getCallerClass() {
#ifndef PRODUCT
  if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
    tty->print_cr("Attempting to inline sun.reflect.Reflection.getCallerClass");
  }
#endif
  if (!jvms()->has_method()) {
#ifndef PRODUCT
    if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
      tty->print_cr("  Bailing out because intrinsic was inlined at top level");
    }
#endif
    return false;
  }
  JVMState* caller_jvms = jvms();
  for (int n = 1; caller_jvms != NULL; caller_jvms = caller_jvms->caller(), n++) {
    ciMethod* m = caller_jvms->method();
    switch (n) {
    case 0:
      fatal("current JVM state does not include the Reflection.getCallerClass frame");
      break;
    case 1:
      if (!m->caller_sensitive()) {
#ifndef PRODUCT
        if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
          tty->print_cr("  Bailing out: CallerSensitive annotation expected at frame %d", n);
        }
#endif
        return false;  // bail-out; let JVM_GetCallerClass do the work
      }
      break;
    default:
      if (!m->is_ignored_by_security_stack_walk()) {
        ciInstanceKlass* caller_klass = caller_jvms->method()->holder();
        ciInstance* caller_mirror = caller_klass->java_mirror();
        set_result(makecon(TypeInstPtr::make(caller_mirror)));
#ifndef PRODUCT
        if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
          tty->print_cr("  Succeeded: caller = %d) %s.%s, JVMS depth = %d", n, caller_klass->name()->as_utf8(), caller_jvms->method()->name()->as_utf8(), jvms()->depth());
          tty->print_cr("  JVM state at this point:");
          for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) {
            ciMethod* m = jvms()->of_depth(i)->method();
            tty->print_cr("   %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8());
          }
        }
#endif
        return true;
      }
      break;
    }
  }
#ifndef PRODUCT
  if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
    tty->print_cr("  Bailing out because caller depth exceeded inlining depth = %d", jvms()->depth());
    tty->print_cr("  JVM state at this point:");
    for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) {
      ciMethod* m = jvms()->of_depth(i)->method();
      tty->print_cr("   %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8());
    }
  }
#endif
  return false;  // bail-out; let JVM_GetCallerClass do the work
}
bool LibraryCallKit::inline_fp_conversions(vmIntrinsics::ID id) {
  Node* arg = argument(0);
  Node* result = NULL;
  switch (id) {
  case vmIntrinsics::_floatToRawIntBits:    result = new (C) MoveF2INode(arg);  break;
  case vmIntrinsics::_intBitsToFloat:       result = new (C) MoveI2FNode(arg);  break;
  case vmIntrinsics::_doubleToRawLongBits:  result = new (C) MoveD2LNode(arg);  break;
  case vmIntrinsics::_longBitsToDouble:     result = new (C) MoveL2DNode(arg);  break;
  case vmIntrinsics::_doubleToLongBits: {
    RegionNode *r = new (C) RegionNode(3);
    Node *phi = new (C) PhiNode(r, TypeLong::LONG);
    Node *cmpisnan = _gvn.transform(new (C) CmpDNode(arg, arg));
    Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
    IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
    Node *opt_isnan = _gvn.transform(ifisnan);
    assert( opt_isnan->is_If(), "Expect an IfNode");
    IfNode *opt_ifisnan = (IfNode*)opt_isnan;
    Node *iftrue = _gvn.transform(new (C) IfTrueNode(opt_ifisnan));
    set_control(iftrue);
    static const jlong nan_bits = CONST64(0x7ff8000000000000);
    Node *slow_result = longcon(nan_bits); // return NaN
    phi->init_req(1, _gvn.transform( slow_result ));
    r->init_req(1, iftrue);
    Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
    set_control(iffalse);
    phi->init_req(2, _gvn.transform(new (C) MoveD2LNode(arg)));
    r->init_req(2, iffalse);
    set_control(_gvn.transform(r));
    record_for_igvn(r);
    C->set_has_split_ifs(true); // Has chance for split-if optimization
    result = phi;
    assert(result->bottom_type()->isa_long(), "must be");
    break;
  }
  case vmIntrinsics::_floatToIntBits: {
    RegionNode *r = new (C) RegionNode(3);
    Node *phi = new (C) PhiNode(r, TypeInt::INT);
    Node *cmpisnan = _gvn.transform(new (C) CmpFNode(arg, arg));
    Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
    IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
    Node *opt_isnan = _gvn.transform(ifisnan);
    assert( opt_isnan->is_If(), "Expect an IfNode");
    IfNode *opt_ifisnan = (IfNode*)opt_isnan;
    Node *iftrue = _gvn.transform(new (C) IfTrueNode(opt_ifisnan));
    set_control(iftrue);
    static const jint nan_bits = 0x7fc00000;
    Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN
    phi->init_req(1, _gvn.transform( slow_result ));
    r->init_req(1, iftrue);
    Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
    set_control(iffalse);
    phi->init_req(2, _gvn.transform(new (C) MoveF2INode(arg)));
    r->init_req(2, iffalse);
    set_control(_gvn.transform(r));
    record_for_igvn(r);
    C->set_has_split_ifs(true); // Has chance for split-if optimization
    result = phi;
    assert(result->bottom_type()->isa_int(), "must be");
    break;
  }
  default:
    fatal_unexpected_iid(id);
    break;
  }
  set_result(_gvn.transform(result));
  return true;
}
#ifdef _LP64
#define XTOP ,top() /*additional argument*/
#else  //_LP64
#define XTOP        /*no additional argument*/
#endif //_LP64
bool LibraryCallKit::inline_unsafe_copyMemory() {
  if (callee()->is_static())  return false;  // caller must have the capability!
  null_check_receiver();  // null-check receiver
  if (stopped())  return true;
  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
  Node* src_ptr =         argument(1);   // type: oop
  Node* src_off = ConvL2X(argument(2));  // type: long
  Node* dst_ptr =         argument(4);   // type: oop
  Node* dst_off = ConvL2X(argument(5));  // type: long
  Node* size    = ConvL2X(argument(7));  // type: long
  assert(Unsafe_field_offset_to_byte_offset(11) == 11,
         "fieldOffset must be byte-scaled");
  Node* src = make_unsafe_address(src_ptr, src_off);
  Node* dst = make_unsafe_address(dst_ptr, dst_off);
  insert_mem_bar(Op_MemBarCPUOrder);
  make_runtime_call(RC_LEAF|RC_NO_FP,
                    OptoRuntime::fast_arraycopy_Type(),
                    StubRoutines::unsafe_arraycopy(),
                    "unsafe_arraycopy",
                    TypeRawPtr::BOTTOM,
                    src, dst, size XTOP);
  insert_mem_bar(Op_MemBarCPUOrder);
  return true;
}
void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) {
  assert(obj_size != NULL, "");
  Node* raw_obj = alloc_obj->in(1);
  assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
  AllocateNode* alloc = NULL;
  if (ReduceBulkZeroing) {
    alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
    guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), "");
    alloc->initialization()->set_complete_with_arraycopy();
  }
  Node* src  = obj;
  Node* dest = alloc_obj;
  Node* size = _gvn.transform(obj_size);
  int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
                            instanceOopDesc::base_offset_in_bytes();
  if (base_off % BytesPerLong != 0) {
    assert(UseCompressedClassPointers, "");
    if (is_array) {
      base_off += sizeof(int);
    } else {
      base_off = instanceOopDesc::klass_offset_in_bytes();
    }
    assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
  }
  src  = basic_plus_adr(src,  base_off);
  dest = basic_plus_adr(dest, base_off);
  Node* countx = size;
  countx = _gvn.transform(new (C) SubXNode(countx, MakeConX(base_off)));
  countx = _gvn.transform(new (C) URShiftXNode(countx, intcon(LogBytesPerLong) ));
  const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
  bool disjoint_bases = true;
  generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
                               src, NULL, dest, NULL, countx,
  if (card_mark) {
    assert(!is_array, "");
    Node* no_particular_value = NULL;
    Node* no_particular_field = NULL;
    int raw_adr_idx = Compile::AliasIdxRaw;
    post_barrier(control(),
                 memory(raw_adr_type),
                 alloc_obj,
                 no_particular_field,
                 raw_adr_idx,
                 no_particular_value,
                 T_OBJECT,
                 false);
  }
  if (alloc != NULL) {
    insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
  } else {
    insert_mem_bar(Op_MemBarCPUOrder);
  }
}
bool LibraryCallKit::inline_native_clone(bool is_virtual) {
  PhiNode* result_val;
  { PreserveReexecuteState preexecs(this);
    jvms()->set_should_reexecute(true);
    Node* obj = null_check_receiver();
    if (stopped())  return true;
    Node* obj_klass = load_object_klass(obj);
    const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
    const TypeOopPtr*   toop   = ((tklass != NULL)
                                ? tklass->as_instance_type()
                                : TypeInstPtr::NOTNULL);
    insert_mem_bar(Op_MemBarCPUOrder);
    enum {
      _slow_path = 1,     // out-of-line call to clone method (virtual or not)
      _objArray_path,     // plain array allocation, plus arrayof_oop_arraycopy
      _array_path,        // plain array allocation, plus arrayof_long_arraycopy
      _instance_path,     // plain instance allocation, plus arrayof_long_arraycopy
      PATH_LIMIT
    };
    RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
    result_val             = new(C) PhiNode(result_reg,
                                            TypeInstPtr::NOTNULL);
    PhiNode*    result_i_o = new(C) PhiNode(result_reg, Type::ABIO);
    PhiNode*    result_mem = new(C) PhiNode(result_reg, Type::MEMORY,
                                            TypePtr::BOTTOM);
    record_for_igvn(result_reg);
    const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
    int raw_adr_idx = Compile::AliasIdxRaw;
    Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
    if (array_ctl != NULL) {
      PreserveJVMState pjvms(this);
      set_control(array_ctl);
      Node* obj_length = load_array_length(obj);
      Node* obj_size  = NULL;
      Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size);  // no arguments to push
      if (!use_ReduceInitialCardMarks()) {
        Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
        if (is_obja != NULL) {
          PreserveJVMState pjvms2(this);
          set_control(is_obja);
          bool disjoint_bases = true;
          bool length_never_negative = true;
          generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
                             obj, intcon(0), alloc_obj, intcon(0),
                             obj_length,
                             disjoint_bases, length_never_negative);
          result_reg->init_req(_objArray_path, control());
          result_val->init_req(_objArray_path, alloc_obj);
          result_i_o ->set_req(_objArray_path, i_o());
          result_mem ->set_req(_objArray_path, reset_memory());
        }
      }
      if (!stopped()) {
        copy_to_clone(obj, alloc_obj, obj_size, true, false);
        result_reg->init_req(_array_path, control());
        result_val->init_req(_array_path, alloc_obj);
        result_i_o ->set_req(_array_path, i_o());
        result_mem ->set_req(_array_path, reset_memory());
      }
    }
    RegionNode* slow_region = new (C) RegionNode(1);
    record_for_igvn(slow_region);
    if (!stopped()) {
      if (is_virtual) {
        generate_virtual_guard(obj_klass, slow_region);
      }
      generate_access_flags_guard(obj_klass,
                                  JVM_ACC_IS_CLONEABLE | JVM_ACC_HAS_FINALIZER,
                                  JVM_ACC_IS_CLONEABLE,
                                  slow_region);
    }
    if (!stopped()) {
      PreserveJVMState pjvms(this);
      Node* obj_size  = NULL;
      Node* alloc_obj = new_instance(obj_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true);
      copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks());
      result_reg->init_req(_instance_path, control());
      result_val->init_req(_instance_path, alloc_obj);
      result_i_o ->set_req(_instance_path, i_o());
      result_mem ->set_req(_instance_path, reset_memory());
    }
    set_control(_gvn.transform(slow_region));
    if (!stopped()) {
      PreserveJVMState pjvms(this);
      CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
      Node* slow_result = set_results_for_java_call(slow_call);
      result_reg->init_req(_slow_path, control());
      result_val->init_req(_slow_path, slow_result);
      result_i_o ->set_req(_slow_path, i_o());
      result_mem ->set_req(_slow_path, reset_memory());
    }
    set_control(    _gvn.transform(result_reg));
    set_i_o(        _gvn.transform(result_i_o));
    set_all_memory( _gvn.transform(result_mem));
  } // original reexecute is set back here
  set_result(_gvn.transform(result_val));
  return true;
}
address LibraryCallKit::basictype2arraycopy(BasicType t,
                                            Node* src_offset,
                                            Node* dest_offset,
                                            bool disjoint_bases,
                                            const char* &name,
                                            bool dest_uninitialized) {
  const TypeInt* src_offset_inttype  = gvn().find_int_type(src_offset);;
  const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
  bool aligned = false;
  bool disjoint = disjoint_bases;
  if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
      dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
    int s_offs = src_offset_inttype->get_con();
    int d_offs = dest_offset_inttype->get_con();
    int element_size = type2aelembytes(t);
    aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
              ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
    if (s_offs >= d_offs)  disjoint = true;
  } else if (src_offset == dest_offset && src_offset != NULL) {
    disjoint = true;
  }
  return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
}
bool LibraryCallKit::inline_arraycopy() {
  Node* src         = argument(0);  // type: oop
  Node* src_offset  = argument(1);  // type: int
  Node* dest        = argument(2);  // type: oop
  Node* dest_offset = argument(3);  // type: int
  Node* length      = argument(4);  // type: int
  const Type* src_type  = src->Value(&_gvn);
  const Type* dest_type = dest->Value(&_gvn);
  const TypeAryPtr* top_src  = src_type->isa_aryptr();
  const TypeAryPtr* top_dest = dest_type->isa_aryptr();
  bool has_src = (top_src != NULL && top_src->klass() != NULL);
  bool has_dest = (top_dest != NULL && top_dest->klass() != NULL);
  bool src_spec = false;
  bool dest_spec = false;
  if (!has_src || !has_dest) {
    bool could_have_src = has_src;
    bool could_have_dest = has_dest;
    ciKlass* src_k = NULL;
    if (!has_src) {
      src_k = src_type->speculative_type();
      if (src_k != NULL && src_k->is_array_klass()) {
        could_have_src = true;
      }
    }
    ciKlass* dest_k = NULL;
    if (!has_dest) {
      dest_k = dest_type->speculative_type();
      if (dest_k != NULL && dest_k->is_array_klass()) {
        could_have_dest = true;
      }
    }
    if (could_have_src && could_have_dest) {
      if (!has_src) {
        src = maybe_cast_profiled_obj(src, src_k);
        src_type  = _gvn.type(src);
        top_src  = src_type->isa_aryptr();
        has_src = (top_src != NULL && top_src->klass() != NULL);
        src_spec = true;
      }
      if (!has_dest) {
        dest = maybe_cast_profiled_obj(dest, dest_k);
        dest_type  = _gvn.type(dest);
        top_dest  = dest_type->isa_aryptr();
        has_dest = (top_dest != NULL && top_dest->klass() != NULL);
        dest_spec = true;
      }
    }
  }
  if (!has_src || !has_dest) {
    insert_mem_bar(Op_MemBarCPUOrder);
    generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT,
                       src, src_offset, dest, dest_offset, length);
    if (!InsertMemBarAfterArraycopy)
      insert_mem_bar(Op_MemBarCPUOrder);
    return true;
  }
  BasicType src_elem  =  top_src->klass()->as_array_klass()->element_type()->basic_type();
  BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
  if (src_elem  == T_ARRAY)  src_elem  = T_OBJECT;
  if (dest_elem == T_ARRAY)  dest_elem = T_OBJECT;
  if (src_elem != dest_elem || dest_elem == T_VOID) {
    generate_slow_arraycopy(TypePtr::BOTTOM,
                            src, src_offset, dest, dest_offset, length,
    return true;
  }
  if (src_elem == T_OBJECT) {
    bool could_have_src = src_spec;
    bool could_have_dest = dest_spec;
    ciKlass* src_k = top_src->klass();
    ciKlass* dest_k = top_dest->klass();
    if (!src_spec) {
      src_k = src_type->speculative_type();
      if (src_k != NULL && src_k->is_array_klass()) {
          could_have_src = true;
      }
    }
    if (!dest_spec) {
      dest_k = dest_type->speculative_type();
      if (dest_k != NULL && dest_k->is_array_klass()) {
        could_have_dest = true;
      }
    }
    if (could_have_src && could_have_dest) {
      if (could_have_src && !src_spec) {
        src = maybe_cast_profiled_obj(src, src_k);
      }
      if (could_have_dest && !dest_spec) {
        dest = maybe_cast_profiled_obj(dest, dest_k);
      }
    }
  }
  RegionNode* slow_region = new (C) RegionNode(1);
  record_for_igvn(slow_region);
  src  = null_check(src,  T_ARRAY);
  dest = null_check(dest, T_ARRAY);
  generate_negative_guard(src_offset, slow_region);
  generate_negative_guard(dest_offset, slow_region);
  generate_limit_guard(src_offset, length,
                       load_array_length(src),
                       slow_region);
  generate_limit_guard(dest_offset, length,
                       load_array_length(dest),
                       slow_region);
  const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
  generate_arraycopy(adr_type, dest_elem,
                     src, src_offset, dest, dest_offset, length,
                     false, false, slow_region);
  return true;
}
void
LibraryCallKit::generate_arraycopy(const TypePtr* adr_type,
                                   BasicType basic_elem_type,
                                   Node* src,  Node* src_offset,
                                   Node* dest, Node* dest_offset,
                                   Node* copy_length,
                                   bool disjoint_bases,
                                   bool length_never_negative,
                                   RegionNode* slow_region) {
  if (slow_region == NULL) {
    slow_region = new(C) RegionNode(1);
    record_for_igvn(slow_region);
  }
  Node* original_dest      = dest;
  AllocateArrayNode* alloc = NULL;  // used for zeroing, if needed
  bool  dest_uninitialized = false;
  if (ReduceBulkZeroing
      && !ZeroTLAB              // pointless if already zeroed
      && basic_elem_type != T_CONFLICT // avoid corner case
      && !src->eqv_uncast(dest)
      && ((alloc = tightly_coupled_allocation(dest, slow_region))
          != NULL)
      && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
      && alloc->maybe_set_complete(&_gvn)) {
    InitializeNode* init = alloc->initialization();
    assert(init->is_complete(), "we just did this");
    init->set_complete_with_arraycopy();
    assert(dest->is_CheckCastPP(), "sanity");
    assert(dest->in(0)->in(0) == init, "dest pinned");
    adr_type = TypeRawPtr::BOTTOM;  // all initializations are into raw memory
    dest_uninitialized = true;
  } else {
    alloc             = NULL;
  }
  enum { fast_path        = 1,  // normal void-returning assembly stub
         checked_path     = 2,  // special assembly stub with cleanup
         slow_call_path   = 3,  // something went wrong; call the VM
         zero_path        = 4,  // bypass when length of copy is zero
         bcopy_path       = 5,  // copy primitive array by 64-bit blocks
         PATH_LIMIT       = 6
  };
  RegionNode* result_region = new(C) RegionNode(PATH_LIMIT);
  PhiNode*    result_i_o    = new(C) PhiNode(result_region, Type::ABIO);
  PhiNode*    result_memory = new(C) PhiNode(result_region, Type::MEMORY, adr_type);
  record_for_igvn(result_region);
  _gvn.set_type_bottom(result_i_o);
  _gvn.set_type_bottom(result_memory);
  assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
  Node* slow_control;
  Node* slow_i_o = i_o();
  Node* slow_mem = memory(adr_type);
  debug_only(slow_control = (Node*) badAddress);
  Node* checked_control = top();
  Node* checked_mem     = NULL;
  Node* checked_i_o     = NULL;
  Node* checked_value   = NULL;
  if (basic_elem_type == T_CONFLICT) {
    assert(!dest_uninitialized, "");
    Node* cv = generate_generic_arraycopy(adr_type,
                                          src, src_offset, dest, dest_offset,
                                          copy_length, dest_uninitialized);
    if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
    checked_control = control();
    checked_i_o     = i_o();
    checked_mem     = memory(adr_type);
    checked_value   = cv;
    set_control(top());         // no fast path
  }
  Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative);
  if (not_pos != NULL) {
    PreserveJVMState pjvms(this);
    set_control(not_pos);
    if (!length_never_negative) {
      generate_negative_guard(copy_length, slow_region);
    }
    if (!stopped() && dest_uninitialized) {
      Node* dest_length = alloc->in(AllocateNode::ALength);
      if (copy_length->eqv_uncast(dest_length)
          || _gvn.find_int_con(dest_length, 1) <= 0) {
      } else {
        generate_clear_array(adr_type, dest, basic_elem_type,
                             intcon(0), NULL,
                             alloc->in(AllocateNode::AllocSize));
        InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
                                                       Compile::AliasIdxRaw,
                                                       top())->as_Initialize();
        init->set_complete(&_gvn);  // (there is no corresponding AllocateNode)
      }
    }
    result_region->init_req(zero_path, control());
    result_i_o   ->init_req(zero_path, i_o());
    result_memory->init_req(zero_path, memory(adr_type));
  }
  if (!stopped() && dest_uninitialized) {
    Node* dest_size   = alloc->in(AllocateNode::AllocSize);
    Node* dest_length = alloc->in(AllocateNode::ALength);
    Node* dest_tail   = _gvn.transform(new(C) AddINode(dest_offset,
                                                          copy_length));
    if (find_int_con(dest_offset, -1) != 0) {
      generate_clear_array(adr_type, dest, basic_elem_type,
                           intcon(0), dest_offset,
                           NULL);
    }
    Node* tail_ctl = NULL;
    if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
      Node* cmp_lt   = _gvn.transform(new(C) CmpINode(dest_tail, dest_length));
      Node* bol_lt   = _gvn.transform(new(C) BoolNode(cmp_lt, BoolTest::lt));
      tail_ctl = generate_slow_guard(bol_lt, NULL);
      assert(tail_ctl != NULL || !stopped(), "must be an outcome");
    }
    if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
      bool didit = false;
      { PreserveJVMState pjvms(this);
        didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
                                         src, src_offset, dest, dest_offset,
                                         dest_size, dest_uninitialized);
        if (didit) {
          result_region->init_req(bcopy_path, control());
          result_i_o   ->init_req(bcopy_path, i_o());
          result_memory->init_req(bcopy_path, memory(adr_type));
        }
      }
      if (didit)
        set_control(top());     // no regular fast path
    }
    if (tail_ctl != NULL) {
      Node* notail_ctl = stopped() ? NULL : control();
      set_control(tail_ctl);
      if (notail_ctl == NULL) {
        generate_clear_array(adr_type, dest, basic_elem_type,
                             dest_tail, NULL,
                             dest_size);
      } else {
        Node* done_ctl = new(C) RegionNode(3);
        Node* done_mem = new(C) PhiNode(done_ctl, Type::MEMORY, adr_type);
        done_ctl->init_req(1, notail_ctl);
        done_mem->init_req(1, memory(adr_type));
        generate_clear_array(adr_type, dest, basic_elem_type,
                             dest_tail, NULL,
                             dest_size);
        done_ctl->init_req(2, control());
        done_mem->init_req(2, memory(adr_type));
        set_control( _gvn.transform(done_ctl));
        set_memory(  _gvn.transform(done_mem), adr_type );
      }
    }
  }
  BasicType copy_type = basic_elem_type;
  assert(basic_elem_type != T_ARRAY, "caller must fix this");
  if (!stopped() && copy_type == T_OBJECT) {
    Node* src_klass  = load_object_klass(src);
    Node* dest_klass = load_object_klass(dest);
    Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
    if (not_subtype_ctrl != top()) {
      PreserveJVMState pjvms(this);
      set_control(not_subtype_ctrl);
      int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
      Node* p1 = basic_plus_adr(dest_klass, ek_offset);
      Node* n1 = LoadKlassNode::make(_gvn, NULL, immutable_memory(), p1, TypeRawPtr::BOTTOM);
      Node* dest_elem_klass = _gvn.transform(n1);
      Node* cv = generate_checkcast_arraycopy(adr_type,
                                              dest_elem_klass,
                                              src, src_offset, dest, dest_offset,
                                              ConvI2X(copy_length), dest_uninitialized);
      if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
      checked_control = control();
      checked_i_o     = i_o();
      checked_mem     = memory(adr_type);
      checked_value   = cv;
    }
    if (alloc != NULL && use_ReduceInitialCardMarks()) {
      copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
      assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
             "sizes agree");
    }
  }
  if (!stopped()) {
    PreserveJVMState pjvms(this);
    generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
                                 src, src_offset, dest, dest_offset,
                                 ConvI2X(copy_length), dest_uninitialized);
    result_region->init_req(fast_path, control());
    result_i_o   ->init_req(fast_path, i_o());
    result_memory->init_req(fast_path, memory(adr_type));
  }
  slow_control = top();
  if (slow_region != NULL)
    slow_control = _gvn.transform(slow_region);
  DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
  set_control(checked_control);
  if (!stopped()) {
    Node* cmp = _gvn.transform(new(C) CmpINode(checked_value, intcon(0)));
    Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::eq));
    IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
    Node* checks_done = _gvn.transform(new(C) IfTrueNode(iff));
    result_region->init_req(checked_path, checks_done);
    result_i_o   ->init_req(checked_path, checked_i_o);
    result_memory->init_req(checked_path, checked_mem);
    set_control( _gvn.transform(new(C) IfFalseNode(iff) ));
    RegionNode* slow_reg2 = new(C) RegionNode(3);
    PhiNode*    slow_i_o2 = new(C) PhiNode(slow_reg2, Type::ABIO);
    PhiNode*    slow_mem2 = new(C) PhiNode(slow_reg2, Type::MEMORY, adr_type);
    record_for_igvn(slow_reg2);
    slow_reg2  ->init_req(1, slow_control);
    slow_i_o2  ->init_req(1, slow_i_o);
    slow_mem2  ->init_req(1, slow_mem);
    slow_reg2  ->init_req(2, control());
    slow_i_o2  ->init_req(2, checked_i_o);
    slow_mem2  ->init_req(2, checked_mem);
    slow_control = _gvn.transform(slow_reg2);
    slow_i_o     = _gvn.transform(slow_i_o2);
    slow_mem     = _gvn.transform(slow_mem2);
    if (alloc != NULL) {
    } else {
      Node* checked_offset = _gvn.transform(new(C) XorINode(checked_value, intcon(-1)));
      Node* slow_offset    = new(C) PhiNode(slow_reg2, TypeInt::INT);
      slow_offset->init_req(1, intcon(0));
      slow_offset->init_req(2, checked_offset);
      slow_offset  = _gvn.transform(slow_offset);
      Node* src_off_plus  = _gvn.transform(new(C) AddINode(src_offset,  slow_offset));
      Node* dest_off_plus = _gvn.transform(new(C) AddINode(dest_offset, slow_offset));
      Node* length_minus  = _gvn.transform(new(C) SubINode(copy_length, slow_offset));
      src_offset  = src_off_plus;
      dest_offset = dest_off_plus;
      copy_length = length_minus;
    }
  }
  set_control(slow_control);
  if (!stopped()) {
    PreserveJVMState pjvms(this);   // replace_in_map may trash the map
    set_memory(slow_mem, adr_type);
    set_i_o(slow_i_o);
    if (dest_uninitialized) {
      generate_clear_array(adr_type, dest, basic_elem_type,
                           intcon(0), NULL,
                           alloc->in(AllocateNode::AllocSize));
    }
    generate_slow_arraycopy(adr_type,
                            src, src_offset, dest, dest_offset,
                            copy_length, /*dest_uninitialized*/false);
    result_region->init_req(slow_call_path, control());
    result_i_o   ->init_req(slow_call_path, i_o());
    result_memory->init_req(slow_call_path, memory(adr_type));
  }
  for (uint i = 1; i < result_region->req(); i++) {
    if (result_region->in(i) == NULL)
      result_region->init_req(i, top());
  }
  set_control( _gvn.transform(result_region));
  set_i_o(     _gvn.transform(result_i_o)    );
  set_memory(  _gvn.transform(result_memory), adr_type );
  if (alloc != NULL) {
    insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
  } else if (InsertMemBarAfterArraycopy)
    insert_mem_bar(Op_MemBarCPUOrder);
}
AllocateArrayNode*
LibraryCallKit::tightly_coupled_allocation(Node* ptr,
                                           RegionNode* slow_region) {
  if (stopped())             return NULL;  // no fast path
  if (C->AliasLevel() == 0)  return NULL;  // no MergeMems around
  AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn);
  if (alloc == NULL)  return NULL;
  Node* rawmem = memory(Compile::AliasIdxRaw);
  if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) {
    return NULL;
  }
  rawmem = rawmem->in(0)->as_Initialize()->memory(Compile::AliasIdxRaw);
  if (!(rawmem->is_Proj() && rawmem->in(0) == alloc)) {
    return NULL;
  }
  for (DUIterator_Fast imax, i = ptr->fast_outs(imax); i < imax; i++) {
    Node* obs = ptr->fast_out(i);
    if (obs != this->map()) {
      return NULL;
    }
  }
  Node* alloc_ctl = ptr->in(0);
  assert(just_allocated_object(alloc_ctl) == ptr, "most recent allo");
  Node* ctl = control();
  while (ctl != alloc_ctl) {
    if ((ctl->is_IfFalse() || ctl->is_IfTrue()) && ctl->in(0)->is_If()) {
      IfNode* iff = ctl->in(0)->as_If();
      Node* not_ctl = iff->proj_out(1 - ctl->as_Proj()->_con);
      assert(not_ctl != NULL && not_ctl != ctl, "found alternate");
      if (slow_region != NULL && slow_region->find_edge(not_ctl) >= 1) {
        ctl = iff->in(0);       // This test feeds the known slow_region.
        continue;
      }
      bool found_trap = false;
      for (DUIterator_Fast jmax, j = not_ctl->fast_outs(jmax); j < jmax; j++) {
        Node* obs = not_ctl->fast_out(j);
        if (obs->in(0) == not_ctl && obs->is_Call() &&
            (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) {
          found_trap = true; break;
        }
      }
      if (found_trap) {
        ctl = iff->in(0);       // This test feeds a harmless uncommon trap.
        continue;
      }
    }
    return NULL;
  }
  return alloc;
}
void
LibraryCallKit::generate_clear_array(const TypePtr* adr_type,
                                     Node* dest,
                                     BasicType basic_elem_type,
                                     Node* slice_idx,
                                     Node* slice_len,
                                     Node* dest_size) {
  assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
  if (slice_len == NULL)  slice_len = top();
  if (dest_size == NULL)  dest_size = top();
  Node* mem = memory(adr_type); // memory slice to operate on
  int scale = exact_log2(type2aelembytes(basic_elem_type));
  int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
  int clear_low = (-1 << scale) & (BytesPerInt  - 1);
  int bump_bit  = (-1 << scale) & BytesPerInt;
  const intptr_t BIG_NEG = -128;
  assert(BIG_NEG + 2*abase < 0, "neg enough");
  intptr_t slice_idx_con = (intptr_t) find_int_con(slice_idx, BIG_NEG);
  intptr_t slice_len_con = (intptr_t) find_int_con(slice_len, BIG_NEG);
  if (slice_len_con == 0) {
    return;                     // nothing to do here
  }
  intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
  intptr_t end_con   = find_intptr_t_con(dest_size, -1);
  if (slice_idx_con >= 0 && slice_len_con >= 0) {
    assert(end_con < 0, "not two cons");
    end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
                       BytesPerLong);
  }
  if (start_con >= 0 && end_con >= 0) {
    mem = ClearArrayNode::clear_memory(control(), mem, dest,
                                       start_con, end_con, &_gvn);
  } else if (start_con >= 0 && dest_size != top()) {
    Node* end = dest_size;
    mem = ClearArrayNode::clear_memory(control(), mem, dest,
                                       start_con, end, &_gvn);
  } else if (start_con >= 0 && slice_len != top()) {
    intptr_t end_base  = abase + (slice_idx_con << scale);
    int      end_round = (-1 << scale) & (BytesPerLong  - 1);
    Node*    end       = ConvI2X(slice_len);
    if (scale != 0)
      end = _gvn.transform(new(C) LShiftXNode(end, intcon(scale) ));
    end_base += end_round;
    end = _gvn.transform(new(C) AddXNode(end, MakeConX(end_base)));
    end = _gvn.transform(new(C) AndXNode(end, MakeConX(~end_round)));
    mem = ClearArrayNode::clear_memory(control(), mem, dest,
                                       start_con, end, &_gvn);
  } else if (start_con < 0 && dest_size != top()) {
    Node* start = slice_idx;
    start = ConvI2X(start);
    if (scale != 0)
      start = _gvn.transform(new(C) LShiftXNode( start, intcon(scale) ));
    start = _gvn.transform(new(C) AddXNode(start, MakeConX(abase)));
    if ((bump_bit | clear_low) != 0) {
      int to_clear = (bump_bit | clear_low);
      if (((abase + bump_bit) & ~to_clear) - bump_bit
          < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
        bump_bit = 0;
        assert((abase & to_clear) == 0, "array base must be long-aligned");
      } else {
        start = _gvn.transform(new(C) AddXNode(start, MakeConX(bump_bit)));
        assert((abase & clear_low) == 0, "array base must be int-aligned");
      }
      start = _gvn.transform(new(C) AndXNode(start, MakeConX(~to_clear)));
      if (bump_bit != 0) {
        Node* x1 = _gvn.transform(new(C) AddXNode(start, MakeConX(-bump_bit)));
        Node* p1 = basic_plus_adr(dest, x1);
        mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT, MemNode::unordered);
        mem = _gvn.transform(mem);
      }
    }
    Node* end = dest_size; // pre-rounded
    mem = ClearArrayNode::clear_memory(control(), mem, dest,
                                       start, end, &_gvn);
  } else {
    ShouldNotReachHere();       // fix caller
  }
  set_memory(mem, adr_type);
}
bool
LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
                                         BasicType basic_elem_type,
                                         AllocateNode* alloc,
                                         Node* src,  Node* src_offset,
                                         Node* dest, Node* dest_offset,
                                         Node* dest_size, bool dest_uninitialized) {
  int scale = exact_log2(type2aelembytes(basic_elem_type));
  if (scale >= LogBytesPerLong)
    return false;               // it is already a block transfer
  int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
  intptr_t src_off_con  = (intptr_t) find_int_con(src_offset, -1);
  intptr_t dest_off_con = (intptr_t) find_int_con(dest_offset, -1);
  if (src_off_con < 0 || dest_off_con < 0)
    return false;
  intptr_t src_off  = abase + (src_off_con  << scale);
  intptr_t dest_off = abase + (dest_off_con << scale);
  if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
    if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
        ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
      Node* sptr = basic_plus_adr(src,  src_off);
      Node* dptr = basic_plus_adr(dest, dest_off);
      const TypePtr* s_adr_type = _gvn.type(sptr)->is_ptr();
      assert(s_adr_type->isa_aryptr(), "impossible slice");
      Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, s_adr_type, MemNode::unordered);
      store_to_memory(control(), dptr, sval, T_INT, adr_type, MemNode::unordered);
      src_off += BytesPerInt;
      dest_off += BytesPerInt;
    } else {
      return false;
    }
  }
  assert(src_off % BytesPerLong == 0, "");
  assert(dest_off % BytesPerLong == 0, "");
  Node* sptr  = basic_plus_adr(src,  src_off);
  Node* dptr  = basic_plus_adr(dest, dest_off);
  Node* countx = dest_size;
  countx = _gvn.transform(new (C) SubXNode(countx, MakeConX(dest_off)));
  countx = _gvn.transform(new (C) URShiftXNode(countx, intcon(LogBytesPerLong)));
  bool disjoint_bases = true;   // since alloc != NULL
  generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
                               sptr, NULL, dptr, NULL, countx, dest_uninitialized);
  return true;
}
void
LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
                                        Node* src,  Node* src_offset,
                                        Node* dest, Node* dest_offset,
                                        Node* copy_length, bool dest_uninitialized) {
  assert(!dest_uninitialized, "Invariant");
  Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
                                 OptoRuntime::slow_arraycopy_Type(),
                                 OptoRuntime::slow_arraycopy_Java(),
                                 "slow_arraycopy", adr_type,
                                 src, src_offset, dest, dest_offset,
                                 copy_length);
  make_slow_call_ex(call, env()->Throwable_klass(), false);
}
Node*
LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
                                             Node* dest_elem_klass,
                                             Node* src,  Node* src_offset,
                                             Node* dest, Node* dest_offset,
                                             Node* copy_length, bool dest_uninitialized) {
  if (stopped())  return NULL;
  address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
  if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
    return NULL;
  }
  int sco_offset = in_bytes(Klass::super_check_offset_offset());
  Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
  Node* n3 = new(C) LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered);
  Node* check_offset = ConvI2X(_gvn.transform(n3));
  Node* check_value  = dest_elem_klass;
  Node* src_start  = array_element_address(src,  src_offset,  T_OBJECT);
  Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
  Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
                                 OptoRuntime::checkcast_arraycopy_Type(),
                                 copyfunc_addr, "checkcast_arraycopy", adr_type,
                                 src_start, dest_start,
                                 copy_length XTOP,
                                 check_offset XTOP,
                                 check_value);
  return _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
}
Node*
LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
                                           Node* src,  Node* src_offset,
                                           Node* dest, Node* dest_offset,
                                           Node* copy_length, bool dest_uninitialized) {
  assert(!dest_uninitialized, "Invariant");
  if (stopped())  return NULL;
  address copyfunc_addr = StubRoutines::generic_arraycopy();
  if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
    return NULL;
  }
  Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
                    OptoRuntime::generic_arraycopy_Type(),
                    copyfunc_addr, "generic_arraycopy", adr_type,
                    src, src_offset, dest, dest_offset, copy_length);
  return _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
}
void
LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
                                             BasicType basic_elem_type,
                                             bool disjoint_bases,
                                             Node* src,  Node* src_offset,
                                             Node* dest, Node* dest_offset,
                                             Node* copy_length, bool dest_uninitialized) {
  if (stopped())  return;               // nothing to do
  Node* src_start  = src;
  Node* dest_start = dest;
  if (src_offset != NULL || dest_offset != NULL) {
    assert(src_offset != NULL && dest_offset != NULL, "");
    src_start  = array_element_address(src,  src_offset,  basic_elem_type);
    dest_start = array_element_address(dest, dest_offset, basic_elem_type);
  }
  const char* copyfunc_name = "arraycopy";
  address     copyfunc_addr =
      basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
                          disjoint_bases, copyfunc_name, dest_uninitialized);
  make_runtime_call(RC_LEAF|RC_NO_FP,
                    OptoRuntime::fast_arraycopy_Type(),
                    copyfunc_addr, copyfunc_name, adr_type,
                    src_start, dest_start, copy_length XTOP);
}
bool LibraryCallKit::inline_encodeISOArray() {
  assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters");
  Node *src         = argument(0);
  Node *src_offset  = argument(1);
  Node *dst         = argument(2);
  Node *dst_offset  = argument(3);
  Node *length      = argument(4);
  const Type* src_type = src->Value(&_gvn);
  const Type* dst_type = dst->Value(&_gvn);
  const TypeAryPtr* top_src = src_type->isa_aryptr();
  const TypeAryPtr* top_dest = dst_type->isa_aryptr();
  if (top_src  == NULL || top_src->klass()  == NULL ||
      top_dest == NULL || top_dest->klass() == NULL) {
    return false;
  }
  BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  if (src_elem != T_CHAR || dst_elem != T_BYTE) {
    return false;
  }
  Node* src_start = array_element_address(src, src_offset, src_elem);
  Node* dst_start = array_element_address(dst, dst_offset, dst_elem);
  const TypeAryPtr* mtype = TypeAryPtr::BYTES;
  Node* enc = new (C) EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length);
  enc = _gvn.transform(enc);
  Node* res_mem = _gvn.transform(new (C) SCMemProjNode(enc));
  set_memory(res_mem, mtype);
  set_result(enc);
  return true;
}
bool LibraryCallKit::inline_multiplyToLen() {
  assert(UseMultiplyToLenIntrinsic, "not implementated on this platform");
  address stubAddr = StubRoutines::multiplyToLen();
  if (stubAddr == NULL) {
    return false; // Intrinsic's stub is not implemented on this platform
  }
  const char* stubName = "multiplyToLen";
  assert(callee()->signature()->size() == 5, "multiplyToLen has 5 parameters");
  Node* x    = argument(0);
  Node* xlen = argument(1);
  Node* y    = argument(2);
  Node* ylen = argument(3);
  Node* z    = argument(4);
  const Type* x_type = x->Value(&_gvn);
  const Type* y_type = y->Value(&_gvn);
  const TypeAryPtr* top_x = x_type->isa_aryptr();
  const TypeAryPtr* top_y = y_type->isa_aryptr();
  if (top_x  == NULL || top_x->klass()  == NULL ||
      top_y == NULL || top_y->klass() == NULL) {
    return false;
  }
  BasicType x_elem = x_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  BasicType y_elem = y_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  if (x_elem != T_INT || y_elem != T_INT) {
    return false;
  }
  { PreserveReexecuteState preexecs(this);
    jvms()->set_should_reexecute(true);
    Node* x_start = array_element_address(x, intcon(0), x_elem);
    Node* y_start = array_element_address(y, intcon(0), y_elem);
    Node* zlen = _gvn.transform(new(C) AddINode(xlen, ylen));
    ciKlass* klass = ciTypeArrayKlass::make(T_INT);
    Node* klass_node = makecon(TypeKlassPtr::make(klass));
    IdealKit ideal(this);
#define __ ideal.
     Node* one = __ ConI(1);
     Node* zero = __ ConI(0);
     IdealVariable need_alloc(ideal), z_alloc(ideal);  __ declarations_done();
     __ set(need_alloc, zero);
     __ set(z_alloc, z);
     __ if_then(z, BoolTest::eq, null()); {
       __ increment (need_alloc, one);
     } __ else_(); {
       sync_kit(ideal);
       Node* zlen_arg = load_array_length(z);
       __ sync_kit(this);
       __ if_then(zlen_arg, BoolTest::lt, zlen); {
         __ increment (need_alloc, one);
       } __ end_if();
     } __ end_if();
     __ if_then(__ value(need_alloc), BoolTest::ne, zero); {
       sync_kit(ideal);
       Node * narr = new_array(klass_node, zlen, 1);
       __ sync_kit(this);
       __ set(z_alloc, narr);
     } __ end_if();
     sync_kit(ideal);
     z = __ value(z_alloc);
     _gvn.set_type(z, TypeOopPtr::make_from_klass(klass));
     final_sync(ideal);
#undef __
    Node* z_start = array_element_address(z, intcon(0), T_INT);
    Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
                                   OptoRuntime::multiplyToLen_Type(),
                                   stubAddr, stubName, TypePtr::BOTTOM,
                                   x_start, xlen, y_start, ylen, z_start, zlen);
  } // original reexecute is set back here
  C->set_has_split_ifs(true); // Has chance for split-if optimization
  set_result(z);
  return true;
}
bool LibraryCallKit::inline_squareToLen() {
  assert(UseSquareToLenIntrinsic, "not implementated on this platform");
  address stubAddr = StubRoutines::squareToLen();
  if (stubAddr == NULL) {
    return false; // Intrinsic's stub is not implemented on this platform
  }
  const char* stubName = "squareToLen";
  assert(callee()->signature()->size() == 4, "implSquareToLen has 4 parameters");
  Node* x    = argument(0);
  Node* len  = argument(1);
  Node* z    = argument(2);
  Node* zlen = argument(3);
  const Type* x_type = x->Value(&_gvn);
  const Type* z_type = z->Value(&_gvn);
  const TypeAryPtr* top_x = x_type->isa_aryptr();
  const TypeAryPtr* top_z = z_type->isa_aryptr();
  if (top_x  == NULL || top_x->klass()  == NULL ||
      top_z  == NULL || top_z->klass()  == NULL) {
    return false;
  }
  BasicType x_elem = x_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  BasicType z_elem = z_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  if (x_elem != T_INT || z_elem != T_INT) {
    return false;
  }
  Node* x_start = array_element_address(x, intcon(0), x_elem);
  Node* z_start = array_element_address(z, intcon(0), z_elem);
  Node*  call = make_runtime_call(RC_LEAF|RC_NO_FP,
                                  OptoRuntime::squareToLen_Type(),
                                  stubAddr, stubName, TypePtr::BOTTOM,
                                  x_start, len, z_start, zlen);
  set_result(z);
  return true;
}
bool LibraryCallKit::inline_mulAdd() {
  assert(UseMulAddIntrinsic, "not implementated on this platform");
  address stubAddr = StubRoutines::mulAdd();
  if (stubAddr == NULL) {
    return false; // Intrinsic's stub is not implemented on this platform
  }
  const char* stubName = "mulAdd";
  assert(callee()->signature()->size() == 5, "mulAdd has 5 parameters");
  Node* out      = argument(0);
  Node* in       = argument(1);
  Node* offset   = argument(2);
  Node* len      = argument(3);
  Node* k        = argument(4);
  const Type* out_type = out->Value(&_gvn);
  const Type* in_type = in->Value(&_gvn);
  const TypeAryPtr* top_out = out_type->isa_aryptr();
  const TypeAryPtr* top_in = in_type->isa_aryptr();
  if (top_out  == NULL || top_out->klass()  == NULL ||
      top_in == NULL || top_in->klass() == NULL) {
    return false;
  }
  BasicType out_elem = out_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  BasicType in_elem = in_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  if (out_elem != T_INT || in_elem != T_INT) {
    return false;
  }
  Node* outlen = load_array_length(out);
  Node* new_offset = _gvn.transform(new (C) SubINode(outlen, offset));
  Node* out_start = array_element_address(out, intcon(0), out_elem);
  Node* in_start = array_element_address(in, intcon(0), in_elem);
  Node*  call = make_runtime_call(RC_LEAF|RC_NO_FP,
                                  OptoRuntime::mulAdd_Type(),
                                  stubAddr, stubName, TypePtr::BOTTOM,
                                  out_start,in_start, new_offset, len, k);
  Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
  set_result(result);
  return true;
}
bool LibraryCallKit::inline_montgomeryMultiply() {
  address stubAddr = StubRoutines::montgomeryMultiply();
  if (stubAddr == NULL) {
    return false; // Intrinsic's stub is not implemented on this platform
  }
  assert(UseMontgomeryMultiplyIntrinsic, "not implemented on this platform");
  const char* stubName = "montgomery_multiply";
  assert(callee()->signature()->size() == 7, "montgomeryMultiply has 7 parameters");
  Node* a    = argument(0);
  Node* b    = argument(1);
  Node* n    = argument(2);
  Node* len  = argument(3);
  Node* inv  = argument(4);
  Node* m    = argument(6);
  const Type* a_type = a->Value(&_gvn);
  const TypeAryPtr* top_a = a_type->isa_aryptr();
  const Type* b_type = b->Value(&_gvn);
  const TypeAryPtr* top_b = b_type->isa_aryptr();
  const Type* n_type = a->Value(&_gvn);
  const TypeAryPtr* top_n = n_type->isa_aryptr();
  const Type* m_type = a->Value(&_gvn);
  const TypeAryPtr* top_m = m_type->isa_aryptr();
  if (top_a  == NULL || top_a->klass()  == NULL ||
      top_b == NULL || top_b->klass()  == NULL ||
      top_n == NULL || top_n->klass()  == NULL ||
      top_m == NULL || top_m->klass()  == NULL) {
    return false;
  }
  BasicType a_elem = a_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  BasicType b_elem = b_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  BasicType n_elem = n_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  BasicType m_elem = m_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  if (a_elem != T_INT || b_elem != T_INT || n_elem != T_INT || m_elem != T_INT) {
    return false;
  }
  {
    Node* a_start = array_element_address(a, intcon(0), a_elem);
    Node* b_start = array_element_address(b, intcon(0), b_elem);
    Node* n_start = array_element_address(n, intcon(0), n_elem);
    Node* m_start = array_element_address(m, intcon(0), m_elem);
    Node* call = NULL;
    if (CCallingConventionRequiresIntsAsLongs) {
      Node* len_I2L = ConvI2L(len);
      call = make_runtime_call(RC_LEAF,
                               OptoRuntime::montgomeryMultiply_Type(),
                               stubAddr, stubName, TypePtr::BOTTOM,
                               a_start, b_start, n_start, len_I2L XTOP, inv,
                               top(), m_start);
    } else {
      call = make_runtime_call(RC_LEAF,
                               OptoRuntime::montgomeryMultiply_Type(),
                               stubAddr, stubName, TypePtr::BOTTOM,
                               a_start, b_start, n_start, len, inv, top(),
                               m_start);
    }
    set_result(m);
  }
  return true;
}
bool LibraryCallKit::inline_montgomerySquare() {
  address stubAddr = StubRoutines::montgomerySquare();
  if (stubAddr == NULL) {
    return false; // Intrinsic's stub is not implemented on this platform
  }
  assert(UseMontgomerySquareIntrinsic, "not implemented on this platform");
  const char* stubName = "montgomery_square";
  assert(callee()->signature()->size() == 6, "montgomerySquare has 6 parameters");
  Node* a    = argument(0);
  Node* n    = argument(1);
  Node* len  = argument(2);
  Node* inv  = argument(3);
  Node* m    = argument(5);
  const Type* a_type = a->Value(&_gvn);
  const TypeAryPtr* top_a = a_type->isa_aryptr();
  const Type* n_type = a->Value(&_gvn);
  const TypeAryPtr* top_n = n_type->isa_aryptr();
  const Type* m_type = a->Value(&_gvn);
  const TypeAryPtr* top_m = m_type->isa_aryptr();
  if (top_a  == NULL || top_a->klass()  == NULL ||
      top_n == NULL || top_n->klass()  == NULL ||
      top_m == NULL || top_m->klass()  == NULL) {
    return false;
  }
  BasicType a_elem = a_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  BasicType n_elem = n_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  BasicType m_elem = m_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  if (a_elem != T_INT || n_elem != T_INT || m_elem != T_INT) {
    return false;
  }
  {
    Node* a_start = array_element_address(a, intcon(0), a_elem);
    Node* n_start = array_element_address(n, intcon(0), n_elem);
    Node* m_start = array_element_address(m, intcon(0), m_elem);
    Node* call = NULL;
    if (CCallingConventionRequiresIntsAsLongs) {
      Node* len_I2L = ConvI2L(len);
      call = make_runtime_call(RC_LEAF,
                               OptoRuntime::montgomerySquare_Type(),
                               stubAddr, stubName, TypePtr::BOTTOM,
                               a_start, n_start, len_I2L XTOP, inv, top(),
                               m_start);
    } else {
      call = make_runtime_call(RC_LEAF,
                               OptoRuntime::montgomerySquare_Type(),
                               stubAddr, stubName, TypePtr::BOTTOM,
                               a_start, n_start, len, inv, top(),
                               m_start);
    }
    set_result(m);
  }
  return true;
}
bool LibraryCallKit::inline_updateCRC32() {
  assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
  assert(callee()->signature()->size() == 2, "update has 2 parameters");
  Node* crc  = argument(0); // type: int
  Node* b    = argument(1); // type: int
  Node* M1 = intcon(-1);
  crc = _gvn.transform(new (C) XorINode(crc, M1));
  Node* result = _gvn.transform(new (C) XorINode(crc, b));
  result = _gvn.transform(new (C) AndINode(result, intcon(0xFF)));
  Node* base = makecon(TypeRawPtr::make(StubRoutines::crc_table_addr()));
  Node* offset = _gvn.transform(new (C) LShiftINode(result, intcon(0x2)));
  Node* adr = basic_plus_adr(top(), base, ConvI2X(offset));
  result = make_load(control(), adr, TypeInt::INT, T_INT, MemNode::unordered);
  crc = _gvn.transform(new (C) URShiftINode(crc, intcon(8)));
  result = _gvn.transform(new (C) XorINode(crc, result));
  result = _gvn.transform(new (C) XorINode(result, M1));
  set_result(result);
  return true;
}
bool LibraryCallKit::inline_updateBytesCRC32() {
  assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
  assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters");
  Node* crc     = argument(0); // type: int
  Node* src     = argument(1); // type: oop
  Node* offset  = argument(2); // type: int
  Node* length  = argument(3); // type: int
  const Type* src_type = src->Value(&_gvn);
  const TypeAryPtr* top_src = src_type->isa_aryptr();
  if (top_src  == NULL || top_src->klass()  == NULL) {
    return false;
  }
  BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  if (src_elem != T_BYTE) {
    return false;
  }
  Node* src_start = array_element_address(src, offset, src_elem);
  address stubAddr = StubRoutines::updateBytesCRC32();
  const char *stubName = "updateBytesCRC32";
  Node* call;
  if (CCallingConventionRequiresIntsAsLongs) {
   call =  make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
                             stubAddr, stubName, TypePtr::BOTTOM,
                             crc XTOP, src_start, length XTOP);
  } else {
    call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
                             stubAddr, stubName, TypePtr::BOTTOM,
                             crc, src_start, length);
  }
  Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
  set_result(result);
  return true;
}
bool LibraryCallKit::inline_updateByteBufferCRC32() {
  assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
  assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long");
  Node* crc     = argument(0); // type: int
  Node* src     = argument(1); // type: long
  Node* offset  = argument(3); // type: int
  Node* length  = argument(4); // type: int
  src = ConvL2X(src);  // adjust Java long to machine word
  Node* base = _gvn.transform(new (C) CastX2PNode(src));
  offset = ConvI2X(offset);
  Node* src_start = basic_plus_adr(top(), base, offset);
  address stubAddr = StubRoutines::updateBytesCRC32();
  const char *stubName = "updateBytesCRC32";
  Node* call;
  if (CCallingConventionRequiresIntsAsLongs) {
    call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
                      stubAddr, stubName, TypePtr::BOTTOM,
                      crc XTOP, src_start, length XTOP);
  } else {
    call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
                             stubAddr, stubName, TypePtr::BOTTOM,
                             crc, src_start, length);
  }
  Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
  set_result(result);
  return true;
}
bool LibraryCallKit::inline_reference_get() {
  const int referent_offset = java_lang_ref_Reference::referent_offset;
  guarantee(referent_offset > 0, "should have already been set");
  Node* reference_obj = null_check_receiver();
  if (stopped()) return true;
  Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
  ciInstanceKlass* klass = env()->Object_klass();
  const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
  Node* no_ctrl = NULL;
  Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT, MemNode::unordered);
  pre_barrier(false /* do_load */,
              control(),
              NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
              result /* pre_val */,
              T_OBJECT);
  insert_mem_bar(Op_MemBarCPUOrder);
  set_result(result);
  return true;
}
Node * LibraryCallKit::load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
                                              bool is_exact=true, bool is_static=false) {
  const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
  assert(tinst != NULL, "obj is null");
  assert(tinst->klass()->is_loaded(), "obj is not loaded");
  assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
  ciField* field = tinst->klass()->as_instance_klass()->get_field_by_name(ciSymbol::make(fieldName),
                                                                          ciSymbol::make(fieldTypeString),
                                                                          is_static);
  if (field == NULL) return (Node *) NULL;
  assert (field != NULL, "undefined field");
  int offset  = field->offset_in_bytes();
  bool is_vol = field->is_volatile();
  ciType* field_klass = field->type();
  assert(field_klass->is_loaded(), "should be loaded");
  const TypePtr* adr_type = C->alias_type(field)->adr_type();
  Node *adr = basic_plus_adr(fromObj, fromObj, offset);
  BasicType bt = field->layout_type();
  const Type *type;
  if (bt == T_OBJECT) {
    type = TypeOopPtr::make_from_klass(field_klass->as_klass());
  } else {
    type = Type::get_const_basic_type(bt);
  }
  Node* leading_membar = NULL;
  if (support_IRIW_for_not_multiple_copy_atomic_cpu && is_vol) {
    leading_membar = insert_mem_bar(Op_MemBarVolatile);   // StoreLoad barrier
  }
  MemNode::MemOrd mo = is_vol ? MemNode::acquire : MemNode::unordered;
  Node* loadedField = make_load(NULL, adr, type, bt, adr_type, mo, LoadNode::DependsOnlyOnTest, is_vol);
  if (is_vol) {
    Node* mb = insert_mem_bar(Op_MemBarAcquire, loadedField);
    mb->as_MemBar()->set_trailing_load();
  }
  return loadedField;
}
bool LibraryCallKit::inline_aescrypt_Block(vmIntrinsics::ID id) {
  address stubAddr = NULL;
  const char *stubName;
  assert(UseAES, "need AES instruction support");
  switch(id) {
  case vmIntrinsics::_aescrypt_encryptBlock:
    stubAddr = StubRoutines::aescrypt_encryptBlock();
    stubName = "aescrypt_encryptBlock";
    break;
  case vmIntrinsics::_aescrypt_decryptBlock:
    stubAddr = StubRoutines::aescrypt_decryptBlock();
    stubName = "aescrypt_decryptBlock";
    break;
  }
  if (stubAddr == NULL) return false;
  Node* aescrypt_object = argument(0);
  Node* src             = argument(1);
  Node* src_offset      = argument(2);
  Node* dest            = argument(3);
  Node* dest_offset     = argument(4);
  const Type* src_type = src->Value(&_gvn);
  const Type* dest_type = dest->Value(&_gvn);
  const TypeAryPtr* top_src = src_type->isa_aryptr();
  const TypeAryPtr* top_dest = dest_type->isa_aryptr();
  assert (top_src  != NULL && top_src->klass()  != NULL &&  top_dest != NULL && top_dest->klass() != NULL, "args are strange");
  Node* src_start  = src;
  Node* dest_start = dest;
  if (src_offset != NULL || dest_offset != NULL) {
    assert(src_offset != NULL && dest_offset != NULL, "");
    src_start  = array_element_address(src,  src_offset,  T_BYTE);
    dest_start = array_element_address(dest, dest_offset, T_BYTE);
  }
  Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object);
  if (k_start == NULL) return false;
  if (Matcher::pass_original_key_for_aes()) {
    Node* original_k_start = get_original_key_start_from_aescrypt_object(aescrypt_object);
    if (original_k_start == NULL) return false;
    make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::aescrypt_block_Type(),
                      stubAddr, stubName, TypePtr::BOTTOM,
                      src_start, dest_start, k_start, original_k_start);
  } else {
    make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::aescrypt_block_Type(),
                      stubAddr, stubName, TypePtr::BOTTOM,
                      src_start, dest_start, k_start);
  }
  return true;
}
bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) {
  address stubAddr = NULL;
  const char *stubName = NULL;
  assert(UseAES, "need AES instruction support");
  switch(id) {
  case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
    stubAddr = StubRoutines::cipherBlockChaining_encryptAESCrypt();
    stubName = "cipherBlockChaining_encryptAESCrypt";
    break;
  case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
    stubAddr = StubRoutines::cipherBlockChaining_decryptAESCrypt();
    stubName = "cipherBlockChaining_decryptAESCrypt";
    break;
  }
  if (stubAddr == NULL) return false;
  Node* cipherBlockChaining_object = argument(0);
  Node* src                        = argument(1);
  Node* src_offset                 = argument(2);
  Node* len                        = argument(3);
  Node* dest                       = argument(4);
  Node* dest_offset                = argument(5);
  const Type* src_type = src->Value(&_gvn);
  const Type* dest_type = dest->Value(&_gvn);
  const TypeAryPtr* top_src = src_type->isa_aryptr();
  const TypeAryPtr* top_dest = dest_type->isa_aryptr();
  assert (top_src  != NULL && top_src->klass()  != NULL
          &&  top_dest != NULL && top_dest->klass() != NULL, "args are strange");
  Node* src_start  = src;
  Node* dest_start = dest;
  if (src_offset != NULL || dest_offset != NULL) {
    assert(src_offset != NULL && dest_offset != NULL, "");
    src_start  = array_element_address(src,  src_offset,  T_BYTE);
    dest_start = array_element_address(dest, dest_offset, T_BYTE);
  }
  Node* embeddedCipherObj = load_field_from_object(cipherBlockChaining_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
  if (embeddedCipherObj == NULL) return false;
  const TypeInstPtr* tinst = _gvn.type(cipherBlockChaining_object)->isa_instptr();
  assert(tinst != NULL, "CBC obj is null");
  assert(tinst->klass()->is_loaded(), "CBC obj is not loaded");
  ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
  assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded");
  ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
  const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt);
  const TypeOopPtr* xtype = aklass->as_instance_type();
  Node* aescrypt_object = new(C) CheckCastPPNode(control(), embeddedCipherObj, xtype);
  aescrypt_object = _gvn.transform(aescrypt_object);
  Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object);
  if (k_start == NULL) return false;
  Node* objRvec = load_field_from_object(cipherBlockChaining_object, "r", "[B", /*is_exact*/ false);
  if (objRvec == NULL) return false;
  Node* r_start = array_element_address(objRvec, intcon(0), T_BYTE);
  Node* cbcCrypt;
  if (Matcher::pass_original_key_for_aes()) {
    Node* original_k_start = get_original_key_start_from_aescrypt_object(aescrypt_object);
    if (original_k_start == NULL) return false;
    cbcCrypt = make_runtime_call(RC_LEAF|RC_NO_FP,
                                 OptoRuntime::cipherBlockChaining_aescrypt_Type(),
                                 stubAddr, stubName, TypePtr::BOTTOM,
                                 src_start, dest_start, k_start, r_start, len, original_k_start);
  } else {
    cbcCrypt = make_runtime_call(RC_LEAF|RC_NO_FP,
                                 OptoRuntime::cipherBlockChaining_aescrypt_Type(),
                                 stubAddr, stubName, TypePtr::BOTTOM,
                                 src_start, dest_start, k_start, r_start, len);
  }
  Node* retvalue = _gvn.transform(new (C) ProjNode(cbcCrypt, TypeFunc::Parms));
  set_result(retvalue);
  return true;
}
Node * LibraryCallKit::get_key_start_from_aescrypt_object(Node *aescrypt_object) {
#ifdef PPC64
  Node* objSessionK = load_field_from_object(aescrypt_object, "sessionK", "[[I", /*is_exact*/ false);
  assert (objSessionK != NULL, "wrong version of com.sun.crypto.provider.AESCrypt");
  if (objSessionK == NULL) {
    return (Node *) NULL;
  }
  Node* objAESCryptKey = load_array_element(control(), objSessionK, intcon(0), TypeAryPtr::OOPS);
#else
  Node* objAESCryptKey = load_field_from_object(aescrypt_object, "K", "[I", /*is_exact*/ false);
#endif // PPC64
  assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt");
  if (objAESCryptKey == NULL) return (Node *) NULL;
  Node* k_start = array_element_address(objAESCryptKey, intcon(0), T_INT);
  return k_start;
}
Node * LibraryCallKit::get_original_key_start_from_aescrypt_object(Node *aescrypt_object) {
  Node* objAESCryptKey = load_field_from_object(aescrypt_object, "lastKey", "[B", /*is_exact*/ false);
  assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt");
  if (objAESCryptKey == NULL) return (Node *) NULL;
  Node* original_k_start = array_element_address(objAESCryptKey, intcon(0), T_BYTE);
  return original_k_start;
}
Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting) {
  Node* objCBC = argument(0);
  Node* embeddedCipherObj = load_field_from_object(objCBC, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
  const TypeInstPtr* tinst = _gvn.type(objCBC)->isa_instptr();
  assert(tinst != NULL, "CBCobj is null");
  assert(tinst->klass()->is_loaded(), "CBCobj is not loaded");
  ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
  if (!klass_AESCrypt->is_loaded()) {
    Node* ctrl = control();
    set_control(top()); // no regular fast path
    return ctrl;
  }
  ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
  Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt)));
  Node* cmp_instof  = _gvn.transform(new (C) CmpINode(instof, intcon(1)));
  Node* bool_instof  = _gvn.transform(new (C) BoolNode(cmp_instof, BoolTest::ne));
  Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN);
  if (!decrypting)
    return instof_false;  // even if it is NULL
  RegionNode* region = new(C) RegionNode(3);
  region->init_req(1, instof_false);
  Node* src = argument(1);
  Node* dest = argument(4);
  Node* cmp_src_dest = _gvn.transform(new (C) CmpPNode(src, dest));
  Node* bool_src_dest = _gvn.transform(new (C) BoolNode(cmp_src_dest, BoolTest::eq));
  Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN);
  region->init_req(2, src_dest_conjoint);
  record_for_igvn(region);
  return _gvn.transform(region);
}
bool LibraryCallKit::inline_ghash_processBlocks() {
  address stubAddr;
  const char *stubName;
  assert(UseGHASHIntrinsics, "need GHASH intrinsics support");
  stubAddr = StubRoutines::ghash_processBlocks();
  stubName = "ghash_processBlocks";
  Node* data           = argument(0);
  Node* offset         = argument(1);
  Node* len            = argument(2);
  Node* state          = argument(3);
  Node* subkeyH        = argument(4);
  Node* state_start  = array_element_address(state, intcon(0), T_LONG);
  assert(state_start, "state is NULL");
  Node* subkeyH_start  = array_element_address(subkeyH, intcon(0), T_LONG);
  assert(subkeyH_start, "subkeyH is NULL");
  Node* data_start  = array_element_address(data, offset, T_BYTE);
  assert(data_start, "data is NULL");
  Node* ghash = make_runtime_call(RC_LEAF|RC_NO_FP,
                                  OptoRuntime::ghash_processBlocks_Type(),
                                  stubAddr, stubName, TypePtr::BOTTOM,
                                  state_start, subkeyH_start, data_start, len);
  return true;
}
bool LibraryCallKit::inline_sha_implCompress(vmIntrinsics::ID id) {
  assert(callee()->signature()->size() == 2, "sha_implCompress has 2 parameters");
  Node* sha_obj = argument(0);
  Node* src     = argument(1); // type oop
  Node* ofs     = argument(2); // type int
  const Type* src_type = src->Value(&_gvn);
  const TypeAryPtr* top_src = src_type->isa_aryptr();
  if (top_src  == NULL || top_src->klass()  == NULL) {
    return false;
  }
  BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  if (src_elem != T_BYTE) {
    return false;
  }
  Node* src_start = array_element_address(src, ofs, src_elem);
  Node* state = NULL;
  address stubAddr;
  const char *stubName;
  switch(id) {
  case vmIntrinsics::_sha_implCompress:
    assert(UseSHA1Intrinsics, "need SHA1 instruction support");
    state = get_state_from_sha_object(sha_obj);
    stubAddr = StubRoutines::sha1_implCompress();
    stubName = "sha1_implCompress";
    break;
  case vmIntrinsics::_sha2_implCompress:
    assert(UseSHA256Intrinsics, "need SHA256 instruction support");
    state = get_state_from_sha_object(sha_obj);
    stubAddr = StubRoutines::sha256_implCompress();
    stubName = "sha256_implCompress";
    break;
  case vmIntrinsics::_sha5_implCompress:
    assert(UseSHA512Intrinsics, "need SHA512 instruction support");
    state = get_state_from_sha5_object(sha_obj);
    stubAddr = StubRoutines::sha512_implCompress();
    stubName = "sha512_implCompress";
    break;
  default:
    fatal_unexpected_iid(id);
    return false;
  }
  if (state == NULL) return false;
  Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::sha_implCompress_Type(),
                                 stubAddr, stubName, TypePtr::BOTTOM,
                                 src_start, state);
  return true;
}
bool LibraryCallKit::inline_digestBase_implCompressMB(int predicate) {
  assert(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics,
         "need SHA1/SHA256/SHA512 instruction support");
  assert((uint)predicate < 3, "sanity");
  assert(callee()->signature()->size() == 3, "digestBase_implCompressMB has 3 parameters");
  Node* digestBase_obj = argument(0); // The receiver was checked for NULL already.
  Node* src            = argument(1); // byte[] array
  Node* ofs            = argument(2); // type int
  Node* limit          = argument(3); // type int
  const Type* src_type = src->Value(&_gvn);
  const TypeAryPtr* top_src = src_type->isa_aryptr();
  if (top_src  == NULL || top_src->klass()  == NULL) {
    return false;
  }
  BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
  if (src_elem != T_BYTE) {
    return false;
  }
  Node* src_start = array_element_address(src, ofs, src_elem);
  const char* klass_SHA_name = NULL;
  const char* stub_name = NULL;
  address     stub_addr = NULL;
  bool        long_state = false;
  switch (predicate) {
  case 0:
    if (UseSHA1Intrinsics) {
      klass_SHA_name = "sun/security/provider/SHA";
      stub_name = "sha1_implCompressMB";
      stub_addr = StubRoutines::sha1_implCompressMB();
    }
    break;
  case 1:
    if (UseSHA256Intrinsics) {
      klass_SHA_name = "sun/security/provider/SHA2";
      stub_name = "sha256_implCompressMB";
      stub_addr = StubRoutines::sha256_implCompressMB();
    }
    break;
  case 2:
    if (UseSHA512Intrinsics) {
      klass_SHA_name = "sun/security/provider/SHA5";
      stub_name = "sha512_implCompressMB";
      stub_addr = StubRoutines::sha512_implCompressMB();
      long_state = true;
    }
    break;
  default:
    fatal(err_msg_res("unknown SHA intrinsic predicate: %d", predicate));
  }
  if (klass_SHA_name != NULL) {
    const TypeInstPtr* tinst = _gvn.type(digestBase_obj)->isa_instptr();
    assert(tinst != NULL, "digestBase_obj is not instance???");
    assert(tinst->klass()->is_loaded(), "DigestBase is not loaded");
    ciKlass* klass_SHA = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make(klass_SHA_name));
    assert(klass_SHA->is_loaded(), "predicate checks that this class is loaded");
    ciInstanceKlass* instklass_SHA = klass_SHA->as_instance_klass();
    return inline_sha_implCompressMB(digestBase_obj, instklass_SHA, long_state, stub_addr, stub_name, src_start, ofs, limit);
  }
  return false;
}
bool LibraryCallKit::inline_sha_implCompressMB(Node* digestBase_obj, ciInstanceKlass* instklass_SHA,
                                               bool long_state, address stubAddr, const char *stubName,
                                               Node* src_start, Node* ofs, Node* limit) {
  const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_SHA);
  const TypeOopPtr* xtype = aklass->as_instance_type();
  Node* sha_obj = new (C) CheckCastPPNode(control(), digestBase_obj, xtype);
  sha_obj = _gvn.transform(sha_obj);
  Node* state;
  if (long_state) {
    state = get_state_from_sha5_object(sha_obj);
  } else {
    state = get_state_from_sha_object(sha_obj);
  }
  if (state == NULL) return false;
  Node *call;
  if (CCallingConventionRequiresIntsAsLongs) {
    call = make_runtime_call(RC_LEAF|RC_NO_FP,
                             OptoRuntime::digestBase_implCompressMB_Type(),
                             stubAddr, stubName, TypePtr::BOTTOM,
                             src_start, state, ofs XTOP, limit XTOP);
  } else {
    call = make_runtime_call(RC_LEAF|RC_NO_FP,
                             OptoRuntime::digestBase_implCompressMB_Type(),
                             stubAddr, stubName, TypePtr::BOTTOM,
                             src_start, state, ofs, limit);
  }
  Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
  set_result(result);
  return true;
}
Node * LibraryCallKit::get_state_from_sha_object(Node *sha_object) {
  Node* sha_state = load_field_from_object(sha_object, "state", "[I", /*is_exact*/ false);
  assert (sha_state != NULL, "wrong version of sun.security.provider.SHA/SHA2");
  if (sha_state == NULL) return (Node *) NULL;
  Node* state = array_element_address(sha_state, intcon(0), T_INT);
  return state;
}
Node * LibraryCallKit::get_state_from_sha5_object(Node *sha_object) {
  Node* sha_state = load_field_from_object(sha_object, "state", "[J", /*is_exact*/ false);
  assert (sha_state != NULL, "wrong version of sun.security.provider.SHA5");
  if (sha_state == NULL) return (Node *) NULL;
  Node* state = array_element_address(sha_state, intcon(0), T_LONG);
  return state;
}
Node* LibraryCallKit::inline_digestBase_implCompressMB_predicate(int predicate) {
  assert(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics,
         "need SHA1/SHA256/SHA512 instruction support");
  assert((uint)predicate < 3, "sanity");
  Node* digestBaseObj = argument(0);
  const TypeInstPtr* tinst = _gvn.type(digestBaseObj)->isa_instptr();
  assert(tinst != NULL, "digestBaseObj is null");
  assert(tinst->klass()->is_loaded(), "DigestBase is not loaded");
  const char* klass_SHA_name = NULL;
  switch (predicate) {
  case 0:
    if (UseSHA1Intrinsics) {
      klass_SHA_name = "sun/security/provider/SHA";
    }
    break;
  case 1:
    if (UseSHA256Intrinsics) {
      klass_SHA_name = "sun/security/provider/SHA2";
    }
    break;
  case 2:
    if (UseSHA512Intrinsics) {
      klass_SHA_name = "sun/security/provider/SHA5";
    }
    break;
  default:
    fatal(err_msg_res("unknown SHA intrinsic predicate: %d", predicate));
  }
  ciKlass* klass_SHA = NULL;
  if (klass_SHA_name != NULL) {
    klass_SHA = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make(klass_SHA_name));
  }
  if ((klass_SHA == NULL) || !klass_SHA->is_loaded()) {
    Node* ctrl = control();
    set_control(top()); // no intrinsic path
    return ctrl;
  }
  ciInstanceKlass* instklass_SHA = klass_SHA->as_instance_klass();
  Node* instofSHA = gen_instanceof(digestBaseObj, makecon(TypeKlassPtr::make(instklass_SHA)));
  Node* cmp_instof = _gvn.transform(new (C) CmpINode(instofSHA, intcon(1)));
  Node* bool_instof = _gvn.transform(new (C) BoolNode(cmp_instof, BoolTest::ne));
  Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN);
  return instof_false;  // even if it is NULL
}
bool LibraryCallKit::inline_profileBoolean() {
  Node* counts = argument(1);
  const TypeAryPtr* ary = NULL;
  ciArray* aobj = NULL;
  if (counts->is_Con()
      && (ary = counts->bottom_type()->isa_aryptr()) != NULL
      && (aobj = ary->const_oop()->as_array()) != NULL
      && (aobj->length() == 2)) {
    jint false_cnt = aobj->element_value(0).as_int();
    jint  true_cnt = aobj->element_value(1).as_int();
    if (C->log() != NULL) {
      C->log()->elem("observe source='profileBoolean' false='%d' true='%d'",
                     false_cnt, true_cnt);
    }
    if (false_cnt + true_cnt == 0) {
      uncommon_trap_exact(Deoptimization::Reason_intrinsic,
                          Deoptimization::Action_reinterpret);
      return true;
    }
    Node* result = argument(0);
    if (false_cnt == 0 || true_cnt == 0) {
      int expected_val = (false_cnt == 0) ? 1 : 0;
      Node* cmp  = _gvn.transform(new (C) CmpINode(result, intcon(expected_val)));
      Node* test = _gvn.transform(new (C) BoolNode(cmp, BoolTest::eq));
      IfNode* check = create_and_map_if(control(), test, PROB_ALWAYS, COUNT_UNKNOWN);
      Node* fast_path = _gvn.transform(new (C) IfTrueNode(check));
      Node* slow_path = _gvn.transform(new (C) IfFalseNode(check));
      { // Slow path: uncommon trap for never seen value and then reexecute
        PreserveJVMState pjvms(this);
        PreserveReexecuteState preexecs(this);
        jvms()->set_should_reexecute(true);
        set_control(slow_path);
        set_i_o(i_o());
        uncommon_trap_exact(Deoptimization::Reason_intrinsic,
                            Deoptimization::Action_reinterpret);
      }
      set_control(fast_path);
      result = intcon(expected_val);
    }
    Node* profile = _gvn.transform(new (C) ProfileBooleanNode(result, false_cnt, true_cnt));
    C->record_for_igvn(profile);
    set_result(profile);
    return true;
  } else {
    return false;
  }
}
C:\hotspot-69087d08d473\src\share\vm/opto/live.cpp
#include "precompiled.hpp"
#include "memory/allocation.inline.hpp"
#include "opto/callnode.hpp"
#include "opto/chaitin.hpp"
#include "opto/live.hpp"
#include "opto/machnode.hpp"
PhaseLive::PhaseLive( const PhaseCFG &cfg, const LRG_List &names, Arena *arena ) : Phase(LIVE), _cfg(cfg), _names(names), _arena(arena), _live(0) {
}
void PhaseLive::compute(uint maxlrg) {
  _maxlrg   = maxlrg;
  _worklist = new (_arena) Block_List();
  _live = (IndexSet*)_arena->Amalloc(sizeof(IndexSet) * _cfg.number_of_blocks());
  uint i;
  for (i = 0; i < _cfg.number_of_blocks(); i++) {
    _live[i].initialize(_maxlrg);
  }
  ResourceMark rm;              // Nuke temp storage on exit
  _defs = NEW_RESOURCE_ARRAY(IndexSet,_cfg.number_of_blocks());
  for (i = 0; i < _cfg.number_of_blocks(); i++) {
    _defs[i].initialize(_maxlrg);
  }
  _deltas = NEW_RESOURCE_ARRAY(IndexSet*,_cfg.number_of_blocks());
  memset( _deltas, 0, sizeof(IndexSet*)* _cfg.number_of_blocks());
  _free_IndexSet = NULL;
  VectorSet first_pass(Thread::current()->resource_area());
  for (uint j = _cfg.number_of_blocks(); j > 0; j--) {
    Block* block = _cfg.get_block(j - 1);
    IndexSet* use = getset(block);
    IndexSet* def = &_defs[block->_pre_order-1];
    DEBUG_ONLY(IndexSet *def_outside = getfreeset();)
    uint i;
    for (i = block->number_of_nodes(); i > 1; i--) {
      Node* n = block->get_node(i-1);
      if (n->is_Phi()) {
        break;
      }
      uint r = _names.at(n->_idx);
      assert(!def_outside->member(r), "Use of external LRG overlaps the same LRG defined in this block");
      def->insert( r );
      use->remove( r );
      uint cnt = n->req();
      for (uint k = 1; k < cnt; k++) {
        Node *nk = n->in(k);
        uint nkidx = nk->_idx;
        if (_cfg.get_block_for_node(nk) != block) {
          uint u = _names.at(nkidx);
          use->insert(u);
          DEBUG_ONLY(def_outside->insert(u);)
        }
      }
    }
#ifdef ASSERT
    def_outside->set_next(_free_IndexSet);
    _free_IndexSet = def_outside;     // Drop onto free list
#endif
    for (uint k = i; k > 0; k--) {
      uint r = _names.at(block->get_node(k - 1)->_idx);
      def->insert(r);
      use->remove(r);
    }
    for (uint l = 1; l < block->num_preds(); l++) {
      Block* p = _cfg.get_block_for_node(block->pred(l));
      add_liveout(p, use, first_pass);
      for (uint k = i; k > 0; k--) {
        add_liveout(p, _names.at(block->get_node(k-1)->in(l)->_idx), first_pass);
      }
    }
    freeset(block);
    first_pass.set(block->_pre_order);
    while (_worklist->size()) {
      Block* block = _worklist->pop();
      IndexSet *delta = getset(block);
      assert( delta->count(), "missing delta set" );
      for (uint l = 1; l < block->num_preds(); l++) {
        Block* predecessor = _cfg.get_block_for_node(block->pred(l));
        add_liveout(predecessor, delta, first_pass);
      }
      freeset(block);
    } // End of while-worklist-not-empty
  } // End of for-all-blocks-outer-loop
  for (i = 0; i < _cfg.number_of_blocks(); i++) {
    _defs[i].clear();
    if (_deltas[i]) {
      _deltas[i]->clear();
    }
  }
  IndexSet *free = _free_IndexSet;
  while (free != NULL) {
    IndexSet *temp = free;
    free = free->next();
    temp->clear();
  }
}
#ifndef PRODUCT
void PhaseLive::stats(uint iters) const {
}
#endif
IndexSet *PhaseLive::getset( Block *p ) {
  IndexSet *delta = _deltas[p->_pre_order-1];
  if( !delta )                  // Not on worklist?
    delta = _deltas[p->_pre_order-1] = getfreeset();
  return delta;                 // Return set of new live-out items
}
IndexSet *PhaseLive::getfreeset( ) {
  IndexSet *f = _free_IndexSet;
  if( !f ) {
    f = new IndexSet;
    f->initialize(_maxlrg, Thread::current()->resource_area());
  } else {
    _free_IndexSet = f->next();
    f->initialize(_maxlrg, Thread::current()->resource_area());
  }
  return f;
}
void PhaseLive::freeset( const Block *p ) {
  IndexSet *f = _deltas[p->_pre_order-1];
  f->set_next(_free_IndexSet);
  _free_IndexSet = f;           // Drop onto free list
  _deltas[p->_pre_order-1] = NULL;
}
void PhaseLive::add_liveout( Block *p, uint r, VectorSet &first_pass ) {
  IndexSet *live = &_live[p->_pre_order-1];
  if( live->insert(r) ) {       // If actually inserted...
    if( !_defs[p->_pre_order-1].member( r ) ) {
      if( !_deltas[p->_pre_order-1] && // Not on worklist?
          first_pass.test(p->_pre_order) )
        _worklist->push(p);     // Actually go on worklist if already 1st pass
      getset(p)->insert(r);
    }
  }
}
void PhaseLive::add_liveout( Block *p, IndexSet *lo, VectorSet &first_pass ) {
  IndexSet *live = &_live[p->_pre_order-1];
  IndexSet *defs = &_defs[p->_pre_order-1];
  IndexSet *on_worklist = _deltas[p->_pre_order-1];
  IndexSet *delta = on_worklist ? on_worklist : getfreeset();
  IndexSetIterator elements(lo);
  uint r;
  while ((r = elements.next()) != 0) {
    if( live->insert(r) &&      // If actually inserted...
        !defs->member( r ) )    // and not defined locally
      delta->insert(r);         // Then add to live-in set
  }
  if( delta->count() ) {                // If actually added things
    _deltas[p->_pre_order-1] = delta; // Flag as on worklist now
    if( !on_worklist &&         // Not on worklist?
        first_pass.test(p->_pre_order) )
      _worklist->push(p);       // Actually go on worklist if already 1st pass
  } else {                      // Nothing there; just free it
    delta->set_next(_free_IndexSet);
    _free_IndexSet = delta;     // Drop onto free list
  }
}
#ifndef PRODUCT
void PhaseLive::dump( const Block *b ) const {
  tty->print("Block %d: ",b->_pre_order);
  tty->print("LiveOut: ");  _live[b->_pre_order-1].dump();
  uint cnt = b->number_of_nodes();
  for( uint i=0; i<cnt; i++ ) {
    tty->print("L%d/", _names.at(b->get_node(i)->_idx));
    b->get_node(i)->dump();
  }
  tty->print("\n");
}
void PhaseChaitin::verify_base_ptrs( ResourceArea *a ) const {
#ifdef ASSERT
  Unique_Node_List worklist(a);
  for (uint i = 0; i < _cfg.number_of_blocks(); i++) {
    Block* block = _cfg.get_block(i);
    for (uint j = block->end_idx() + 1; j > 1; j--) {
      Node* n = block->get_node(j-1);
      if (n->is_Phi()) {
        break;
      }
      if (n->is_MachSafePoint()) {
        MachSafePointNode *sfpt = n->as_MachSafePoint();
        JVMState* jvms = sfpt->jvms();
        if (jvms != NULL) {
          if (jvms->oopoff() < sfpt->req()) {
            for (uint idx = jvms->oopoff(); idx < sfpt->req(); idx++) {
              Node *check = sfpt->in(idx);
              bool is_derived = ((idx - jvms->oopoff()) & 1) == 0;
              worklist.clear();
              worklist.push(check);
              uint k = 0;
              while( k < worklist.size() ) {
                check = worklist.at(k);
                assert(check,"Bad base or derived pointer");
                int isc = check->is_Copy();
                if( isc ) {
                  worklist.push(check->in(isc));
                } else if( check->is_Phi() ) {
                  for (uint m = 1; m < check->req(); m++)
                    worklist.push(check->in(m));
                } else if( check->is_Con() ) {
                  if (is_derived) {
                    assert(!is_derived || check->bottom_type()->is_ptr()->ptr() == TypePtr::Null,"Bad derived pointer");
                  } else {
                    assert(check->bottom_type()->is_ptr()->_offset == 0,"Bad base pointer");
                    if (check->is_Mach()) {
                      assert(check->as_Mach()->ideal_Opcode() == Op_ConP,"Bad base pointer");
                    } else {
                      assert(check->Opcode() == Op_ConP &&
                             check->bottom_type()->is_ptr()->ptr() == TypePtr::Null,"Bad base pointer");
                    }
                  }
                } else if( check->bottom_type()->is_ptr()->_offset == 0 ) {
                  if(check->is_Proj() || check->is_Mach() &&
                     (check->as_Mach()->ideal_Opcode() == Op_CreateEx ||
                      check->as_Mach()->ideal_Opcode() == Op_ThreadLocal ||
                      check->as_Mach()->ideal_Opcode() == Op_CMoveP ||
                      check->as_Mach()->ideal_Opcode() == Op_CheckCastPP ||
#ifdef _LP64
                      UseCompressedOops && check->as_Mach()->ideal_Opcode() == Op_CastPP ||
                      UseCompressedOops && check->as_Mach()->ideal_Opcode() == Op_DecodeN ||
                      UseCompressedClassPointers && check->as_Mach()->ideal_Opcode() == Op_DecodeNKlass ||
#endif
                      check->as_Mach()->ideal_Opcode() == Op_LoadP ||
                      check->as_Mach()->ideal_Opcode() == Op_LoadKlass)) {
                  } else {
                    check->dump();
                    assert(false,"Bad base or derived pointer");
                  }
                } else {
                  assert(is_derived,"Bad base pointer");
                  assert(check->is_Mach() && check->as_Mach()->ideal_Opcode() == Op_AddP,"Bad derived pointer");
                }
                k++;
                assert(k < 100000,"Derived pointer checking in infinite loop");
              } // End while
            }
          } // End of check for derived pointers
        } // End of Kcheck for debug info
      } // End of if found a safepoint
    } // End of forall instructions in block
  } // End of forall blocks
#endif
}
void PhaseChaitin::verify( ResourceArea *a, bool verify_ifg ) const {
#ifdef ASSERT
  if( VerifyOpto || VerifyRegisterAllocator ) {
    _cfg.verify();
    verify_base_ptrs(a);
    if(verify_ifg)
      _ifg->verify(this);
  }
#endif
}
#endif
C:\hotspot-69087d08d473\src\share\vm/opto/live.hpp
#ifndef SHARE_VM_OPTO_LIVE_HPP
#define SHARE_VM_OPTO_LIVE_HPP
#include "libadt/port.hpp"
#include "libadt/vectset.hpp"
#include "opto/block.hpp"
#include "opto/indexSet.hpp"
#include "opto/phase.hpp"
#include "opto/regmask.hpp"
class Block;
class PhaseCFG;
class VectorSet;
class IndexSet;
typedef GrowableArray<uint> LRG_List;
class PhaseLive : public Phase {
  IndexSet *_live;
  IndexSet *_defs;
  IndexSet **_deltas;
  IndexSet *_free_IndexSet;     // Free list of same
  Block_List *_worklist;        // Worklist for iterative solution
  const PhaseCFG &_cfg;         // Basic blocks
  const LRG_List &_names;       // Mapping from Nodes to live ranges
  uint _maxlrg;                 // Largest live-range number
  Arena *_arena;
  IndexSet *getset( Block *p );
  IndexSet *getfreeset( );
  void freeset( const Block *p );
  void add_liveout( Block *p, uint r, VectorSet &first_pass );
  void add_liveout( Block *p, IndexSet *lo, VectorSet &first_pass );
public:
  PhaseLive(const PhaseCFG &cfg, const LRG_List &names, Arena *arena);
  ~PhaseLive() {}
  void compute(uint maxlrg);
  void reset() { _live = NULL; }
  IndexSet *live( const Block * b ) { return &_live[b->_pre_order-1]; }
#ifndef PRODUCT
  void dump( const Block *b ) const;
  void stats(uint iters) const;
#endif
};
#endif // SHARE_VM_OPTO_LIVE_HPP
C:\hotspot-69087d08d473\src\share\vm/opto/locknode.cpp
#include "precompiled.hpp"
#include "opto/locknode.hpp"
#include "opto/parse.hpp"
#include "opto/rootnode.hpp"
#include "opto/runtime.hpp"
const RegMask &BoxLockNode::in_RegMask(uint i) const {
  return _inmask;
}
const RegMask &BoxLockNode::out_RegMask() const {
  return *Matcher::idealreg2regmask[Op_RegP];
}
uint BoxLockNode::size_of() const { return sizeof(*this); }
BoxLockNode::BoxLockNode( int slot ) : Node( Compile::current()->root() ),
                                       _slot(slot), _is_eliminated(false) {
  init_class_id(Class_BoxLock);
  init_flags(Flag_rematerialize);
  OptoReg::Name reg = OptoReg::stack2reg(_slot);
  _inmask.Insert(reg);
}
uint BoxLockNode::hash() const {
  if (EliminateNestedLocks)
    return NO_HASH; // Each locked region has own BoxLock node
  return Node::hash() + _slot + (_is_eliminated ? Compile::current()->fixed_slots() : 0);
}
uint BoxLockNode::cmp( const Node &n ) const {
  if (EliminateNestedLocks)
    return (&n == this); // Always fail except on self
  const BoxLockNode &bn = (const BoxLockNode &)n;
  return bn._slot == _slot && bn._is_eliminated == _is_eliminated;
}
BoxLockNode* BoxLockNode::box_node(Node* box) {
  while (!box->is_BoxLock()) {
    assert(box->is_SpillCopy() || box->is_Phi(), "Bad spill of Lock.");
    box = box->in(1);
  }
  return box->as_BoxLock();
}
OptoReg::Name BoxLockNode::reg(Node* box) {
  return box_node(box)->in_RegMask(0).find_first_elem();
}
bool BoxLockNode::is_simple_lock_region(LockNode** unique_lock, Node* obj) {
  LockNode* lock = NULL;
  bool has_one_lock = false;
  for (uint i = 0; i < this->outcnt(); i++) {
    Node* n = this->raw_out(i);
    assert(!n->is_Phi(), "should not merge BoxLock nodes");
    if (n->is_AbstractLock()) {
      AbstractLockNode* alock = n->as_AbstractLock();
      if (alock->box_node() == this) {
        if (alock->obj_node()->eqv_uncast(obj)) {
          if ((unique_lock != NULL) && alock->is_Lock()) {
            if (lock == NULL) {
              lock = alock->as_Lock();
              has_one_lock = true;
            } else if (lock != alock->as_Lock()) {
              has_one_lock = false;
            }
          }
        } else {
          return false; // Different objects
        }
      }
    }
  }
#ifdef ASSERT
  for (uint i = 0; i < this->outcnt(); i++) {
    Node* n = this->raw_out(i);
    if (n->is_FastLock()) {
      FastLockNode* flock = n->as_FastLock();
      assert((flock->box_node() == this) && flock->obj_node()->eqv_uncast(obj),"");
    }
  }
#endif
  if (unique_lock != NULL && has_one_lock) {
  }
  return true;
}
uint FastLockNode::hash() const { return NO_HASH; }
uint FastLockNode::size_of() const { return sizeof(*this); }
uint FastLockNode::cmp( const Node &n ) const {
  return (&n == this);                // Always fail except on self
}
uint FastUnlockNode::hash() const { return NO_HASH; }
uint FastUnlockNode::cmp( const Node &n ) const {
  return (&n == this);                // Always fail except on self
}
void FastLockNode::create_lock_counter(JVMState* state) {
  BiasedLockingNamedCounter* blnc = (BiasedLockingNamedCounter*)
           OptoRuntime::new_named_counter(state, NamedCounter::BiasedLockingCounter);
  _counters = blnc->counters();
}
void FastLockNode::create_rtm_lock_counter(JVMState* state) {
#if INCLUDE_RTM_OPT
  Compile* C = Compile::current();
  if (C->profile_rtm() || (PrintPreciseRTMLockingStatistics && C->use_rtm())) {
    RTMLockingNamedCounter* rlnc = (RTMLockingNamedCounter*)
           OptoRuntime::new_named_counter(state, NamedCounter::RTMLockingCounter);
    _rtm_counters = rlnc->counters();
    if (UseRTMForStackLocks) {
      rlnc = (RTMLockingNamedCounter*)
           OptoRuntime::new_named_counter(state, NamedCounter::RTMLockingCounter);
      _stack_rtm_counters = rlnc->counters();
    }
  }
#endif
}
void Parse::do_monitor_enter() {
  kill_dead_locals();
  Node* obj = null_check(peek());
  if (stopped()) return;
  pop();
  shared_lock(obj);
}
void Parse::do_monitor_exit() {
  kill_dead_locals();
  pop();                        // Pop oop to unlock
  shared_unlock(map()->peek_monitor_box(), map()->peek_monitor_obj());
}
C:\hotspot-69087d08d473\src\share\vm/opto/locknode.hpp
#ifndef SHARE_VM_OPTO_LOCKNODE_HPP
#define SHARE_VM_OPTO_LOCKNODE_HPP
#include "opto/node.hpp"
#include "opto/opcodes.hpp"
#include "opto/subnode.hpp"
#if defined AD_MD_HPP
# include AD_MD_HPP
#elif defined TARGET_ARCH_MODEL_x86_32
# include "adfiles/ad_x86_32.hpp"
#elif defined TARGET_ARCH_MODEL_x86_64
# include "adfiles/ad_x86_64.hpp"
#elif defined TARGET_ARCH_MODEL_aarch64
# include "adfiles/ad_aarch64.hpp"
#elif defined TARGET_ARCH_MODEL_sparc
# include "adfiles/ad_sparc.hpp"
#elif defined TARGET_ARCH_MODEL_zero
# include "adfiles/ad_zero.hpp"
#elif defined TARGET_ARCH_MODEL_ppc_64
# include "adfiles/ad_ppc_64.hpp"
#endif
class BoxLockNode : public Node {
  const int     _slot; // stack slot
  RegMask     _inmask; // OptoReg corresponding to stack slot
  bool _is_eliminated; // Associated locks were safely eliminated
public:
  BoxLockNode( int lock );
  virtual int Opcode() const;
  virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const;
  virtual uint size(PhaseRegAlloc *ra_) const;
  virtual const RegMask &in_RegMask(uint) const;
  virtual const RegMask &out_RegMask() const;
  virtual uint size_of() const;
  virtual uint hash() const;
  virtual uint cmp( const Node &n ) const;
  virtual const class Type *bottom_type() const { return TypeRawPtr::BOTTOM; }
  virtual uint ideal_reg() const { return Op_RegP; }
  static OptoReg::Name reg(Node* box_node);
  static BoxLockNode* box_node(Node* box_node);
  static bool same_slot(Node* box1, Node* box2) {
    return box1->as_BoxLock()->_slot == box2->as_BoxLock()->_slot;
  }
  int stack_slot() const { return _slot; }
  bool is_eliminated() const { return _is_eliminated; }
  void set_eliminated()      { _is_eliminated = true; }
  bool is_simple_lock_region(LockNode** unique_lock, Node* obj);
#ifndef PRODUCT
  virtual void format( PhaseRegAlloc *, outputStream *st ) const;
  virtual void dump_spec(outputStream *st) const { st->print("  Lock %d",_slot); }
#endif
};
class FastLockNode: public CmpNode {
private:
  BiasedLockingCounters*        _counters;
  RTMLockingCounters*       _rtm_counters; // RTM lock counters for inflated locks
  RTMLockingCounters* _stack_rtm_counters; // RTM lock counters for stack locks
public:
  FastLockNode(Node *ctrl, Node *oop, Node *box) : CmpNode(oop,box) {
    init_req(0,ctrl);
    init_class_id(Class_FastLock);
    _counters = NULL;
    _rtm_counters = NULL;
    _stack_rtm_counters = NULL;
  }
  Node* obj_node() const { return in(1); }
  Node* box_node() const { return in(2); }
  void  set_box_node(Node* box) { set_req(2, box); }
  virtual uint hash() const ;                  // { return NO_HASH; }
  virtual uint size_of() const;
  virtual uint cmp( const Node &n ) const ;    // Always fail, except on self
  virtual int Opcode() const;
  virtual const Type *Value( PhaseTransform *phase ) const { return TypeInt::CC; }
  const Type *sub(const Type *t1, const Type *t2) const { return TypeInt::CC;}
  void create_lock_counter(JVMState* s);
  void create_rtm_lock_counter(JVMState* state);
  BiasedLockingCounters*        counters() const { return _counters; }
  RTMLockingCounters*       rtm_counters() const { return _rtm_counters; }
  RTMLockingCounters* stack_rtm_counters() const { return _stack_rtm_counters; }
};
class FastUnlockNode: public CmpNode {
public:
  FastUnlockNode(Node *ctrl, Node *oop, Node *box) : CmpNode(oop,box) {
    init_req(0,ctrl);
    init_class_id(Class_FastUnlock);
  }
  Node* obj_node() const { return in(1); }
  Node* box_node() const { return in(2); }
  virtual uint hash() const ;                  // { return NO_HASH; }
  virtual uint cmp( const Node &n ) const ;    // Always fail, except on self
  virtual int Opcode() const;
  virtual const Type *Value( PhaseTransform *phase ) const { return TypeInt::CC; }
  const Type *sub(const Type *t1, const Type *t2) const { return TypeInt::CC;}
};
#endif // SHARE_VM_OPTO_LOCKNODE_HPP
C:\hotspot-69087d08d473\src\share\vm/opto/loopnode.cpp
#include "precompiled.hpp"
#include "ci/ciMethodData.hpp"
#include "compiler/compileLog.hpp"
#include "libadt/vectset.hpp"
#include "memory/allocation.inline.hpp"
#include "opto/addnode.hpp"
#include "opto/callnode.hpp"
#include "opto/connode.hpp"
#include "opto/divnode.hpp"
#include "opto/idealGraphPrinter.hpp"
#include "opto/loopnode.hpp"
#include "opto/mulnode.hpp"
#include "opto/rootnode.hpp"
#include "opto/superword.hpp"
const Node* Node::is_loop_iv() const {
  if (this->is_Phi() && !this->as_Phi()->is_copy() &&
      this->as_Phi()->region()->is_CountedLoop() &&
      this->as_Phi()->region()->as_CountedLoop()->phi() == this) {
    return this;
  } else {
    return NULL;
  }
}
#ifndef PRODUCT
void LoopNode::dump_spec(outputStream *st) const {
  if (is_inner_loop()) st->print( "inner " );
  if (is_partial_peel_loop()) st->print( "partial_peel " );
  if (partial_peel_has_failed()) st->print( "partial_peel_failed " );
}
#endif
bool LoopNode::is_valid_counted_loop() const {
  if (is_CountedLoop()) {
    CountedLoopNode*    l  = as_CountedLoop();
    CountedLoopEndNode* le = l->loopexit();
    if (le != NULL &&
        le->proj_out(1 /* true */) == l->in(LoopNode::LoopBackControl)) {
      Node* phi  = l->phi();
      Node* exit = le->proj_out(0 /* false */);
      if (exit != NULL && exit->Opcode() == Op_IfFalse &&
          phi != NULL && phi->is_Phi() &&
          phi->in(LoopNode::LoopBackControl) == l->incr() &&
          le->loopnode() == l && le->stride_is_con()) {
        return true;
      }
    }
  }
  return false;
}
Node *PhaseIdealLoop::get_early_ctrl( Node *n ) {
  assert( !n->is_Phi() && !n->is_CFG(), "this code only handles data nodes" );
  uint i;
  Node *early;
  if (n->in(0) && !n->is_expensive()) {
    early = n->in(0);
    if (!early->is_CFG()) // Might be a non-CFG multi-def
      early = get_ctrl(early);        // So treat input as a straight data input
    i = 1;
  } else {
    early = get_ctrl(n->in(1));
    i = 2;
  }
  uint e_d = dom_depth(early);
  assert( early, "" );
  for (; i < n->req(); i++) {
    Node *cin = get_ctrl(n->in(i));
    assert( cin, "" );
    uint c_d = dom_depth(cin);
    if (c_d > e_d) {           // Deeper guy?
      early = cin;              // Keep deepest found so far
      e_d = c_d;
    } else if (c_d == e_d &&    // Same depth?
               early != cin) { // If not equal, must use slower algorithm
      Node *n1 = early;
      Node *n2 = cin;
      while (1) {
        n1 = idom(n1);          // Walk up until break cycle
        n2 = idom(n2);
        if (n1 == cin ||        // Walked early up to cin
            dom_depth(n2) < c_d)
          break;                // early is deeper; keep him
        if (n2 == early ||      // Walked cin up to early
            dom_depth(n1) < c_d) {
          early = cin;          // cin is deeper; keep him
          break;
        }
      }
      e_d = dom_depth(early);   // Reset depth register cache
    }
  }
  assert(early == find_non_split_ctrl(early), "unexpected early control");
  if (n->is_expensive()) {
    assert(n->in(0), "should have control input");
    early = get_early_ctrl_for_expensive(n, early);
  }
  return early;
}
Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) {
  assert(n->in(0) && n->is_expensive(), "expensive node with control input here");
  assert(OptimizeExpensiveOps, "optimization off?");
  Node* ctl = n->in(0);
  assert(ctl->is_CFG(), "expensive input 0 must be cfg");
  uint min_dom_depth = dom_depth(earliest);
#ifdef ASSERT
  if (!is_dominator(ctl, earliest) && !is_dominator(earliest, ctl)) {
    dump_bad_graph("Bad graph detected in get_early_ctrl_for_expensive", n, earliest, ctl);
    assert(false, "Bad graph detected in get_early_ctrl_for_expensive");
  }
#endif
  if (dom_depth(ctl) < min_dom_depth) {
    return earliest;
  }
  while (1) {
    Node *next = ctl;
    if (ctl->is_CountedLoop() && ctl->in(1) != NULL && ctl->in(1)->in(0) != NULL && ctl->in(1)->in(0)->is_If()) {
      if (!ctl->in(1)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) {
        break;
      }
      next = idom(ctl->in(1)->in(0));
    } else if (ctl->is_Proj()) {
      Node* parent_ctl = ctl->in(0);
      if (parent_ctl == NULL) {
        break;
      } else if (parent_ctl->is_CountedLoopEnd() && parent_ctl->as_CountedLoopEnd()->loopnode() != NULL) {
        next = parent_ctl->as_CountedLoopEnd()->loopnode()->init_control();
      } else if (parent_ctl->is_If()) {
        if (!ctl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) {
          break;
        }
        assert(idom(ctl) == parent_ctl, "strange");
        next = idom(parent_ctl);
      } else if (ctl->is_CatchProj()) {
        if (ctl->as_Proj()->_con != CatchProjNode::fall_through_index) {
          break;
        }
        assert(parent_ctl->in(0)->in(0)->is_Call(), "strange graph");
        next = parent_ctl->in(0)->in(0)->in(0);
      } else {
        int nb_ctl_proj = 0;
        for (DUIterator_Fast imax, i = parent_ctl->fast_outs(imax); i < imax; i++) {
          Node *p = parent_ctl->fast_out(i);
          if (p->is_Proj() && p->is_CFG()) {
            nb_ctl_proj++;
            if (nb_ctl_proj > 1) {
              break;
            }
          }
        }
        if (nb_ctl_proj > 1) {
          break;
        }
        assert(parent_ctl->is_Start() || parent_ctl->is_MemBar() || parent_ctl->is_Call(), "unexpected node");
        assert(idom(ctl) == parent_ctl, "strange");
        next = idom(parent_ctl);
      }
    } else {
      next = idom(ctl);
    }
    if (next->is_Root() || next->is_Start() || dom_depth(next) < min_dom_depth) {
      break;
    }
    ctl = next;
  }
  if (ctl != n->in(0)) {
    _igvn.hash_delete(n);
    n->set_req(0, ctl);
    _igvn.hash_insert(n);
  }
  return ctl;
}
void PhaseIdealLoop::set_early_ctrl( Node *n ) {
  Node *early = get_early_ctrl(n);
  set_ctrl(n, early);
}
void PhaseIdealLoop::set_subtree_ctrl( Node *n ) {
  if( _nodes[n->_idx] ) return;
  uint i;
  for( i = 0; i < n->req(); ++i ) {
    Node *m = n->in(i);
    if( m && m != C->root() )
      set_subtree_ctrl( m );
  }
  set_early_ctrl( n );
}
bool PhaseIdealLoop::is_counted_loop( Node *x, IdealLoopTree *loop ) {
  PhaseGVN *gvn = &_igvn;
  if (x->in(LoopNode::Self) == NULL || x->req() != 3 || loop->_irreducible) {
    return false;
  }
  Node *init_control = x->in(LoopNode::EntryControl);
  Node *back_control = x->in(LoopNode::LoopBackControl);
  if (init_control == NULL || back_control == NULL)    // Partially dead
    return false;
  if (init_control->is_top() || back_control->is_top())
    return false;
  if (back_control->Opcode() == Op_SafePoint) {
    if (UseCountedLoopSafepoints) {
      return false;
    }
    back_control = back_control->in(TypeFunc::Control);
  }
  Node *iftrue = back_control;
  uint iftrue_op = iftrue->Opcode();
  if (iftrue_op != Op_IfTrue &&
      iftrue_op != Op_IfFalse)
    return false; // Obscure back-control
  Node *iff = iftrue->in(0);
  if (get_loop(iff) != loop || !iff->in(1)->is_Bool())
    return false;
  BoolNode *test = iff->in(1)->as_Bool();
  BoolTest::mask bt = test->_test._test;
  float cl_prob = iff->as_If()->_prob;
  if (iftrue_op == Op_IfFalse) {
    bt = BoolTest(bt).negate();
    cl_prob = 1.0 - cl_prob;
  }
  Node *cmp = test->in(1);
  int cmp_op = cmp->Opcode();
  if (cmp_op != Op_CmpI)
    return false;                // Avoid pointer & float compares
  Node *incr  = cmp->in(1);
  Node *limit = cmp->in(2);
  if (!is_member(loop, get_ctrl(incr))) { // Swapped trip counter and limit?
    Node *tmp = incr;            // Then reverse order into the CmpI
    incr = limit;
    limit = tmp;
    bt = BoolTest(bt).commute(); // And commute the exit test
  }
  if (is_member(loop, get_ctrl(limit))) // Limit must be loop-invariant
    return false;
  if (!is_member(loop, get_ctrl(incr))) // Trip counter must be loop-variant
    return false;
  Node* phi_incr = NULL;
  if (incr->is_Phi()) {
    if (incr->as_Phi()->region() != x || incr->req() != 3)
      return false; // Not simple trip counter expression
    phi_incr = incr;
    incr = phi_incr->in(LoopNode::LoopBackControl); // Assume incr is on backedge of Phi
    if (!is_member(loop, get_ctrl(incr))) // Trip counter must be loop-variant
      return false;
  }
  Node* trunc1 = NULL;
  Node* trunc2 = NULL;
  const TypeInt* iv_trunc_t = NULL;
  if (!(incr = CountedLoopNode::match_incr_with_optional_truncation(incr, &trunc1, &trunc2, &iv_trunc_t))) {
    return false; // Funny increment opcode
  }
  assert(incr->Opcode() == Op_AddI, "wrong increment code");
  Node *xphi = incr->in(1);
  Node *stride = incr->in(2);
  if (!stride->is_Con()) {     // Oops, swap these
    if (!xphi->is_Con())       // Is the other guy a constant?
      return false;             // Nope, unknown stride, bail out
    Node *tmp = xphi;           // 'incr' is commutative, so ok to swap
    xphi = stride;
    stride = tmp;
  }
  int stride_con = stride->get_int();
  if (stride_con == 0)
    return false; // missed some peephole opt
  if (!xphi->is_Phi())
    return false; // Too much math on the trip counter
  if (phi_incr != NULL && phi_incr != xphi)
    return false;
  PhiNode *phi = xphi->as_Phi();
  if (phi->region() != x)
    return false;
  if (trunc1 == NULL && phi->in(LoopNode::LoopBackControl) != incr ||
      trunc1 != NULL && phi->in(LoopNode::LoopBackControl) != trunc1) {
    return false;
  }
  Node *init_trip = phi->in(LoopNode::EntryControl);
  if (!TypeInt::INT->higher_equal(iv_trunc_t)) {
    assert(trunc1 != NULL, "must have found some truncation");
    const TypeInt* phi_ft = filtered_type(phi);
    if (stride_con > 0) {
      if (iv_trunc_t->_hi - phi_ft->_hi < stride_con ||
          iv_trunc_t->_lo > phi_ft->_lo) {
        return false;  // truncation may occur
      }
    } else if (stride_con < 0) {
      if (iv_trunc_t->_lo - phi_ft->_lo > stride_con ||
          iv_trunc_t->_hi < phi_ft->_hi) {
        return false;  // truncation may occur
      }
    }
  } else {
    assert(trunc1 == NULL && trunc2 == NULL, "no truncation for int");
  }
  if (bt == BoolTest::eq || // Bail out, but this loop trips at most twice!
      bt == BoolTest::ne && stride_con != 1 && stride_con != -1 ||
      (bt == BoolTest::le || bt == BoolTest::lt) && stride_con < 0 ||
      (bt == BoolTest::ge || bt == BoolTest::gt) && stride_con > 0) {
    return false; // Bail out
  }
  const TypeInt* init_t = gvn->type(init_trip)->is_int();
  const TypeInt* limit_t = gvn->type(limit)->is_int();
  if (stride_con > 0) {
    jlong init_p = (jlong)init_t->_lo + stride_con;
    if (init_p > (jlong)max_jint || init_p > (jlong)limit_t->_hi)
      return false; // cyclic loop or this loop trips only once
  } else {
    jlong init_p = (jlong)init_t->_hi + stride_con;
    if (init_p < (jlong)min_jint || init_p < (jlong)limit_t->_lo)
      return false; // cyclic loop or this loop trips only once
  }
  if (phi_incr != NULL) {
    if (stride_con > 0) {
      if (init_t->_hi > max_jint - stride_con) {
        return false;
      }
    } else {
      if (init_t->_lo < min_jint - stride_con) {
        return false;
      }
    }
  }
  assert(x->Opcode() == Op_Loop, "regular loops only");
  C->print_method(PHASE_BEFORE_CLOOPS, 3);
  Node *hook = new (C) Node(6);
  if (LoopLimitCheck) {
  bool incl_limit = (bt == BoolTest::le || bt == BoolTest::ge);
  int stride_m  = stride_con - (incl_limit ? 0 : (stride_con > 0 ? 1 : -1));
  if (phi_incr != NULL) {
    stride_m  += stride_con;
  }
  if (limit->is_Con()) {
    int limit_con = limit->get_int();
    if ((stride_con > 0 && limit_con > (max_jint - stride_m)) ||
        (stride_con < 0 && limit_con < (min_jint - stride_m))) {
      return false;
    }
  } else if ((stride_con > 0 && limit_t->_hi <= (max_jint - stride_m)) ||
             (stride_con < 0 && limit_t->_lo >= (min_jint - stride_m))) {
  } else {
    ProjNode *limit_check_proj = find_predicate_insertion_point(init_control, Deoptimization::Reason_loop_limit_check);
    if (!limit_check_proj) {
#ifdef ASSERT
      if (TraceLoopLimitCheck) {
        tty->print("missing loop limit check:");
        loop->dump_head();
        x->dump(1);
      }
#endif
      return false;
    }
    IfNode* check_iff = limit_check_proj->in(0)->as_If();
    Node* cmp_limit;
    Node* bol;
    if (stride_con > 0) {
      cmp_limit = new (C) CmpINode(limit, _igvn.intcon(max_jint - stride_m));
      bol = new (C) BoolNode(cmp_limit, BoolTest::le);
    } else {
      cmp_limit = new (C) CmpINode(limit, _igvn.intcon(min_jint - stride_m));
      bol = new (C) BoolNode(cmp_limit, BoolTest::ge);
    }
    cmp_limit = _igvn.register_new_node_with_optimizer(cmp_limit);
    bol = _igvn.register_new_node_with_optimizer(bol);
    set_subtree_ctrl(bol);
    assert(check_iff->in(1)->Opcode() == Op_Conv2B &&
           check_iff->in(1)->in(1)->Opcode() == Op_Opaque1, "");
    Node* opq = check_iff->in(1)->in(1);
    _igvn.hash_delete(opq);
    opq->set_req(1, bol);
    set_ctrl(opq, check_iff->in(0));
    set_ctrl(check_iff->in(1), check_iff->in(0));
#ifndef PRODUCT
    if (TraceLoopLimitCheck) {
      tty->print_cr("Counted Loop Limit Check generated:");
      debug_only( bol->dump(2); )
    }
#endif
  }
  if (phi_incr != NULL) {
    limit = gvn->transform(new (C) AddINode(limit, stride));
  }
  if (bt == BoolTest::ne) {
    assert(stride_con == 1 || stride_con == -1, "simple increment only");
    if (stride_con > 0 && init_t->_hi < limit_t->_lo)
      bt = BoolTest::lt;
    if (stride_con < 0 && init_t->_lo > limit_t->_hi)
      bt = BoolTest::gt;
  }
  if (incl_limit) {
    Node* one = (stride_con > 0) ? gvn->intcon( 1) : gvn->intcon(-1);
    limit = gvn->transform(new (C) AddINode(limit, one));
    if (bt == BoolTest::le)
      bt = BoolTest::lt;
    else if (bt == BoolTest::ge)
      bt = BoolTest::gt;
    else
      ShouldNotReachHere();
  }
  set_subtree_ctrl( limit );
  } else { // LoopLimitCheck
  if (cmp->in(1) == phi || cmp->in(2) == phi)
    limit = gvn->transform(new (C) AddINode(limit,stride));
  Node *one_p = gvn->intcon( 1);
  Node *one_m = gvn->intcon(-1);
  Node *trip_count = NULL;
  switch( bt ) {
  case BoolTest::eq:
    ShouldNotReachHere();
  case BoolTest::ne:            // Ahh, the case we desire
    if (stride_con == 1)
      trip_count = gvn->transform(new (C) SubINode(limit,init_trip));
    else if (stride_con == -1)
      trip_count = gvn->transform(new (C) SubINode(init_trip,limit));
    else
      ShouldNotReachHere();
    set_subtree_ctrl(trip_count);
    break;
  case BoolTest::le:            // Maybe convert to '<' case
    limit = gvn->transform(new (C) AddINode(limit,one_p));
    set_subtree_ctrl( limit );
    hook->init_req(4, limit);
    bt = BoolTest::lt;
  case BoolTest::lt: {          // Maybe convert to '!=' case
    if (stride_con < 0) // Count down loop rolls through MAXINT
      ShouldNotReachHere();
    Node *range = gvn->transform(new (C) SubINode(limit,init_trip));
    set_subtree_ctrl( range );
    hook->init_req(0, range);
    Node *bias  = gvn->transform(new (C) AddINode(range,stride));
    set_subtree_ctrl( bias );
    hook->init_req(1, bias);
    Node *bias1 = gvn->transform(new (C) AddINode(bias,one_m));
    set_subtree_ctrl( bias1 );
    hook->init_req(2, bias1);
    trip_count  = gvn->transform(new (C) DivINode(0,bias1,stride));
    set_subtree_ctrl( trip_count );
    hook->init_req(3, trip_count);
    break;
  }
  case BoolTest::ge:            // Maybe convert to '>' case
    limit = gvn->transform(new (C) AddINode(limit,one_m));
    set_subtree_ctrl( limit );
    hook->init_req(4 ,limit);
    bt = BoolTest::gt;
  case BoolTest::gt: {          // Maybe convert to '!=' case
    if (stride_con > 0) // count up loop rolls through MININT
      ShouldNotReachHere();
    Node *range = gvn->transform(new (C) SubINode(limit,init_trip));
    set_subtree_ctrl( range );
    hook->init_req(0, range);
    Node *bias  = gvn->transform(new (C) AddINode(range,stride));
    set_subtree_ctrl( bias );
    hook->init_req(1, bias);
    Node *bias1 = gvn->transform(new (C) AddINode(bias,one_p));
    set_subtree_ctrl( bias1 );
    hook->init_req(2, bias1);
    trip_count  = gvn->transform(new (C) DivINode(0,bias1,stride));
    set_subtree_ctrl( trip_count );
    hook->init_req(3, trip_count);
    break;
  }
  } // switch( bt )
  Node *span = gvn->transform(new (C) MulINode(trip_count,stride));
  set_subtree_ctrl( span );
  hook->init_req(5, span);
  limit = gvn->transform(new (C) AddINode(span,init_trip));
  set_subtree_ctrl( limit );
  } // LoopLimitCheck
  if (!UseCountedLoopSafepoints) {
    Node *sfpt = x->in(LoopNode::LoopBackControl);
    if (sfpt->Opcode() == Op_SafePoint && is_deleteable_safept(sfpt)) {
      lazy_replace( sfpt, iftrue );
      if (loop->_safepts != NULL) {
        loop->_safepts->yank(sfpt);
      }
      loop->_tail = iftrue;
    }
  }
  incr = incr->clone();
  incr->set_req(1,phi);
  incr->set_req(2,stride);
  incr = _igvn.register_new_node_with_optimizer(incr);
  set_early_ctrl( incr );
  _igvn.hash_delete(phi);
  phi->set_req_X( LoopNode::LoopBackControl, incr, &_igvn );
  if (!TypeInt::INT->higher_equal(phi->bottom_type())) {
    Node* nphi = PhiNode::make(phi->in(0), phi->in(LoopNode::EntryControl), TypeInt::INT);
    nphi->set_req(LoopNode::LoopBackControl, phi->in(LoopNode::LoopBackControl));
    nphi = _igvn.register_new_node_with_optimizer(nphi);
    set_ctrl(nphi, get_ctrl(phi));
    _igvn.replace_node(phi, nphi);
    phi = nphi->as_Phi();
  }
  cmp = cmp->clone();
  cmp->set_req(1,incr);
  cmp->set_req(2,limit);
  cmp = _igvn.register_new_node_with_optimizer(cmp);
  set_ctrl(cmp, iff->in(0));
  test = test->clone()->as_Bool();
  (*(BoolTest*)&test->_test)._test = bt;
  test->set_req(1,cmp);
  _igvn.register_new_node_with_optimizer(test);
  set_ctrl(test, iff->in(0));
  Node *lex = _igvn.register_new_node_with_optimizer(new (C) CountedLoopEndNode( iff->in(0), test, cl_prob, iff->as_If()->_fcnt ));
  IfNode *le = lex->as_If();
  uint dd = dom_depth(iff);
  set_idom(le, le->in(0), dd); // Update dominance for loop exit
  set_loop(le, loop);
  Node *iffalse = iff->as_If()->proj_out(!(iftrue_op == Op_IfTrue));
  if (iftrue_op == Op_IfFalse) {
    Node *ift2=_igvn.register_new_node_with_optimizer(new (C) IfTrueNode (le));
    Node *iff2=_igvn.register_new_node_with_optimizer(new (C) IfFalseNode(le));
    loop->_tail = back_control = ift2;
    set_loop(ift2, loop);
    set_loop(iff2, get_loop(iffalse));
    lazy_replace(iffalse, iff2);
    lazy_replace(iftrue,  ift2);
    iffalse = iff2;
    iftrue  = ift2;
  } else {
    _igvn.hash_delete(iffalse);
    _igvn.hash_delete(iftrue);
    iffalse->set_req_X( 0, le, &_igvn );
    iftrue ->set_req_X( 0, le, &_igvn );
  }
  set_idom(iftrue,  le, dd+1);
  set_idom(iffalse, le, dd+1);
  assert(iff->outcnt() == 0, "should be dead now");
  lazy_replace( iff, le ); // fix 'get_ctrl'
  CountedLoopNode *l = new (C) CountedLoopNode(init_control, back_control);
  l->set_unswitch_count(x->as_Loop()->unswitch_count()); // Preserve
  _igvn.register_new_node_with_optimizer(l);
  set_loop(l, loop);
  loop->_head = l;
  lazy_replace( x, l );
  set_idom(l, init_control, dom_depth(x));
  if (!UseCountedLoopSafepoints) {
    Node *sfpt2 = le->in(0);
    if (sfpt2->Opcode() == Op_SafePoint && is_deleteable_safept(sfpt2)) {
      lazy_replace( sfpt2, sfpt2->in(TypeFunc::Control));
      if (loop->_safepts != NULL) {
        loop->_safepts->yank(sfpt2);
      }
    }
  }
  _igvn.remove_dead_node(hook);
#ifdef ASSERT
  assert(l->is_valid_counted_loop(), "counted loop shape is messed up");
  assert(l == loop->_head && l->phi() == phi && l->loopexit() == lex, "" );
#endif
#ifndef PRODUCT
  if (TraceLoopOpts) {
    tty->print("Counted      ");
    loop->dump_head();
  }
#endif
  C->print_method(PHASE_AFTER_CLOOPS, 3);
  return true;
}
Node* PhaseIdealLoop::exact_limit( IdealLoopTree *loop ) {
  assert(loop->_head->is_CountedLoop(), "");
  CountedLoopNode *cl = loop->_head->as_CountedLoop();
  assert(cl->is_valid_counted_loop(), "");
  if (!LoopLimitCheck || ABS(cl->stride_con()) == 1 ||
      cl->limit()->Opcode() == Op_LoopLimit) {
    return cl->limit();
  }
  Node *limit = NULL;
#ifdef ASSERT
  BoolTest::mask bt = cl->loopexit()->test_trip();
  assert(bt == BoolTest::lt || bt == BoolTest::gt, "canonical test is expected");
#endif
  if (cl->has_exact_trip_count()) {
    int stride_con = cl->stride_con();
    jlong  init_con = cl->init_trip()->get_int();
    jlong limit_con = cl->limit()->get_int();
    julong trip_cnt = cl->trip_count();
    jlong final_con = init_con + trip_cnt*stride_con;
    int final_int = (int)final_con;
    assert(final_con == (jlong)final_int, "final value should be integer");
    limit = _igvn.intcon(final_int);
  } else {
    limit = new (C) LoopLimitNode(C, cl->init_trip(), cl->limit(), cl->stride());
    register_new_node(limit, cl->in(LoopNode::EntryControl));
  }
  assert(limit != NULL, "sanity");
  return limit;
}
Node *LoopNode::Ideal(PhaseGVN *phase, bool can_reshape) {
  if (!can_be_counted_loop(phase)) {
    phase->C->set_major_progress();
  }
  return RegionNode::Ideal(phase, can_reshape);
}
Node *CountedLoopNode::Ideal(PhaseGVN *phase, bool can_reshape) {
  return RegionNode::Ideal(phase, can_reshape);
}
#ifndef PRODUCT
void CountedLoopNode::dump_spec(outputStream *st) const {
  LoopNode::dump_spec(st);
  if (stride_is_con()) {
    st->print("stride: %d ",stride_con());
  }
  if (is_pre_loop ()) st->print("pre of N%d" , _main_idx);
  if (is_main_loop()) st->print("main of N%d", _idx);
  if (is_post_loop()) st->print("post of N%d", _main_idx);
}
#endif
int CountedLoopEndNode::stride_con() const {
  return stride()->bottom_type()->is_int()->get_con();
}
const Type *LoopLimitNode::Value( PhaseTransform *phase ) const {
  const Type* init_t   = phase->type(in(Init));
  const Type* limit_t  = phase->type(in(Limit));
  const Type* stride_t = phase->type(in(Stride));
  if (init_t   == Type::TOP) return Type::TOP;
  if (limit_t  == Type::TOP) return Type::TOP;
  if (stride_t == Type::TOP) return Type::TOP;
  int stride_con = stride_t->is_int()->get_con();
  if (stride_con == 1)
    return NULL;  // Identity
  if (init_t->is_int()->is_con() && limit_t->is_int()->is_con()) {
    jlong init_con   =  init_t->is_int()->get_con();
    jlong limit_con  = limit_t->is_int()->get_con();
    int  stride_m   = stride_con - (stride_con > 0 ? 1 : -1);
    jlong trip_count = (limit_con - init_con + stride_m)/stride_con;
    jlong final_con  = init_con + stride_con*trip_count;
    int final_int = (int)final_con;
    assert(final_con == (jlong)final_int, "final value should be integer");
    return TypeInt::make(final_int);
  }
  return bottom_type(); // TypeInt::INT
}
Node *LoopLimitNode::Ideal(PhaseGVN *phase, bool can_reshape) {
  if (phase->type(in(Init))   == Type::TOP ||
      phase->type(in(Limit))  == Type::TOP ||
      phase->type(in(Stride)) == Type::TOP)
    return NULL;  // Dead
  int stride_con = phase->type(in(Stride))->is_int()->get_con();
  if (stride_con == 1)
    return NULL;  // Identity
  if (in(Init)->is_Con() && in(Limit)->is_Con())
    return NULL;  // Value
  if (!can_reshape || phase->C->major_progress())
    return NULL;
  const TypeInt* init_t  = phase->type(in(Init) )->is_int();
  const TypeInt* limit_t = phase->type(in(Limit))->is_int();
  int stride_p;
  jlong lim, ini;
  julong max;
  if (stride_con > 0) {
    stride_p = stride_con;
    lim = limit_t->_hi;
    ini = init_t->_lo;
    max = (julong)max_jint;
  } else {
    stride_p = -stride_con;
    lim = init_t->_hi;
    ini = limit_t->_lo;
    max = (julong)min_jint;
  }
  julong range = lim - ini + stride_p;
  if (range <= max) {
    Node* stride_m = phase->intcon(stride_con - (stride_con > 0 ? 1 : -1));
    Node *range = phase->transform(new (phase->C) SubINode(in(Limit), in(Init)));
    Node *bias  = phase->transform(new (phase->C) AddINode(range, stride_m));
    Node *trip  = phase->transform(new (phase->C) DivINode(0, bias, in(Stride)));
    Node *span  = phase->transform(new (phase->C) MulINode(trip, in(Stride)));
    return new (phase->C) AddINode(span, in(Init)); // exact limit
  }
  if (is_power_of_2(stride_p) ||                // divisor is 2^n
      !Matcher::has_match_rule(Op_LoopLimit)) { // or no specialized Mach node?
    Node*   init   = phase->transform( new (phase->C) ConvI2LNode(in(Init)));
    Node*  limit   = phase->transform( new (phase->C) ConvI2LNode(in(Limit)));
    Node* stride   = phase->longcon(stride_con);
    Node* stride_m = phase->longcon(stride_con - (stride_con > 0 ? 1 : -1));
    Node *range = phase->transform(new (phase->C) SubLNode(limit, init));
    Node *bias  = phase->transform(new (phase->C) AddLNode(range, stride_m));
    Node *span;
    if (stride_con > 0 && is_power_of_2(stride_p)) {
      Node* neg_stride   = phase->longcon(-stride_con);
      span = phase->transform(new (phase->C) AndLNode(bias, neg_stride));
    } else {
      Node *trip  = phase->transform(new (phase->C) DivLNode(0, bias, stride));
      span = phase->transform(new (phase->C) MulLNode(trip, stride));
    }
    Node *span_int = phase->transform(new (phase->C) ConvL2INode(span));
    return new (phase->C) AddINode(span_int, in(Init)); // exact limit
  }
  return NULL;    // No progress
}
Node *LoopLimitNode::Identity( PhaseTransform *phase ) {
  int stride_con = phase->type(in(Stride))->is_int()->get_con();
  if (stride_con == 1 || stride_con == -1)
    return in(Limit);
  return this;
}
Node* CountedLoopNode::match_incr_with_optional_truncation(
                      Node* expr, Node** trunc1, Node** trunc2, const TypeInt** trunc_type) {
  if (expr == NULL || expr->req() != 3)  return NULL;
  Node *t1 = NULL;
  Node *t2 = NULL;
  const TypeInt* trunc_t = TypeInt::INT;
  Node* n1 = expr;
  int   n1op = n1->Opcode();
  if (n1op == Op_AndI &&
      n1->in(2)->is_Con() &&
      n1->in(2)->bottom_type()->is_int()->get_con() == 0x7fff) {
    t1 = n1;
    n1 = t1->in(1);
    n1op = n1->Opcode();
    trunc_t = TypeInt::CHAR;
  } else if (n1op == Op_RShiftI &&
             n1->in(1) != NULL &&
             n1->in(1)->Opcode() == Op_LShiftI &&
             n1->in(2) == n1->in(1)->in(2) &&
             n1->in(2)->is_Con()) {
    jint shift = n1->in(2)->bottom_type()->is_int()->get_con();
    if (shift == 16 || shift == 8) {
      t1 = n1;
      t2 = t1->in(1);
      n1 = t2->in(1);
      n1op = n1->Opcode();
      if (shift == 16) {
        trunc_t = TypeInt::SHORT;
      } else if (shift == 8) {
        trunc_t = TypeInt::BYTE;
      }
    }
  }
  if (n1op == Op_AddI) {
    return n1;
  }
  return NULL;
}
const TypeInt* PhaseIdealLoop::filtered_type( Node *n, Node* n_ctrl) {
  assert(n && n->bottom_type()->is_int(), "must be int");
  const TypeInt* filtered_t = NULL;
  if (!n->is_Phi()) {
    assert(n_ctrl != NULL || n_ctrl == C->top(), "valid control");
    filtered_t = filtered_type_from_dominators(n, n_ctrl);
  } else {
    Node* phi    = n->as_Phi();
    Node* region = phi->in(0);
    assert(n_ctrl == NULL || n_ctrl == region, "ctrl parameter must be region");
    if (region && region != C->top()) {
      for (uint i = 1; i < phi->req(); i++) {
        Node* val   = phi->in(i);
        Node* use_c = region->in(i);
        const TypeInt* val_t = filtered_type_from_dominators(val, use_c);
        if (val_t != NULL) {
          if (filtered_t == NULL) {
            filtered_t = val_t;
          } else {
            filtered_t = filtered_t->meet(val_t)->is_int();
          }
        }
      }
    }
  }
  const TypeInt* n_t = _igvn.type(n)->is_int();
  if (filtered_t != NULL) {
    n_t = n_t->join(filtered_t)->is_int();
  }
  return n_t;
}
const TypeInt* PhaseIdealLoop::filtered_type_from_dominators( Node* val, Node *use_ctrl) {
  if (val->is_Con()) {
     return val->bottom_type()->is_int();
  }
  uint if_limit = 10; // Max number of dominating if's visited
  const TypeInt* rtn_t = NULL;
  if (use_ctrl && use_ctrl != C->top()) {
    Node* val_ctrl = get_ctrl(val);
    uint val_dom_depth = dom_depth(val_ctrl);
    Node* pred = use_ctrl;
    uint if_cnt = 0;
    while (if_cnt < if_limit) {
      if ((pred->Opcode() == Op_IfTrue || pred->Opcode() == Op_IfFalse)) {
        if_cnt++;
        const TypeInt* if_t = IfNode::filtered_int_type(&_igvn, val, pred);
        if (if_t != NULL) {
          if (rtn_t == NULL) {
            rtn_t = if_t;
          } else {
            rtn_t = rtn_t->join(if_t)->is_int();
          }
        }
      }
      pred = idom(pred);
      if (pred == NULL || pred == C->top()) {
        break;
      }
      if (dom_depth(pred) < val_dom_depth) {
        break;
      }
    }
  }
  return rtn_t;
}
#ifndef PRODUCT
void CountedLoopEndNode::dump_spec(outputStream *st) const {
  if( in(TestValue)->is_Bool() ) {
    BoolTest bt( test_trip()); // Added this for g++.
    st->print("[");
    bt.dump_on(st);
    st->print("]");
  }
  st->print(" ");
  IfNode::dump_spec(st);
}
#endif
int IdealLoopTree::is_member( const IdealLoopTree *l ) const {
  while( l->_nest > _nest ) l = l->_parent;
  return l == this;
}
int IdealLoopTree::set_nest( uint depth ) {
  _nest = depth;
  int bits = _has_call;
  if( _child ) bits |= _child->set_nest(depth+1);
  if( bits ) _has_call = 1;
  if( _next  ) bits |= _next ->set_nest(depth  );
  return bits;
}
void IdealLoopTree::split_fall_in( PhaseIdealLoop *phase, int fall_in_cnt ) {
  PhaseIterGVN &igvn = phase->_igvn;
  uint i;
  Node *landing_pad = new (phase->C) RegionNode( fall_in_cnt+1 );
  phase->set_loop(landing_pad,_parent);
  uint icnt = fall_in_cnt;
  uint oreq = _head->req();
  for( i = oreq-1; i>0; i-- )
    if( !phase->is_member( this, _head->in(i) ) )
      landing_pad->set_req(icnt--,_head->in(i));
  for (DUIterator_Fast jmax, j = _head->fast_outs(jmax); j < jmax; j++) {
    Node *oj = _head->fast_out(j);
    if( oj->is_Phi() ) {
      PhiNode* old_phi = oj->as_Phi();
      assert( old_phi->region() == _head, "" );
      igvn.hash_delete(old_phi);   // Yank from hash before hacking edges
      Node *p = PhiNode::make_blank(landing_pad, old_phi);
      uint icnt = fall_in_cnt;
      for( i = oreq-1; i>0; i-- ) {
        if( !phase->is_member( this, _head->in(i) ) ) {
          p->init_req(icnt--, old_phi->in(i));
          old_phi->del_req(i);
        }
      }
      Node *p2 = igvn.hash_find_insert(p); // Look for a CSE
      if( p2 ) {                // Found CSE
        p->destruct();          // Recover useless new node
        p = p2;                 // Use old node
      } else {
        igvn.register_new_node_with_optimizer(p, old_phi);
      }
      old_phi->add_req(p);
      Node *id_old_phi = old_phi->Identity( &igvn );
      if( id_old_phi != old_phi ) { // Found a simple identity?
        for (DUIterator_Last imin, i = old_phi->last_outs(imin); i >= imin; ) {
          Node* use = old_phi->last_out(i);
          igvn.rehash_node_delayed(use);
          uint uses_found = 0;
          for (uint j = 0; j < use->len(); j++) {
            if (use->in(j) == old_phi) {
              if (j < use->req()) use->set_req (j, id_old_phi);
              else                use->set_prec(j, id_old_phi);
              uses_found++;
            }
          }
          i -= uses_found;    // we deleted 1 or more copies of this edge
        }
      }
      igvn._worklist.push(old_phi);
    }
  }
  for( i = oreq-1; i>0; i-- ) {
    if( !phase->is_member( this, _head->in(i) ) ) {
      _head->del_req(i);
    }
  }
  igvn.register_new_node_with_optimizer(landing_pad, _head);
  _head->add_req(landing_pad);
}
void IdealLoopTree::split_outer_loop( PhaseIdealLoop *phase ) {
  PhaseIterGVN &igvn = phase->_igvn;
  uint outer_idx = 1;
  while( _head->in(outer_idx) != _tail ) outer_idx++;
  Node *ctl = _head->in(LoopNode::EntryControl);
  Node *outer = new (phase->C) LoopNode( ctl, _head->in(outer_idx) );
  outer = igvn.register_new_node_with_optimizer(outer, _head);
  phase->set_created_loop_node();
  _head->set_req(LoopNode::EntryControl, outer);
  _head->del_req(outer_idx);
  for (DUIterator_Fast jmax, j = _head->fast_outs(jmax); j < jmax; j++) {
    Node *out = _head->fast_out(j);
    if( out->is_Phi() ) {
      PhiNode *old_phi = out->as_Phi();
      assert( old_phi->region() == _head, "" );
      Node *phi = PhiNode::make_blank(outer, old_phi);
      phi->init_req(LoopNode::EntryControl,    old_phi->in(LoopNode::EntryControl));
      phi->init_req(LoopNode::LoopBackControl, old_phi->in(outer_idx));
      phi = igvn.register_new_node_with_optimizer(phi, old_phi);
      igvn.replace_input_of(old_phi, LoopNode::EntryControl, phi);
      old_phi->del_req(outer_idx);
    }
  }
  _head = outer;
  phase->set_loop(_head, this);
}
static void fix_parent( IdealLoopTree *loop, IdealLoopTree *parent ) {
  loop->_parent = parent;
  if( loop->_child ) fix_parent( loop->_child, loop   );
  if( loop->_next  ) fix_parent( loop->_next , parent );
}
static float estimate_path_freq( Node *n ) {
  IfNode *iff;
  for( int i = 0; i < 50; i++ ) { // Skip through a bunch of uncommon tests
    uint nop = n->Opcode();
    if( nop == Op_SafePoint ) {   // Skip any safepoint
      n = n->in(0);
      continue;
    }
    if( nop == Op_CatchProj ) {   // Get count from a prior call
      assert( n->is_CatchProj(), "" );
      if( ((CatchProjNode*)n)->_con != CatchProjNode::fall_through_index )
        return 0.0f;            // Assume call exception path is rare
      Node *call = n->in(0)->in(0)->in(0);
      assert( call->is_Call(), "expect a call here" );
      const JVMState *jvms = ((CallNode*)call)->jvms();
      ciMethodData* methodData = jvms->method()->method_data();
      if (!methodData->is_mature())  return 0.0f; // No call-site data
      ciProfileData* data = methodData->bci_to_data(jvms->bci());
      if ((data == NULL) || !data->is_CounterData()) {
        n = n->in(0);
        continue;
      }
      return data->as_CounterData()->count()/FreqCountInvocations;
    }
    Node *n_c = n->in(0);
    if( !n_c->is_If() ) break;       // No estimate available
    iff = n_c->as_If();
    if( iff->_fcnt != COUNT_UNKNOWN )   // Have a valid count?
      return ((nop == Op_IfTrue) ? iff->_prob : 1.0f - iff->_prob) * iff->_fcnt;
    if( (nop == Op_IfTrue  && iff->_prob < PROB_LIKELY_MAG(5)) ||
        (nop == Op_IfFalse && iff->_prob > PROB_UNLIKELY_MAG(5)) )
      break;
    n = iff->in(0);
  }
  return 0.0f;                  // No estimate available
}
void IdealLoopTree::merge_many_backedges( PhaseIdealLoop *phase ) {
  uint i;
  float hotcnt = 0.0f;
  float warmcnt = 0.0f;
  uint hot_idx = 0;
  for( i = 2; i < _head->req(); i++ ) {
    float cnt = estimate_path_freq(_head->in(i));
    if( cnt > hotcnt ) {       // Grab hottest path
      warmcnt = hotcnt;
      hotcnt = cnt;
      hot_idx = i;
    } else if( cnt > warmcnt ) { // And 2nd hottest path
      warmcnt = cnt;
    }
  }
  if( hotcnt <= 0.0001 ||
      hotcnt < 2.0*warmcnt ) hot_idx = 0;// No hot backedge
  PhaseIterGVN &igvn = phase->_igvn;
  Node *hot_tail = NULL;
  Node *r = new (phase->C) RegionNode(1);
  for( i = 2; i < _head->req(); i++ ) {
    if( i != hot_idx )
      r->add_req( _head->in(i) );
    else hot_tail = _head->in(i);
  }
  igvn.register_new_node_with_optimizer(r, _head);
  while( _head->req() > 3 ) _head->del_req( _head->req()-1 );
  _head->set_req(2, r);
  if( hot_idx ) _head->add_req(hot_tail);
  for (DUIterator_Fast jmax, j = _head->fast_outs(jmax); j < jmax; j++) {
    Node *out = _head->fast_out(j);
    if( out->is_Phi() ) {
      PhiNode* n = out->as_Phi();
      igvn.hash_delete(n);      // Delete from hash before hacking edges
      Node *hot_phi = NULL;
      Node *phi = new (phase->C) PhiNode(r, n->type(), n->adr_type());
      uint j = 1;
      for( uint i = 2; i < n->req(); i++ ) {
        if( i != hot_idx )
          phi->set_req( j++, n->in(i) );
        else hot_phi = n->in(i);
      }
      igvn.register_new_node_with_optimizer(phi, n);
      while( n->req() > 3 ) n->del_req( n->req()-1 );
      n->set_req(2, phi);
      if( hot_idx ) n->add_req(hot_phi);
    }
  }
  IdealLoopTree *ilt = new IdealLoopTree( phase, _head, _tail );
  phase->set_loop(_tail,ilt);   // Adjust tail
  _tail = r;                    // Self's tail is new merge point
  phase->set_loop(r,this);
  ilt->_child = _child;         // New guy has my children
  _child = ilt;                 // Self has new guy as only child
  ilt->_parent = this;          // new guy has self for parent
  ilt->_nest = _nest;           // Same nesting depth (for now)
  IdealLoopTree **pilt = &_child;
  while( ilt ) {
    if( ilt->_head == _head ) {
      uint i;
      for( i = 2; i < _head->req(); i++ )
        if( _head->in(i) == ilt->_tail )
          break;                // Still a loop
      if( i == _head->req() ) { // No longer a loop
        IdealLoopTree **cp = &ilt->_child;
        while( *cp ) cp = &(*cp)->_next;   // Find end of child list
        ilt->_head = NULL;      // Flag as a loop UNIONED into parent
        ilt = ilt->_child;      // Repeat using new ilt
        continue;               // do not advance over ilt->_child
      }
      assert( ilt->_tail == hot_tail, "expected to only find the hot inner loop here" );
      phase->set_loop(_head,ilt);
    }
    pilt = &ilt->_child;        // Advance to next
    ilt = *pilt;
  }
  if( _child ) fix_parent( _child, this );
}
bool IdealLoopTree::beautify_loops( PhaseIdealLoop *phase ) {
  bool result = false;
  PhaseIterGVN &igvn = phase->_igvn;
  igvn.hash_delete(_head);      // Yank from hash before hacking edges
  int fall_in_cnt = 0;
  for( uint i = 1; i < _head->req(); i++ )
    if( !phase->is_member( this, _head->in(i) ) )
      fall_in_cnt++;
  assert( fall_in_cnt, "at least 1 fall-in path" );
  if( fall_in_cnt > 1 )         // Need a loop landing pad to merge fall-ins
    split_fall_in( phase, fall_in_cnt );
  fall_in_cnt = 1;
  while( phase->is_member( this, _head->in(fall_in_cnt) ) )
    fall_in_cnt++;
  if( fall_in_cnt > 1 ) {
    Node *tmp = _head->in(1);
    _head->set_req( 1, _head->in(fall_in_cnt) );
    _head->set_req( fall_in_cnt, tmp );
    for (DUIterator_Fast imax, i = _head->fast_outs(imax); i < imax; i++) {
      Node* phi = _head->fast_out(i);
      if( phi->is_Phi() ) {
        igvn.hash_delete(phi); // Yank from hash before hacking edges
        tmp = phi->in(1);
        phi->set_req( 1, phi->in(fall_in_cnt) );
        phi->set_req( fall_in_cnt, tmp );
      }
    }
  }
  assert( !phase->is_member( this, _head->in(1) ), "left edge is fall-in" );
  assert(  phase->is_member( this, _head->in(2) ), "right edge is loop" );
  if (_head->req() > 3) {
    if (!_irreducible) {
      merge_many_backedges( phase );
    }
    result = true;
  }
  if (_head->req() > 3 && !_irreducible) {
    split_outer_loop( phase );
    result = true;
  } else if (!_head->is_Loop() && !_irreducible) {
    Node *l = new (phase->C) LoopNode( _head->in(1), _head->in(2) );
    l = igvn.register_new_node_with_optimizer(l, _head);
    phase->set_created_loop_node();
    phase->_igvn.replace_node( _head, l );
    _head = l;
    phase->set_loop(_head, this);
  }
  if( _child ) result |= _child->beautify_loops( phase );
  if( _next  ) result |= _next ->beautify_loops( phase );
  return result;
}
void IdealLoopTree::allpaths_check_safepts(VectorSet &visited, Node_List &stack) {
  assert(stack.size() == 0, "empty stack");
  stack.push(_tail);
  visited.Clear();
  visited.set(_tail->_idx);
  while (stack.size() > 0) {
    Node* n = stack.pop();
    if (n->is_Call() && n->as_Call()->guaranteed_safepoint()) {
    } else if (n->Opcode() == Op_SafePoint) {
      if (_phase->get_loop(n) != this) {
        if (_required_safept == NULL) _required_safept = new Node_List();
        _required_safept->push(n);  // save the one closest to the tail
      }
    } else {
      uint start = n->is_Region() ? 1 : 0;
      uint end   = n->is_Region() && !n->is_Loop() ? n->req() : start + 1;
      for (uint i = start; i < end; i++) {
        Node* in = n->in(i);
        assert(in->is_CFG(), "must be");
        if (!visited.test_set(in->_idx) && is_member(_phase->get_loop(in))) {
          stack.push(in);
        }
      }
    }
  }
}
void IdealLoopTree::check_safepts(VectorSet &visited, Node_List &stack) {
  IdealLoopTree* ch = _child;
  if (_child) _child->check_safepts(visited, stack);
  if (_next)  _next ->check_safepts(visited, stack);
  if (!_head->is_CountedLoop() && !_has_sfpt && _parent != NULL && !_irreducible) {
    bool  has_call         = false; // call on dom-path
    bool  has_local_ncsfpt = false; // ncsfpt on dom-path at this loop depth
    Node* nonlocal_ncsfpt  = NULL;  // ncsfpt on dom-path at a deeper depth
    for (Node* n = tail(); n != _head; n = _phase->idom(n)) {
      if (n->is_Call() && n->as_Call()->guaranteed_safepoint()) {
        has_call = true;
        _has_sfpt = 1;          // Then no need for a safept!
        break;
      } else if (n->Opcode() == Op_SafePoint) {
        if (_phase->get_loop(n) == this) {
          has_local_ncsfpt = true;
          break;
        }
        if (nonlocal_ncsfpt == NULL) {
          nonlocal_ncsfpt = n; // save the one closest to the tail
        }
      } else {
        IdealLoopTree* nlpt = _phase->get_loop(n);
        if (this != nlpt) {
          assert(is_member(nlpt), "nested loop");
          Node* tail = nlpt->_tail;
          if (tail->in(0)->is_If()) tail = tail->in(0);
          if (n == tail) {
            if (nlpt->_has_sfpt) {
              has_call = true;
              _has_sfpt = 1;
              break;
            }
            assert(_phase->is_dominator(_head, nlpt->_head), "inner head dominated by outer head");
            n = nlpt->_head;
          }
        }
      }
    }
    if (_child != NULL && !has_call && !has_local_ncsfpt) {
      if (nonlocal_ncsfpt != NULL) {
        if (_required_safept == NULL) _required_safept = new Node_List();
        _required_safept->push(nonlocal_ncsfpt);
      } else {
        allpaths_check_safepts(visited, stack);
      }
    }
  }
}
bool PhaseIdealLoop::is_deleteable_safept(Node* sfpt) {
  assert(sfpt->Opcode() == Op_SafePoint, "");
  IdealLoopTree* lp = get_loop(sfpt)->_parent;
  while (lp != NULL) {
    Node_List* sfpts = lp->_required_safept;
    if (sfpts != NULL) {
      for (uint i = 0; i < sfpts->size(); i++) {
        if (sfpt == sfpts->at(i))
          return false;
      }
    }
    lp = lp->_parent;
  }
  return true;
}
void PhaseIdealLoop::replace_parallel_iv(IdealLoopTree *loop) {
  assert(loop->_head->is_CountedLoop(), "");
  CountedLoopNode *cl = loop->_head->as_CountedLoop();
  if (!cl->is_valid_counted_loop())
    return;         // skip malformed counted loop
  Node *incr = cl->incr();
  if (incr == NULL)
    return;         // Dead loop?
  Node *init = cl->init_trip();
  Node *phi  = cl->phi();
  int stride_con = cl->stride_con();
  for (DUIterator i = cl->outs(); cl->has_out(i); i++) {
    Node *out = cl->out(i);
    if (!out->is_Phi() || out == phi || !has_node(out))
      continue;
    PhiNode* phi2 = out->as_Phi();
    Node *incr2 = phi2->in( LoopNode::LoopBackControl );
    if (phi2->region() != loop->_head ||
        incr2->req() != 3 ||
        incr2->in(1) != phi2 ||
        incr2 == incr ||
        incr2->Opcode() != Op_AddI ||
        !incr2->in(2)->is_Con())
      continue;
    Node *init2 = phi2->in( LoopNode::EntryControl );
    int stride_con2 = incr2->in(2)->get_int();
    if (stride_con2 == min_jint && stride_con == -1) {
      continue;
    }
    int ratio_con = stride_con2/stride_con;
    if ((ratio_con * stride_con) == stride_con2) { // Check for exact
#ifndef PRODUCT
      if (TraceLoopOpts) {
        tty->print("Parallel IV: %d ", phi2->_idx);
        loop->dump_head();
      }
#endif
      Node* ratio = _igvn.intcon(ratio_con);
      set_ctrl(ratio, C->root());
      Node* ratio_init = new (C) MulINode(init, ratio);
      _igvn.register_new_node_with_optimizer(ratio_init, init);
      set_early_ctrl(ratio_init);
      Node* diff = new (C) SubINode(init2, ratio_init);
      _igvn.register_new_node_with_optimizer(diff, init2);
      set_early_ctrl(diff);
      Node* ratio_idx = new (C) MulINode(phi, ratio);
      _igvn.register_new_node_with_optimizer(ratio_idx, phi);
      set_ctrl(ratio_idx, cl);
      Node* add = new (C) AddINode(ratio_idx, diff);
      _igvn.register_new_node_with_optimizer(add);
      set_ctrl(add, cl);
      _igvn.replace_node( phi2, add );
      if (add->outcnt() == 0) {
        _igvn.remove_dead_node(add);
      }
      --i; // deleted this phi; rescan starting with next position
      continue;
    }
  }
}
void IdealLoopTree::remove_safepoints(PhaseIdealLoop* phase, bool keep_one) {
  Node* keep = NULL;
  if (keep_one) {
    for (Node* i = tail(); i != _head; i = phase->idom(i)) {
      if (i->Opcode() == Op_SafePoint && phase->get_loop(i) == this) {
        keep = i;
        break; // Found one
      }
    }
  }
  bool prune = !keep_one || keep != NULL;
  Node_List* sfpts = _safepts;
  if (prune && sfpts != NULL) {
    assert(keep == NULL || keep->Opcode() == Op_SafePoint, "not safepoint");
    for (uint i = 0; i < sfpts->size(); i++) {
      Node* n = sfpts->at(i);
      assert(phase->get_loop(n) == this, "");
      if (n != keep && phase->is_deleteable_safept(n)) {
        phase->lazy_replace(n, n->in(TypeFunc::Control));
      }
    }
  }
}
void IdealLoopTree::counted_loop( PhaseIdealLoop *phase ) {
  if (!_child) {
    if (_head->is_Loop()) _head->as_Loop()->set_inner_loop();
  }
  if (_head->is_CountedLoop() ||
      phase->is_counted_loop(_head, this)) {
    if (!UseCountedLoopSafepoints) {
      _has_sfpt = 1;
    }
    bool keep_one_sfpt = !(_has_call || _has_sfpt);
    remove_safepoints(phase, keep_one_sfpt);
    phase->replace_parallel_iv(this);
  } else if (_parent != NULL && !_irreducible) {
    bool keep_one_sfpt = true;
    remove_safepoints(phase, keep_one_sfpt);
  }
  if (_child) _child->counted_loop( phase );
  if (_next)  _next ->counted_loop( phase );
}
#ifndef PRODUCT
void IdealLoopTree::dump_head( ) const {
  for (uint i=0; i<_nest; i++)
    tty->print("  ");
  tty->print("Loop: N%d/N%d ",_head->_idx,_tail->_idx);
  if (_irreducible) tty->print(" IRREDUCIBLE");
  Node* entry = _head->in(LoopNode::EntryControl);
  if (LoopLimitCheck) {
    Node* predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
    if (predicate != NULL ) {
      tty->print(" limit_check");
      entry = entry->in(0)->in(0);
    }
  }
  if (UseLoopPredicate) {
    entry = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
    if (entry != NULL) {
      tty->print(" predicated");
    }
  }
  if (_head->is_CountedLoop()) {
    CountedLoopNode *cl = _head->as_CountedLoop();
    tty->print(" counted");
    Node* init_n = cl->init_trip();
    if (init_n  != NULL &&  init_n->is_Con())
      tty->print(" [%d,", cl->init_trip()->get_int());
    else
      tty->print(" [int,");
    Node* limit_n = cl->limit();
    if (limit_n  != NULL &&  limit_n->is_Con())
      tty->print("%d),", cl->limit()->get_int());
    else
      tty->print("int),");
    int stride_con  = cl->stride_con();
    if (stride_con > 0) tty->print("+");
    tty->print("%d", stride_con);
    tty->print(" (%d iters) ", (int)cl->profile_trip_cnt());
    if (cl->is_pre_loop ()) tty->print(" pre" );
    if (cl->is_main_loop()) tty->print(" main");
    if (cl->is_post_loop()) tty->print(" post");
  }
  if (_has_call) tty->print(" has_call");
  if (_has_sfpt) tty->print(" has_sfpt");
  if (_rce_candidate) tty->print(" rce");
  if (_safepts != NULL && _safepts->size() > 0) {
    tty->print(" sfpts={"); _safepts->dump_simple(); tty->print(" }");
  }
  if (_required_safept != NULL && _required_safept->size() > 0) {
    tty->print(" req={"); _required_safept->dump_simple(); tty->print(" }");
  }
  tty->cr();
}
void IdealLoopTree::dump( ) const {
  dump_head();
  if (_child) _child->dump();
  if (_next)  _next ->dump();
}
#endif
static void log_loop_tree(IdealLoopTree* root, IdealLoopTree* loop, CompileLog* log) {
  if (loop == root) {
    if (loop->_child != NULL) {
      log->begin_head("loop_tree");
      log->end_head();
      if( loop->_child ) log_loop_tree(root, loop->_child, log);
      log->tail("loop_tree");
      assert(loop->_next == NULL, "what?");
    }
  } else {
    Node* head = loop->_head;
    log->begin_head("loop");
    log->print(" idx='%d' ", head->_idx);
    if (loop->_irreducible) log->print("irreducible='1' ");
    if (head->is_Loop()) {
      if (head->as_Loop()->is_inner_loop()) log->print("inner_loop='1' ");
      if (head->as_Loop()->is_partial_peel_loop()) log->print("partial_peel_loop='1' ");
    }
    if (head->is_CountedLoop()) {
      CountedLoopNode* cl = head->as_CountedLoop();
      if (cl->is_pre_loop())  log->print("pre_loop='%d' ",  cl->main_idx());
      if (cl->is_main_loop()) log->print("main_loop='%d' ", cl->_idx);
      if (cl->is_post_loop()) log->print("post_loop='%d' ",  cl->main_idx());
    }
    log->end_head();
    if( loop->_child ) log_loop_tree(root, loop->_child, log);
    log->tail("loop");
    if( loop->_next  ) log_loop_tree(root, loop->_next, log);
  }
}
void PhaseIdealLoop::collect_potentially_useful_predicates(
                         IdealLoopTree * loop, Unique_Node_List &useful_predicates) {
  if (loop->_child) { // child
    collect_potentially_useful_predicates(loop->_child, useful_predicates);
  }
  if (loop->_head->is_Loop() &&
      !loop->_irreducible    &&
      !loop->tail()->is_top()) {
    LoopNode* lpn = loop->_head->as_Loop();
    Node* entry = lpn->in(LoopNode::EntryControl);
    Node* predicate_proj = find_predicate(entry); // loop_limit_check first
    if (predicate_proj != NULL ) { // right pattern that can be used by loop predication
      assert(entry->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1, "must be");
      useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one
      entry = entry->in(0)->in(0);
    }
    predicate_proj = find_predicate(entry); // Predicate
    if (predicate_proj != NULL ) {
      useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one
    }
  }
  if (loop->_next) { // sibling
    collect_potentially_useful_predicates(loop->_next, useful_predicates);
  }
}
void PhaseIdealLoop::eliminate_useless_predicates() {
  if (C->predicate_count() == 0)
    return; // no predicate left
  Unique_Node_List useful_predicates; // to store useful predicates
  if (C->has_loops()) {
    collect_potentially_useful_predicates(_ltree_root->_child, useful_predicates);
  }
  for (int i = C->predicate_count(); i > 0; i--) {
     Node * n = C->predicate_opaque1_node(i-1);
     assert(n->Opcode() == Op_Opaque1, "must be");
     if (!useful_predicates.member(n)) { // not in the useful list
       _igvn.replace_node(n, n->in(1));
     }
  }
}
bool PhaseIdealLoop::process_expensive_nodes() {
  assert(OptimizeExpensiveOps, "optimization off?");
  C->sort_expensive_nodes();
  bool progress = false;
  for (int i = 0; i < C->expensive_count(); ) {
    Node* n = C->expensive_node(i);
    int start = i;
    i++;
    for (; i < C->expensive_count() && Compile::cmp_expensive_nodes(n, C->expensive_node(i)) == 0; i++);
    int end = i;
    for (int j = start; j < end; j++) {
      Node* n1 = C->expensive_node(j);
      if (is_node_unreachable(n1)) {
        continue;
      }
      for (int k = j+1; k < end; k++) {
        Node* n2 = C->expensive_node(k);
        if (is_node_unreachable(n2)) {
          continue;
        }
        assert(n1 != n2, "should be pair of nodes");
        Node* c1 = n1->in(0);
        Node* c2 = n2->in(0);
        Node* parent_c1 = c1;
        Node* parent_c2 = c2;
        if (c1->is_Loop()) {
          parent_c1 = c1->in(1);
        }
        if (c2->is_Loop()) {
          parent_c2 = c2->in(1);
        }
        if (parent_c1 == parent_c2) {
          _igvn._worklist.push(n1);
          _igvn._worklist.push(n2);
          continue;
        }
        if (is_dominator(c1, c2)) {
          c2 = c1;
        } else if (is_dominator(c2, c1)) {
          c1 = c2;
        } else if (parent_c1->is_Proj() && parent_c1->in(0)->is_If() &&
                   parent_c2->is_Proj() && parent_c1->in(0) == parent_c2->in(0)) {
          c1 = c2 = idom(parent_c1->in(0));
        }
        if (n1->in(0) != c1) {
          _igvn.hash_delete(n1);
          n1->set_req(0, c1);
          _igvn.hash_insert(n1);
          _igvn._worklist.push(n1);
          progress = true;
        }
        if (n2->in(0) != c2) {
          _igvn.hash_delete(n2);
          n2->set_req(0, c2);
          _igvn.hash_insert(n2);
          _igvn._worklist.push(n2);
          progress = true;
        }
      }
    }
  }
  return progress;
}
void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool skip_loop_opts) {
  ResourceMark rm;
  int old_progress = C->major_progress();
  uint orig_worklist_size = _igvn._worklist.size();
  C->clear_major_progress();
#ifndef PRODUCT
  uint unique = C->unique();
  _loop_invokes++;
  _loop_work += unique;
#endif
  _has_irreducible_loops = false;
  _created_loop_node = false;
  Arena *a = Thread::current()->resource_area();
  VectorSet visited(a);
  _nodes.map(C->unique(), NULL);
  memset(_nodes.adr(), 0, wordSize * C->unique());
  _ltree_root = new IdealLoopTree( this, C->root(), C->root() );
  _ltree_root->_has_sfpt = 1;
  _idom_size = 0;
  _idom      = NULL;
  _dom_depth = NULL;
  _dom_stk   = NULL;
  allocate_preorders();
  build_loop_tree();
  if (C->failing()) {
    return;
  }
  if( !_ltree_root->_child && !_verify_only ) C->set_has_loops(false);
  if (!has_node(C->root())) {
    if (!_verify_only) {
      C->clear_major_progress();
      C->record_method_not_compilable("empty program detected during loop optimization");
    }
    return;
  }
  bool stop_early = !C->has_loops() && !skip_loop_opts && !do_split_ifs && !_verify_me && !_verify_only;
  bool do_expensive_nodes = C->should_optimize_expensive_nodes(_igvn);
  if (stop_early && !do_expensive_nodes) {
    _igvn.optimize();           // Cleanup NeverBranches
    return;
  }
  _ltree_root->set_nest( 0 );
  if( !_verify_me && !_verify_only && _ltree_root->_child ) {
    C->print_method(PHASE_BEFORE_BEAUTIFY_LOOPS, 3);
    if( _ltree_root->_child->beautify_loops( this ) ) {
      _ltree_root->_child = NULL;
      _nodes.clear();
      reallocate_preorders();
      build_loop_tree();
      if (C->failing()) {
        return;
      }
      _ltree_root->set_nest( 0 );
      C->print_method(PHASE_AFTER_BEAUTIFY_LOOPS, 3);
    }
  }
  _idom_size = C->unique();
  _idom      = NEW_RESOURCE_ARRAY( Node*, _idom_size );
  _dom_depth = NEW_RESOURCE_ARRAY( uint,  _idom_size );
  _dom_stk   = NULL; // Allocated on demand in recompute_dom_depth
  memset( _dom_depth, 0, _idom_size * sizeof(uint) );
  Dominators();
  if (!_verify_only) {
    for( uint i = 1; i < C->root()->req(); i++ ) {
      if( !_nodes[C->root()->in(i)->_idx] ) {    // Dead path into Root?
        _igvn.delete_input_of(C->root(), i);
        i--;                      // Rerun same iteration on compressed edges
      }
    }
    Node_List cisstack(a);
    _ltree_root->check_safepts(visited, cisstack);
  }
  int stack_size = (C->live_nodes() >> 1) + 16; // (live_nodes>>1)+16 from Java2D stats
  Node_Stack nstack( a, stack_size );
  visited.Clear();
  Node_List worklist(a);
  worklist.push( C->top() );
  visited.set( C->top()->_idx ); // Set C->top() as visited now
  build_loop_early( visited, worklist, nstack );
  if( !_verify_me && !_verify_only )
    _ltree_root->counted_loop( this );
  visited.Clear();
  init_dom_lca_tags();
  worklist.push( C->root() );
  NOT_PRODUCT( C->verify_graph_edges(); )
  worklist.push( C->top() );
  build_loop_late( visited, worklist, nstack );
  if (_verify_only) {
    for (int i = 0; i < old_progress; i++)
      C->set_major_progress();
    assert(C->unique() == unique, "verification mode made Nodes? ? ?");
    assert(_igvn._worklist.size() == orig_worklist_size, "shouldn't push anything");
    return;
  }
  while (_deadlist.size()) {
    _igvn.remove_globally_dead_node(_deadlist.pop());
  }
  if (stop_early) {
    assert(do_expensive_nodes, "why are we here?");
    if (process_expensive_nodes()) {
      C->set_major_progress();
    }
    _igvn.optimize();
    return;
  }
  if (UseLoopPredicate || LoopLimitCheck) {
    eliminate_useless_predicates();
  }
#ifndef PRODUCT
  C->verify_graph_edges();
  if (_verify_me) {             // Nested verify pass?
    assert(C->unique() == unique, "non-optimize mode made Nodes? ? ?");
    return;
  }
  if(VerifyLoopOptimizations) verify();
  if(TraceLoopOpts && C->has_loops()) {
    _ltree_root->dump();
  }
#endif
  if (skip_loop_opts) {
    for (int i = 0; i < old_progress; i++) {
      C->set_major_progress();
    }
    _igvn.optimize();
    if (C->log() != NULL) {
      log_loop_tree(_ltree_root, _ltree_root, C->log());
    }
    return;
  }
  if (ReassociateInvariants) {
    for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
      IdealLoopTree* lpt = iter.current();
      if (!lpt->is_counted() || !lpt->is_inner()) continue;
      lpt->reassociate_invariants(this);
      if (SplitIfBlocks && do_split_ifs) {
        if (lpt->policy_range_check(this)) {
          lpt->_rce_candidate = 1; // = true
        }
      }
    }
  }
  if( SplitIfBlocks && do_split_ifs ) {
    visited.Clear();
    split_if_with_blocks( visited, nstack );
    NOT_PRODUCT( if( VerifyLoopOptimizations ) verify(); );
  }
  if (!C->major_progress() && do_expensive_nodes && process_expensive_nodes()) {
    C->set_major_progress();
  }
  if (C->has_loops() && !C->major_progress() && (C->predicate_count() > 0)) {
    _ltree_root->_child->loop_predication(this);
  }
  if (OptimizeFill && UseLoopPredicate && C->has_loops() && !C->major_progress()) {
    if (do_intrinsify_fill()) {
      C->set_major_progress();
    }
  }
  if (C->has_loops() && !C->major_progress()) {
    memset( worklist.adr(), 0, worklist.Size()*sizeof(Node*) );
    _ltree_root->_child->iteration_split( this, worklist );
  }
  NOT_PRODUCT( C->verify_graph_edges(); );
  if (!do_split_ifs) {
    C->set_major_progress();
  }
  if (created_loop_node()) {
    C->set_major_progress();
  }
  if (!C->major_progress() && (C->predicate_count() > 0)) {
     C->cleanup_loop_predicates(_igvn);
#ifndef PRODUCT
     if (TraceLoopOpts) {
       tty->print_cr("PredicatesOff");
     }
#endif
     C->set_major_progress();
  }
  if (UseSuperWord && C->has_loops() && !C->major_progress()) {
    SuperWord sw(this);
    for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
      IdealLoopTree* lpt = iter.current();
      if (lpt->is_counted()) {
        sw.transform_loop(lpt);
      }
    }
  }
  _igvn.optimize();
  if (C->log() != NULL) {
    log_loop_tree(_ltree_root, _ltree_root, C->log());
  }
}
#ifndef PRODUCT
int PhaseIdealLoop::_loop_invokes=0;// Count of PhaseIdealLoop invokes
int PhaseIdealLoop::_loop_work=0; // Sum of PhaseIdealLoop x unique
void PhaseIdealLoop::print_statistics() {
  tty->print_cr("PhaseIdealLoop=%d, sum _unique=%d", _loop_invokes, _loop_work);
}
static int fail;                // debug only, so its multi-thread dont care
void PhaseIdealLoop::verify() const {
  int old_progress = C->major_progress();
  ResourceMark rm;
  PhaseIdealLoop loop_verify( _igvn, this );
  VectorSet visited(Thread::current()->resource_area());
  fail = 0;
  verify_compare( C->root(), &loop_verify, visited );
  assert( fail == 0, "verify loops failed" );
  _ltree_root->verify_tree(loop_verify._ltree_root, NULL);
  for( int i=0; i<old_progress; i++ )
    C->set_major_progress();
}
void PhaseIdealLoop::verify_compare( Node *n, const PhaseIdealLoop *loop_verify, VectorSet &visited ) const {
  if( !n ) return;
  if( visited.test_set( n->_idx ) ) return;
  if( !_nodes[n->_idx] ) {      // Unreachable
    assert( !loop_verify->_nodes[n->_idx], "both should be unreachable" );
    return;
  }
  uint i;
  for( i = 0; i < n->req(); i++ )
    verify_compare( n->in(i), loop_verify, visited );
  i = n->_idx;
  if( has_ctrl(n) ) {           // We have control; verify has loop or ctrl
    if( _nodes[i] != loop_verify->_nodes[i] &&
        get_ctrl_no_update(n) != loop_verify->get_ctrl_no_update(n) ) {
      tty->print("Mismatched control setting for: ");
      n->dump();
      if( fail++ > 10 ) return;
      Node *c = get_ctrl_no_update(n);
      tty->print("We have it as: ");
      if( c->in(0) ) c->dump();
        else tty->print_cr("N%d",c->_idx);
      tty->print("Verify thinks: ");
      if( loop_verify->has_ctrl(n) )
        loop_verify->get_ctrl_no_update(n)->dump();
      else
        loop_verify->get_loop_idx(n)->dump();
      tty->cr();
    }
  } else {                    // We have a loop
    IdealLoopTree *us = get_loop_idx(n);
    if( loop_verify->has_ctrl(n) ) {
      tty->print("Mismatched loop setting for: ");
      n->dump();
      if( fail++ > 10 ) return;
      tty->print("We have it as: ");
      us->dump();
      tty->print("Verify thinks: ");
      loop_verify->get_ctrl_no_update(n)->dump();
      tty->cr();
    } else if (!C->major_progress()) {
      IdealLoopTree *them = loop_verify->get_loop_idx(n);
      if( us->_head != them->_head ||  us->_tail != them->_tail ) {
        tty->print("Unequals loops for: ");
        n->dump();
        if( fail++ > 10 ) return;
        tty->print("We have it as: ");
        us->dump();
        tty->print("Verify thinks: ");
        them->dump();
        tty->cr();
      }
    }
  }
  if( i >= _idom_size ) {
    if( !n->is_CFG() ) return;
    tty->print("CFG Node with no idom: ");
    n->dump();
    return;
  }
  if( !n->is_CFG() ) return;
  if( n == C->root() ) return; // No IDOM here
  assert(n->_idx == i, "sanity");
  Node *id = idom_no_update(n);
  if( id != loop_verify->idom_no_update(n) ) {
    tty->print("Unequals idoms for: ");
    n->dump();
    if( fail++ > 10 ) return;
    tty->print("We have it as: ");
    id->dump();
    tty->print("Verify thinks: ");
    loop_verify->idom_no_update(n)->dump();
    tty->cr();
  }
}
void IdealLoopTree::verify_tree(IdealLoopTree *loop, const IdealLoopTree *parent) const {
  assert( _parent == parent, "Badly formed loop tree" );
  if( _head != loop->_head ) {
    IdealLoopTree **pp = &loop->_parent->_child;
    while( *pp != loop )
      pp = &((*pp)->_next);
    IdealLoopTree **nn = &loop->_next;
    while( (*nn) && (*nn)->_head != _head )
      nn = &((*nn)->_next);
    if( !(*nn) ) {
      if (_irreducible && Compile::current()->major_progress())  return;
      assert( 0, "failed to match loop tree" );
    }
    IdealLoopTree *hit = *nn;
    hit->_next = loop;
    loop = hit;
  }
  assert( _head  == loop->_head , "mismatched loop head" );
  Node *tail = _tail;           // Inline a non-updating version of
  while( !tail->in(0) )         // the 'tail()' call.
    tail = tail->in(1);
  assert( tail == loop->_tail, "mismatched loop tail" );
  if( _head->is_CountedLoop() && _head->as_CountedLoop()->is_main_loop() ) {
    CountedLoopNode *cl = _head->as_CountedLoop();
    Node *init = cl->init_trip();
    Node *ctrl = cl->in(LoopNode::EntryControl);
    assert( ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "" );
    Node *iff  = ctrl->in(0);
    assert( iff->Opcode() == Op_If, "" );
    Node *bol  = iff->in(1);
    assert( bol->Opcode() == Op_Bool, "" );
    Node *cmp  = bol->in(1);
    assert( cmp->Opcode() == Op_CmpI, "" );
    Node *add  = cmp->in(1);
    Node *opaq;
    if( add->Opcode() == Op_Opaque1 ) {
      opaq = add;
    } else {
      assert( add->Opcode() == Op_AddI || add->Opcode() == Op_ConI , "" );
      assert( add == init, "" );
      opaq = cmp->in(2);
    }
    assert( opaq->Opcode() == Op_Opaque1, "" );
  }

评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值