// Check loop body for tests of trip-counter plus loop-invariant vs
// loop-invariant.
for (uint i = 0; i < _body.size(); i++) {
Node *iff = _body[i];
if (iff->Opcode() == Op_If) { // Test?
// Comparing trip+off vs limit
Node *bol = iff->in(1);
if (bol->req() != 2) continue; // dead constant test
if (!bol->is_Bool()) {
assert(UseLoopPredicate && bol->Opcode() == Op_Conv2B, "predicate check only");
continue;
}
if (bol->as_Bool()->_test._test == BoolTest::ne)
continue; // not RC
Node *cmp = bol->in(1);
Node *rc_exp = cmp->in(1);
Node *limit = cmp->in(2);
Node *limit_c = phase->get_ctrl(limit);
if( limit_c == phase->C->top() )
return false; // Found dead test on live IF? No RCE!
if( is_member(phase->get_loop(limit_c) ) ) {
// Compare might have operands swapped; commute them
rc_exp = cmp->in(2);
limit = cmp->in(1);
limit_c = phase->get_ctrl(limit);
if( is_member(phase->get_loop(limit_c) ) )
continue; // Both inputs are loop varying; cannot RCE
}
if (!phase->is_scaled_iv_plus_offset(rc_exp, trip_counter, NULL, NULL)) {
continue;
}
// Yeah! Found a test like 'trip+off vs limit'
// Test is an IfNode, has 2 projections. If BOTH are in the loop
// we need loop unswitching instead of iteration splitting.
if( is_loop_exit(iff) )
return true; // Found reason to split iterations
} // End of is IF
}
return false;
}
//------------------------------policy_peel_only-------------------------------
// Return TRUE or FALSE if the loop should NEVER be RCE'd or aligned. Useful
// for unrolling loops with NO array accesses.
bool IdealLoopTree::policy_peel_only( PhaseIdealLoop *phase ) const {
for( uint i = 0; i < _body.size(); i++ )
if( _body[i]->is_Mem() )
return false;
// No memory accesses at all!
return true;
}
//------------------------------clone_up_backedge_goo--------------------------
// If Node n lives in the back_ctrl block and cannot float, we clone a private
// version of n in preheader_ctrl block and return that, otherwise return n.
Node *PhaseIdealLoop::clone_up_backedge_goo( Node *back_ctrl, Node *preheader_ctrl, Node *n, VectorSet &visited, Node_Stack &clones ) {
if( get_ctrl(n) != back_ctrl ) return n;
// Only visit once
if (visited.test_set(n->_idx)) {
Node *x = clones.find(n->_idx);
if (x != NULL)
return x;
return n;
}
Node *x = NULL; // If required, a clone of 'n'
// Check for 'n' being pinned in the backedge.
if( n->in(0) && n->in(0) == back_ctrl ) {
assert(clones.find(n->_idx) == NULL, "dead loop");
x = n->clone(); // Clone a copy of 'n' to preheader
clones.push(x, n->_idx);
x->set_req( 0, preheader_ctrl ); // Fix x's control input to preheader
}
// Recursive fixup any other input edges into x.
// If there are no changes we can just return 'n', otherwise
// we need to clone a private copy and change it.
for( uint i = 1; i < n->req(); i++ ) {
Node *g = clone_up_backedge_goo( back_ctrl, preheader_ctrl, n->in(i), visited, clones );
if( g != n->in(i) ) {
if( !x ) {
assert(clones.find(n->_idx) == NULL, "dead loop");
x = n->clone();
clones.push(x, n->_idx);
}
x->set_req(i, g);
}
}
if( x ) { // x can legally float to pre-header location
register_new_node( x, preheader_ctrl );
return x;
} else { // raise n to cover LCA of uses
set_ctrl( n, find_non_split_ctrl(back_ctrl->in(0)) );
}
return n;
}
bool PhaseIdealLoop::cast_incr_before_loop(Node* incr, Node* ctrl, Node* loop) {
Node* castii = new (C) CastIINode(incr, TypeInt::INT, true);
castii->set_req(0, ctrl);
register_new_node(castii, ctrl);
for (DUIterator_Fast imax, i = incr->fast_outs(imax); i < imax; i++) {
Node* n = incr->fast_out(i);
if (n->is_Phi() && n->in(0) == loop) {
int nrep = n->replace_edge(incr, castii);
return true;
}
}
return false;
}
//------------------------------insert_pre_post_loops--------------------------
// Insert pre and post loops. If peel_only is set, the pre-loop can not have
// more iterations added. It acts as a 'peel' only, no lower-bound RCE, no
// alignment. Useful to unroll loops that do no array accesses.
void PhaseIdealLoop::insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_new, bool peel_only ) {
#ifndef PRODUCT
if (TraceLoopOpts) {
if (peel_only)
tty->print("PeelMainPost ");
else
tty->print("PreMainPost ");
loop->dump_head();
}
#endif
C->set_major_progress();
// Find common pieces of the loop being guarded with pre & post loops
CountedLoopNode *main_head = loop->_head->as_CountedLoop();
assert( main_head->is_normal_loop(), "" );
CountedLoopEndNode *main_end = main_head->loopexit();
guarantee(main_end != NULL, "no loop exit node");
assert( main_end->outcnt() == 2, "1 true, 1 false path only" );
uint dd_main_head = dom_depth(main_head);
uint max = main_head->outcnt();
Node *pre_header= main_head->in(LoopNode::EntryControl);
Node *init = main_head->init_trip();
Node *incr = main_end ->incr();
Node *limit = main_end ->limit();
Node *stride = main_end ->stride();
Node *cmp = main_end ->cmp_node();
BoolTest::mask b_test = main_end->test_trip();
// Need only 1 user of 'bol' because I will be hacking the loop bounds.
Node *bol = main_end->in(CountedLoopEndNode::TestValue);
if( bol->outcnt() != 1 ) {
bol = bol->clone();
register_new_node(bol,main_end->in(CountedLoopEndNode::TestControl));
_igvn.hash_delete(main_end);
main_end->set_req(CountedLoopEndNode::TestValue, bol);
}
// Need only 1 user of 'cmp' because I will be hacking the loop bounds.
if( cmp->outcnt() != 1 ) {
cmp = cmp->clone();
register_new_node(cmp,main_end->in(CountedLoopEndNode::TestControl));
_igvn.hash_delete(bol);
bol->set_req(1, cmp);
}
//------------------------------
// Step A: Create Post-Loop.
Node* main_exit = main_end->proj_out(false);
assert( main_exit->Opcode() == Op_IfFalse, "" );
int dd_main_exit = dom_depth(main_exit);
// Step A1: Clone the loop body. The clone becomes the post-loop. The main
// loop pre-header illegally has 2 control users (old & new loops).
clone_loop( loop, old_new, dd_main_exit );
assert( old_new[main_end ->_idx]->Opcode() == Op_CountedLoopEnd, "" );
CountedLoopNode *post_head = old_new[main_head->_idx]->as_CountedLoop();
post_head->set_post_loop(main_head);
// Reduce the post-loop trip count.
CountedLoopEndNode* post_end = old_new[main_end ->_idx]->as_CountedLoopEnd();
post_end->_prob = PROB_FAIR;
// Build the main-loop normal exit.
IfFalseNode *new_main_exit = new (C) IfFalseNode(main_end);
_igvn.register_new_node_with_optimizer( new_main_exit );
set_idom(new_main_exit, main_end, dd_main_exit );
set_loop(new_main_exit, loop->_parent);
// Step A2: Build a zero-trip guard for the post-loop. After leaving the
// main-loop, the post-loop may not execute at all. We 'opaque' the incr
// (the main-loop trip-counter exit value) because we will be changing
// the exit value (via unrolling) so we cannot constant-fold away the zero
// trip guard until all unrolling is done.
Node *zer_opaq = new (C) Opaque1Node(C, incr);
Node *zer_cmp = new (C) CmpINode( zer_opaq, limit );
Node *zer_bol = new (C) BoolNode( zer_cmp, b_test );
register_new_node( zer_opaq, new_main_exit );
register_new_node( zer_cmp , new_main_exit );
register_new_node( zer_bol , new_main_exit );
// Build the IfNode
IfNode *zer_iff = new (C) IfNode( new_main_exit, zer_bol, PROB_FAIR, COUNT_UNKNOWN );
_igvn.register_new_node_with_optimizer( zer_iff );
set_idom(zer_iff, new_main_exit, dd_main_exit);
set_loop(zer_iff, loop->_parent);
// Plug in the false-path, taken if we need to skip post-loop
_igvn.replace_input_of(main_exit, 0, zer_iff);
set_idom(main_exit, zer_iff, dd_main_exit);
set_idom(main_exit->unique_out(), zer_iff, dd_main_exit);
// Make the true-path, must enter the post loop
Node *zer_taken = new (C) IfTrueNode( zer_iff );
_igvn.register_new_node_with_optimizer( zer_taken );
set_idom(zer_taken, zer_iff, dd_main_exit);
set_loop(zer_taken, loop->_parent);
// Plug in the true path
_igvn.hash_delete( post_head );
post_head->set_req(LoopNode::EntryControl, zer_taken);
set_idom(post_head, zer_taken, dd_main_exit);
Arena *a = Thread::current()->resource_area();
VectorSet visited(a);
Node_Stack clones(a, main_head->back_control()->outcnt());
// Step A3: Make the fall-in values to the post-loop come from the
// fall-out values of the main-loop.
for (DUIterator_Fast imax, i = main_head->fast_outs(imax); i < imax; i++) {
Node* main_phi = main_head->fast_out(i);
if( main_phi->is_Phi() && main_phi->in(0) == main_head && main_phi->outcnt() >0 ) {
Node *post_phi = old_new[main_phi->_idx];
Node *fallmain = clone_up_backedge_goo(main_head->back_control(),
post_head->init_control(),
main_phi->in(LoopNode::LoopBackControl),
visited, clones);
_igvn.hash_delete(post_phi);
post_phi->set_req( LoopNode::EntryControl, fallmain );
}
}
// Update local caches for next stanza
main_exit = new_main_exit;
//------------------------------
// Step B: Create Pre-Loop.
// Step B1: Clone the loop body. The clone becomes the pre-loop. The main
// loop pre-header illegally has 2 control users (old & new loops).
clone_loop( loop, old_new, dd_main_head );
CountedLoopNode* pre_head = old_new[main_head->_idx]->as_CountedLoop();
CountedLoopEndNode* pre_end = old_new[main_end ->_idx]->as_CountedLoopEnd();
pre_head->set_pre_loop(main_head);
Node *pre_incr = old_new[incr->_idx];
// Reduce the pre-loop trip count.
pre_end->_prob = PROB_FAIR;
// Find the pre-loop normal exit.
Node* pre_exit = pre_end->proj_out(false);
assert( pre_exit->Opcode() == Op_IfFalse, "" );
IfFalseNode *new_pre_exit = new (C) IfFalseNode(pre_end);
_igvn.register_new_node_with_optimizer( new_pre_exit );
set_idom(new_pre_exit, pre_end, dd_main_head);
set_loop(new_pre_exit, loop->_parent);
// Step B2: Build a zero-trip guard for the main-loop. After leaving the
// pre-loop, the main-loop may not execute at all. Later in life this
// zero-trip guard will become the minimum-trip guard when we unroll
// the main-loop.
Node *min_opaq = new (C) Opaque1Node(C, limit);
Node *min_cmp = new (C) CmpINode( pre_incr, min_opaq );
Node *min_bol = new (C) BoolNode( min_cmp, b_test );
register_new_node( min_opaq, new_pre_exit );
register_new_node( min_cmp , new_pre_exit );
register_new_node( min_bol , new_pre_exit );
// Build the IfNode (assume the main-loop is executed always).
IfNode *min_iff = new (C) IfNode( new_pre_exit, min_bol, PROB_ALWAYS, COUNT_UNKNOWN );
_igvn.register_new_node_with_optimizer( min_iff );
set_idom(min_iff, new_pre_exit, dd_main_head);
set_loop(min_iff, loop->_parent);
// Plug in the false-path, taken if we need to skip main-loop
_igvn.hash_delete( pre_exit );
pre_exit->set_req(0, min_iff);
set_idom(pre_exit, min_iff, dd_main_head);
set_idom(pre_exit->unique_out(), min_iff, dd_main_head);
// Make the true-path, must enter the main loop
Node *min_taken = new (C) IfTrueNode( min_iff );
_igvn.register_new_node_with_optimizer( min_taken );
set_idom(min_taken, min_iff, dd_main_head);
set_loop(min_taken, loop->_parent);
// Plug in the true path
_igvn.hash_delete( main_head );
main_head->set_req(LoopNode::EntryControl, min_taken);
set_idom(main_head, min_taken, dd_main_head);
visited.Clear();
clones.clear();
// Step B3: Make the fall-in values to the main-loop come from the
// fall-out values of the pre-loop.
for (DUIterator_Fast i2max, i2 = main_head->fast_outs(i2max); i2 < i2max; i2++) {
Node* main_phi = main_head->fast_out(i2);
if( main_phi->is_Phi() && main_phi->in(0) == main_head && main_phi->outcnt() > 0 ) {
Node *pre_phi = old_new[main_phi->_idx];
Node *fallpre = clone_up_backedge_goo(pre_head->back_control(),
main_head->init_control(),
pre_phi->in(LoopNode::LoopBackControl),
visited, clones);
_igvn.hash_delete(main_phi);
main_phi->set_req( LoopNode::EntryControl, fallpre );
}
}
// Nodes inside the loop may be control dependent on a predicate
// that was moved before the preloop. If the back branch of the main
// or post loops becomes dead, those nodes won't be dependent on the
// test that guards that loop nest anymore which could lead to an
// incorrect array access because it executes independently of the
// test that was guarding the loop nest. We add a special CastII on
// the if branch that enters the loop, between the input induction
// variable value and the induction variable Phi to preserve correct
// dependencies.
// CastII for the post loop:
bool inserted = cast_incr_before_loop(zer_opaq->in(1), zer_taken, post_head);
assert(inserted, "no castII inserted");
// CastII for the main loop:
inserted = cast_incr_before_loop(pre_incr, min_taken, main_head);
assert(inserted, "no castII inserted");
// Step B4: Shorten the pre-loop to run only 1 iteration (for now).
// RCE and alignment may change this later.
Node *cmp_end = pre_end->cmp_node();
assert( cmp_end->in(2) == limit, "" );
Node *pre_limit = new (C) AddINode( init, stride );
// Save the original loop limit in this Opaque1 node for
// use by range check elimination.
Node *pre_opaq = new (C) Opaque1Node(C, pre_limit, limit);
register_new_node( pre_limit, pre_head->in(0) );
register_new_node( pre_opaq , pre_head->in(0) );
// Since no other users of pre-loop compare, I can hack limit directly
assert( cmp_end->outcnt() == 1, "no other users" );
_igvn.hash_delete(cmp_end);
cmp_end->set_req(2, peel_only ? pre_limit : pre_opaq);
// Special case for not-equal loop bounds:
// Change pre loop test, main loop test, and the
// main loop guard test to use lt or gt depending on stride
// direction:
// positive stride use <
// negative stride use >
//
// not-equal test is kept for post loop to handle case
// when init > limit when stride > 0 (and reverse).
if (pre_end->in(CountedLoopEndNode::TestValue)->as_Bool()->_test._test == BoolTest::ne) {
BoolTest::mask new_test = (main_end->stride_con() > 0) ? BoolTest::lt : BoolTest::gt;
// Modify pre loop end condition
Node* pre_bol = pre_end->in(CountedLoopEndNode::TestValue)->as_Bool();
BoolNode* new_bol0 = new (C) BoolNode(pre_bol->in(1), new_test);
register_new_node( new_bol0, pre_head->in(0) );
_igvn.hash_delete(pre_end);
pre_end->set_req(CountedLoopEndNode::TestValue, new_bol0);
// Modify main loop guard condition
assert(min_iff->in(CountedLoopEndNode::TestValue) == min_bol, "guard okay");
BoolNode* new_bol1 = new (C) BoolNode(min_bol->in(1), new_test);
register_new_node( new_bol1, new_pre_exit );
_igvn.hash_delete(min_iff);
min_iff->set_req(CountedLoopEndNode::TestValue, new_bol1);
// Modify main loop end condition
BoolNode* main_bol = main_end->in(CountedLoopEndNode::TestValue)->as_Bool();
BoolNode* new_bol2 = new (C) BoolNode(main_bol->in(1), new_test);
register_new_node( new_bol2, main_end->in(CountedLoopEndNode::TestControl) );
_igvn.hash_delete(main_end);
main_end->set_req(CountedLoopEndNode::TestValue, new_bol2);
}
// Flag main loop
main_head->set_main_loop();
if( peel_only ) main_head->set_main_no_pre_loop();
// Subtract a trip count for the pre-loop.
main_head->set_trip_count(main_head->trip_count() - 1);
// It's difficult to be precise about the trip-counts
// for the pre/post loops. They are usually very short,
// so guess that 4 trips is a reasonable value.
post_head->set_profile_trip_cnt(4.0);
pre_head->set_profile_trip_cnt(4.0);
// Now force out all loop-invariant dominating tests. The optimizer
// finds some, but we _know_ they are all useless.
peeled_dom_test_elim(loop,old_new);
loop->record_for_igvn();
}
//------------------------------is_invariant-----------------------------
// Return true if n is invariant
bool IdealLoopTree::is_invariant(Node* n) const {
Node *n_c = _phase->has_ctrl(n) ? _phase->get_ctrl(n) : n;
if (n_c->is_top()) return false;
return !is_member(_phase->get_loop(n_c));
}
//------------------------------do_unroll--------------------------------------
// Unroll the loop body one step - make each trip do 2 iterations.
void PhaseIdealLoop::do_unroll( IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip ) {
assert(LoopUnrollLimit, "");
CountedLoopNode *loop_head = loop->_head->as_CountedLoop();
CountedLoopEndNode *loop_end = loop_head->loopexit();
assert(loop_end, "");
#ifndef PRODUCT
if (PrintOpto && VerifyLoopOptimizations) {
tty->print("Unrolling ");
loop->dump_head();
} else if (TraceLoopOpts) {
if (loop_head->trip_count() < (uint)LoopUnrollLimit) {
tty->print("Unroll %d(%2d) ", loop_head->unrolled_count()*2, loop_head->trip_count());
} else {
tty->print("Unroll %d ", loop_head->unrolled_count()*2);
}
loop->dump_head();
}
#endif
// Remember loop node count before unrolling to detect
// if rounds of unroll,optimize are making progress
loop_head->set_node_count_before_unroll(loop->_body.size());
Node *ctrl = loop_head->in(LoopNode::EntryControl);
Node *limit = loop_head->limit();
Node *init = loop_head->init_trip();
Node *stride = loop_head->stride();
Node *opaq = NULL;
if (adjust_min_trip) { // If not maximally unrolling, need adjustment
// Search for zero-trip guard.
// Check the shape of the graph at the loop entry. If an inappropriate
// graph shape is encountered, the compiler bails out loop unrolling;
// compilation of the method will still succeed.
if (!is_canonical_main_loop_entry(loop_head)) {
return;
}
opaq = ctrl->in(0)->in(1)->in(1)->in(2);
// Zero-trip test uses an 'opaque' node which is not shared.
assert(opaq->outcnt() == 1 && opaq->in(1) == limit, "");
}
C->set_major_progress();
Node* new_limit = NULL;
if (UnrollLimitCheck) {
int stride_con = stride->get_int();
int stride_p = (stride_con > 0) ? stride_con : -stride_con;
uint old_trip_count = loop_head->trip_count();
// Verify that unroll policy result is still valid.
assert(old_trip_count > 1 &&
(!adjust_min_trip || stride_p <= (1<<3)*loop_head->unrolled_count()), "sanity");
// Adjust loop limit to keep valid iterations number after unroll.
// Use (limit - stride) instead of (((limit - init)/stride) & (-2))*stride
// which may overflow.
if (!adjust_min_trip) {
assert(old_trip_count > 1 && (old_trip_count & 1) == 0,
"odd trip count for maximally unroll");
// Don't need to adjust limit for maximally unroll since trip count is even.
} else if (loop_head->has_exact_trip_count() && init->is_Con()) {
// Loop's limit is constant. Loop's init could be constant when pre-loop
// become peeled iteration.
jlong init_con = init->get_int();
// We can keep old loop limit if iterations count stays the same:
// old_trip_count == new_trip_count * 2
// Note: since old_trip_count >= 2 then new_trip_count >= 1
// so we also don't need to adjust zero trip test.
jlong limit_con = limit->get_int();
// (stride_con*2) not overflow since stride_con <= 8.
int new_stride_con = stride_con * 2;
int stride_m = new_stride_con - (stride_con > 0 ? 1 : -1);
jlong trip_count = (limit_con - init_con + stride_m)/new_stride_con;
// New trip count should satisfy next conditions.
assert(trip_count > 0 && (julong)trip_count < (julong)max_juint/2, "sanity");
uint new_trip_count = (uint)trip_count;
adjust_min_trip = (old_trip_count != new_trip_count*2);
}
if (adjust_min_trip) {
// Step 2: Adjust the trip limit if it is called for.
// The adjustment amount is -stride. Need to make sure if the
// adjustment underflows or overflows, then the main loop is skipped.
Node* cmp = loop_end->cmp_node();
assert(cmp->in(2) == limit, "sanity");
assert(opaq != NULL && opaq->in(1) == limit, "sanity");
// Verify that policy_unroll result is still valid.
const TypeInt* limit_type = _igvn.type(limit)->is_int();
assert(stride_con > 0 && ((limit_type->_hi - stride_con) < limit_type->_hi) ||
stride_con < 0 && ((limit_type->_lo - stride_con) > limit_type->_lo), "sanity");
if (limit->is_Con()) {
// The check in policy_unroll and the assert above guarantee
// no underflow if limit is constant.
new_limit = _igvn.intcon(limit->get_int() - stride_con);
set_ctrl(new_limit, C->root());
} else {
// Limit is not constant.
if (loop_head->unrolled_count() == 1) { // only for first unroll
// Separate limit by Opaque node in case it is an incremented
// variable from previous loop to avoid using pre-incremented
// value which could increase register pressure.
// Otherwise reorg_offsets() optimization will create a separate
// Opaque node for each use of trip-counter and as result
// zero trip guard limit will be different from loop limit.
assert(has_ctrl(opaq), "should have it");
Node* opaq_ctrl = get_ctrl(opaq);
limit = new (C) Opaque2Node( C, limit );
register_new_node( limit, opaq_ctrl );
}
if (stride_con > 0 && (java_subtract(limit_type->_lo, stride_con) < limit_type->_lo) ||
stride_con < 0 && (java_subtract(limit_type->_hi, stride_con) > limit_type->_hi)) {
// No underflow.
new_limit = new (C) SubINode(limit, stride);
} else {
// (limit - stride) may underflow.
// Clamp the adjustment value with MININT or MAXINT:
//
// new_limit = limit-stride
// if (stride > 0)
// new_limit = (limit < new_limit) ? MININT : new_limit;
// else
// new_limit = (limit > new_limit) ? MAXINT : new_limit;
//
BoolTest::mask bt = loop_end->test_trip();
assert(bt == BoolTest::lt || bt == BoolTest::gt, "canonical test is expected");
Node* adj_max = _igvn.intcon((stride_con > 0) ? min_jint : max_jint);
set_ctrl(adj_max, C->root());
Node* old_limit = NULL;
Node* adj_limit = NULL;
Node* bol = limit->is_CMove() ? limit->in(CMoveNode::Condition) : NULL;
if (loop_head->unrolled_count() > 1 &&
limit->is_CMove() && limit->Opcode() == Op_CMoveI &&
limit->in(CMoveNode::IfTrue) == adj_max &&
bol->as_Bool()->_test._test == bt &&
bol->in(1)->Opcode() == Op_CmpI &&
bol->in(1)->in(2) == limit->in(CMoveNode::IfFalse)) {
// Loop was unrolled before.
// Optimize the limit to avoid nested CMove:
// use original limit as old limit.
old_limit = bol->in(1)->in(1);
// Adjust previous adjusted limit.
adj_limit = limit->in(CMoveNode::IfFalse);
adj_limit = new (C) SubINode(adj_limit, stride);
} else {
old_limit = limit;
adj_limit = new (C) SubINode(limit, stride);
}
assert(old_limit != NULL && adj_limit != NULL, "");
register_new_node( adj_limit, ctrl ); // adjust amount
Node* adj_cmp = new (C) CmpINode(old_limit, adj_limit);
register_new_node( adj_cmp, ctrl );
Node* adj_bool = new (C) BoolNode(adj_cmp, bt);
register_new_node( adj_bool, ctrl );
new_limit = new (C) CMoveINode(adj_bool, adj_limit, adj_max, TypeInt::INT);
}
register_new_node(new_limit, ctrl);
}
assert(new_limit != NULL, "");
// Replace in loop test.
assert(loop_end->in(1)->in(1) == cmp, "sanity");
if (cmp->outcnt() == 1 && loop_end->in(1)->outcnt() == 1) {
// Don't need to create new test since only one user.
_igvn.hash_delete(cmp);
cmp->set_req(2, new_limit);
} else {
// Create new test since it is shared.
Node* ctrl2 = loop_end->in(0);
Node* cmp2 = cmp->clone();
cmp2->set_req(2, new_limit);
register_new_node(cmp2, ctrl2);
Node* bol2 = loop_end->in(1)->clone();
bol2->set_req(1, cmp2);
register_new_node(bol2, ctrl2);
_igvn.hash_delete(loop_end);
loop_end->set_req(1, bol2);
}
// Step 3: Find the min-trip test guaranteed before a 'main' loop.
// Make it a 1-trip test (means at least 2 trips).
// Guard test uses an 'opaque' node which is not shared. Hence I
// can edit it's inputs directly. Hammer in the new limit for the
// minimum-trip guard.
assert(opaq->outcnt() == 1, "");
_igvn.hash_delete(opaq);
opaq->set_req(1, new_limit);
}
// Adjust max trip count. The trip count is intentionally rounded
// down here (e.g. 15-> 7-> 3-> 1) because if we unwittingly over-unroll,
// the main, unrolled, part of the loop will never execute as it is protected
// by the min-trip test. See bug 4834191 for a case where we over-unrolled
// and later determined that part of the unrolled loop was dead.
loop_head->set_trip_count(old_trip_count / 2);
// Double the count of original iterations in the unrolled loop body.
loop_head->double_unrolled_count();
} else { // LoopLimitCheck
// Adjust max trip count. The trip count is intentionally rounded
// down here (e.g. 15-> 7-> 3-> 1) because if we unwittingly over-unroll,
// the main, unrolled, part of the loop will never execute as it is protected
// by the min-trip test. See bug 4834191 for a case where we over-unrolled
// and later determined that part of the unrolled loop was dead.
loop_head->set_trip_count(loop_head->trip_count() / 2);
// Double the count of original iterations in the unrolled loop body.
loop_head->double_unrolled_count();
// -----------
// Step 2: Cut back the trip counter for an unroll amount of 2.
// Loop will normally trip (limit - init)/stride_con. Since it's a
// CountedLoop this is exact (stride divides limit-init exactly).
// We are going to double the loop body, so we want to knock off any
// odd iteration: (trip_cnt & ~1). Then back compute a new limit.
Node *span = new (C) SubINode( limit, init );
register_new_node( span, ctrl );
Node *trip = new (C) DivINode( 0, span, stride );
register_new_node( trip, ctrl );
Node *mtwo = _igvn.intcon(-2);
set_ctrl(mtwo, C->root());
Node *rond = new (C) AndINode( trip, mtwo );
register_new_node( rond, ctrl );
Node *spn2 = new (C) MulINode( rond, stride );
register_new_node( spn2, ctrl );
new_limit = new (C) AddINode( spn2, init );
register_new_node( new_limit, ctrl );
// Hammer in the new limit
Node *ctrl2 = loop_end->in(0);
Node *cmp2 = new (C) CmpINode( loop_head->incr(), new_limit );
register_new_node( cmp2, ctrl2 );
Node *bol2 = new (C) BoolNode( cmp2, loop_end->test_trip() );
register_new_node( bol2, ctrl2 );
_igvn.hash_delete(loop_end);
loop_end->set_req(CountedLoopEndNode::TestValue, bol2);
// Step 3: Find the min-trip test guaranteed before a 'main' loop.
// Make it a 1-trip test (means at least 2 trips).
if( adjust_min_trip ) {
assert( new_limit != NULL, "" );
// Guard test uses an 'opaque' node which is not shared. Hence I
// can edit it's inputs directly. Hammer in the new limit for the
// minimum-trip guard.
assert( opaq->outcnt() == 1, "" );
_igvn.hash_delete(opaq);
opaq->set_req(1, new_limit);
}
} // LoopLimitCheck
// ---------
// Step 4: Clone the loop body. Move it inside the loop. This loop body
// represents the odd iterations; since the loop trips an even number of
// times its backedge is never taken. Kill the backedge.
uint dd = dom_depth(loop_head);
clone_loop( loop, old_new, dd );
// Make backedges of the clone equal to backedges of the original.
// Make the fall-in from the original come from the fall-out of the clone.
for (DUIterator_Fast jmax, j = loop_head->fast_outs(jmax); j < jmax; j++) {
Node* phi = loop_head->fast_out(j);
if( phi->is_Phi() && phi->in(0) == loop_head && phi->outcnt() > 0 ) {
Node *newphi = old_new[phi->_idx];
_igvn.hash_delete( phi );
_igvn.hash_delete( newphi );
phi ->set_req(LoopNode:: EntryControl, newphi->in(LoopNode::LoopBackControl));
newphi->set_req(LoopNode::LoopBackControl, phi ->in(LoopNode::LoopBackControl));
phi ->set_req(LoopNode::LoopBackControl, C->top());
}
}
Node *clone_head = old_new[loop_head->_idx];
_igvn.hash_delete( clone_head );
loop_head ->set_req(LoopNode:: EntryControl, clone_head->in(LoopNode::LoopBackControl));
clone_head->set_req(LoopNode::LoopBackControl, loop_head ->in(LoopNode::LoopBackControl));
loop_head ->set_req(LoopNode::LoopBackControl, C->top());
loop->_head = clone_head; // New loop header
set_idom(loop_head, loop_head ->in(LoopNode::EntryControl), dd);
set_idom(clone_head, clone_head->in(LoopNode::EntryControl), dd);
// Kill the clone's backedge
Node *newcle = old_new[loop_end->_idx];
_igvn.hash_delete( newcle );
Node *one = _igvn.intcon(1);
set_ctrl(one, C->root());
newcle->set_req(1, one);
// Force clone into same loop body
uint max = loop->_body.size();
for( uint k = 0; k < max; k++ ) {
Node *old = loop->_body.at(k);
Node *nnn = old_new[old->_idx];
loop->_body.push(nnn);
if (!has_ctrl(old))
set_loop(nnn, loop);
}
loop->record_for_igvn();
}
//------------------------------do_maximally_unroll----------------------------
void PhaseIdealLoop::do_maximally_unroll( IdealLoopTree *loop, Node_List &old_new ) {
CountedLoopNode *cl = loop->_head->as_CountedLoop();
assert(cl->has_exact_trip_count(), "trip count is not exact");
assert(cl->trip_count() > 0, "");
#ifndef PRODUCT
if (TraceLoopOpts) {
tty->print("MaxUnroll %d ", cl->trip_count());
loop->dump_head();
}
#endif
// If loop is tripping an odd number of times, peel odd iteration
if ((cl->trip_count() & 1) == 1) {
do_peeling(loop, old_new);
}
// Now its tripping an even number of times remaining. Double loop body.
// Do not adjust pre-guards; they are not needed and do not exist.
if (cl->trip_count() > 0) {
assert((cl->trip_count() & 1) == 0, "missed peeling");
do_unroll(loop, old_new, false);
}
}
//------------------------------dominates_backedge---------------------------------
// Returns true if ctrl is executed on every complete iteration
bool IdealLoopTree::dominates_backedge(Node* ctrl) {
assert(ctrl->is_CFG(), "must be control");
Node* backedge = _head->as_Loop()->in(LoopNode::LoopBackControl);
return _phase->dom_lca_internal(ctrl, backedge) == ctrl;
}
//------------------------------adjust_limit-----------------------------------
// Helper function that computes new loop limit as (rc_limit-offset)/scale
Node* PhaseIdealLoop::adjust_limit(bool is_positive_stride, Node* scale, Node* offset, Node* rc_limit, Node* old_limit, Node* pre_ctrl, bool round) {
Node* sub = new (C) SubLNode(rc_limit, offset);
register_new_node(sub, pre_ctrl);
Node* limit = new (C) DivLNode(NULL, sub, scale);
register_new_node(limit, pre_ctrl);
// When the absolute value of scale is greater than one, the division
// may round limit down/up, so add/sub one to/from the limit.
if (round) {
limit = new (C) AddLNode(limit, _igvn.longcon(is_positive_stride ? -1 : 1));
register_new_node(limit, pre_ctrl);
}
// Clamp the limit to handle integer under-/overflows.
// When reducing the limit, clamp to [min_jint, old_limit]:
// MIN(old_limit, MAX(limit, min_jint))
// When increasing the limit, clamp to [old_limit, max_jint]:
// MAX(old_limit, MIN(limit, max_jint))
Node* cmp = new (C) CmpLNode(limit, _igvn.longcon(is_positive_stride ? min_jint : max_jint));
register_new_node(cmp, pre_ctrl);
Node* bol = new (C) BoolNode(cmp, is_positive_stride ? BoolTest::lt : BoolTest::gt);
register_new_node(bol, pre_ctrl);
limit = new (C) ConvL2INode(limit);
register_new_node(limit, pre_ctrl);
limit = new (C) CMoveINode(bol, limit, _igvn.intcon(is_positive_stride ? min_jint : max_jint), TypeInt::INT);
register_new_node(limit, pre_ctrl);
limit = is_positive_stride ? (Node*)(new (C) MinINode(old_limit, limit))
: (Node*)(new (C) MaxINode(old_limit, limit));
register_new_node(limit, pre_ctrl);
return limit;
}
//------------------------------add_constraint---------------------------------
// Constrain the main loop iterations so the conditions:
// low_limit <= scale_con*I + offset < upper_limit
// always hold true. That is, either increase the number of iterations in the
// pre-loop or reduce the number of iterations in the main-loop until the condition
// holds true in the main-loop. Stride, scale, offset and limit are all loop
// invariant. Further, stride and scale are constants (offset and limit often are).
void PhaseIdealLoop::add_constraint(jlong stride_con, jlong scale_con, Node* offset, Node* low_limit, Node* upper_limit, Node* pre_ctrl, Node** pre_limit, Node** main_limit) {
assert(_igvn.type(offset)->isa_long() != NULL && _igvn.type(low_limit)->isa_long() != NULL &&
_igvn.type(upper_limit)->isa_long() != NULL, "arguments should be long values");
// For a positive stride, we need to reduce the main-loop limit and
// increase the pre-loop limit. This is reversed for a negative stride.
bool is_positive_stride = (stride_con > 0);
// If the absolute scale value is greater one, division in 'adjust_limit' may require
// rounding. Make sure the ABS method correctly handles min_jint.
// Only do this for the pre-loop, one less iteration of the main loop doesn't hurt.
bool round = ABS(scale_con) > 1;
Node* scale = _igvn.longcon(scale_con);
set_ctrl(scale, C->root());
if ((stride_con^scale_con) >= 0) { // Use XOR to avoid overflow
// Positive stride*scale: the affine function is increasing,
// the pre-loop checks for underflow and the post-loop for overflow.
// The overflow limit: scale*I+offset < upper_limit
// For the main-loop limit compute:
// ( if (scale > 0) /* and stride > 0 */
// I < (upper_limit-offset)/scale
// else /* scale < 0 and stride < 0 */
// I > (upper_limit-offset)/scale
// )
*main_limit = adjust_limit(is_positive_stride, scale, offset, upper_limit, *main_limit, pre_ctrl, false);
// The underflow limit: low_limit <= scale*I+offset
// For the pre-loop limit compute:
// NOT(scale*I+offset >= low_limit)
// scale*I+offset < low_limit
// ( if (scale > 0) /* and stride > 0 */
// I < (low_limit-offset)/scale
// else /* scale < 0 and stride < 0 */
// I > (low_limit-offset)/scale
// )
*pre_limit = adjust_limit(!is_positive_stride, scale, offset, low_limit, *pre_limit, pre_ctrl, round);
} else {
// Negative stride*scale: the affine function is decreasing,
// the pre-loop checks for overflow and the post-loop for underflow.
// The overflow limit: scale*I+offset < upper_limit
// For the pre-loop limit compute:
// NOT(scale*I+offset < upper_limit)
// scale*I+offset >= upper_limit
// scale*I+offset+1 > upper_limit
// ( if (scale < 0) /* and stride > 0 */
// I < (upper_limit-(offset+1))/scale
// else /* scale > 0 and stride < 0 */
// I > (upper_limit-(offset+1))/scale
// )
Node* one = _igvn.longcon(1);
set_ctrl(one, C->root());
Node* plus_one = new (C) AddLNode(offset, one);
register_new_node( plus_one, pre_ctrl );
*pre_limit = adjust_limit(!is_positive_stride, scale, plus_one, upper_limit, *pre_limit, pre_ctrl, round);
// The underflow limit: low_limit <= scale*I+offset
// For the main-loop limit compute:
// scale*I+offset+1 > low_limit
// ( if (scale < 0) /* and stride > 0 */
// I < (low_limit-(offset+1))/scale
// else /* scale > 0 and stride < 0 */
// I > (low_limit-(offset+1))/scale
// )
*main_limit = adjust_limit(is_positive_stride, scale, plus_one, low_limit, *main_limit, pre_ctrl, false);
}
}
//------------------------------is_scaled_iv---------------------------------
// Return true if exp is a constant times an induction var
bool PhaseIdealLoop::is_scaled_iv(Node* exp, Node* iv, int* p_scale) {
if (exp == iv) {
if (p_scale != NULL) {
*p_scale = 1;
}
return true;
}
int opc = exp->Opcode();
if (opc == Op_MulI) {
if (exp->in(1) == iv && exp->in(2)->is_Con()) {
if (p_scale != NULL) {
*p_scale = exp->in(2)->get_int();
}
return true;
}
if (exp->in(2) == iv && exp->in(1)->is_Con()) {
if (p_scale != NULL) {
*p_scale = exp->in(1)->get_int();
}
return true;
}
} else if (opc == Op_LShiftI) {
if (exp->in(1) == iv && exp->in(2)->is_Con()) {
if (p_scale != NULL) {
*p_scale = 1 << exp->in(2)->get_int();
}
return true;
}
}
return false;
}
//-----------------------------is_scaled_iv_plus_offset------------------------------
// Return true if exp is a simple induction variable expression: k1*iv + (invar + k2)
bool PhaseIdealLoop::is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset, int depth) {
if (is_scaled_iv(exp, iv, p_scale)) {
if (p_offset != NULL) {
Node *zero = _igvn.intcon(0);
set_ctrl(zero, C->root());
*p_offset = zero;
}
return true;
}
int opc = exp->Opcode();
if (opc == Op_AddI) {
if (is_scaled_iv(exp->in(1), iv, p_scale)) {
if (p_offset != NULL) {
*p_offset = exp->in(2);
}
return true;
}
if (is_scaled_iv(exp->in(2), iv, p_scale)) {
if (p_offset != NULL) {
*p_offset = exp->in(1);
}
return true;
}
if (exp->in(2)->is_Con()) {
Node* offset2 = NULL;
if (depth < 2 &&
is_scaled_iv_plus_offset(exp->in(1), iv, p_scale,
p_offset != NULL ? &offset2 : NULL, depth+1)) {
if (p_offset != NULL) {
Node *ctrl_off2 = get_ctrl(offset2);
Node* offset = new (C) AddINode(offset2, exp->in(2));
register_new_node(offset, ctrl_off2);
*p_offset = offset;
}
return true;
}
}
} else if (opc == Op_SubI) {
if (is_scaled_iv(exp->in(1), iv, p_scale)) {
if (p_offset != NULL) {
Node *zero = _igvn.intcon(0);
set_ctrl(zero, C->root());
Node *ctrl_off = get_ctrl(exp->in(2));
Node* offset = new (C) SubINode(zero, exp->in(2));
register_new_node(offset, ctrl_off);
*p_offset = offset;
}
return true;
}
if (is_scaled_iv(exp->in(2), iv, p_scale)) {
if (p_offset != NULL) {
*p_scale *= -1;
*p_offset = exp->in(1);
}
return true;
}
}
return false;
}
//------------------------------do_range_check---------------------------------
// Eliminate range-checks and other trip-counter vs loop-invariant tests.
void PhaseIdealLoop::do_range_check( IdealLoopTree *loop, Node_List &old_new ) {
#ifndef PRODUCT
if (PrintOpto && VerifyLoopOptimizations) {
tty->print("Range Check Elimination ");
loop->dump_head();
} else if (TraceLoopOpts) {
tty->print("RangeCheck ");
loop->dump_head();
}
#endif
assert(RangeCheckElimination, "");
CountedLoopNode *cl = loop->_head->as_CountedLoop();
// protect against stride not being a constant
if (!cl->stride_is_con())
return;
// Find the trip counter; we are iteration splitting based on it
Node *trip_counter = cl->phi();
// Find the main loop limit; we will trim it's iterations
// to not ever trip end tests
Node *main_limit = cl->limit();
// Check graph shape. Cannot optimize a loop if zero-trip
// Opaque1 node is optimized away and then another round
// of loop opts attempted.
if (!is_canonical_main_loop_entry(cl)) {
return;
}
// Need to find the main-loop zero-trip guard
Node *ctrl = cl->in(LoopNode::EntryControl);
Node *iffm = ctrl->in(0);
Node *opqzm = iffm->in(1)->in(1)->in(2);
assert(opqzm->in(1) == main_limit, "do not understand situation");
// Find the pre-loop limit; we will expand it's iterations to
// not ever trip low tests.
Node *p_f = iffm->in(0);
// pre loop may have been optimized out
if (p_f->Opcode() != Op_IfFalse) {
return;
}
CountedLoopEndNode *pre_end = p_f->in(0)->as_CountedLoopEnd();
assert(pre_end->loopnode()->is_pre_loop(), "");
Node *pre_opaq1 = pre_end->limit();
// Occasionally it's possible for a pre-loop Opaque1 node to be
// optimized away and then another round of loop opts attempted.
// We can not optimize this particular loop in that case.
if (pre_opaq1->Opcode() != Op_Opaque1)
return;
Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
Node *pre_limit = pre_opaq->in(1);
// Where do we put new limit calculations
Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl);
// Ensure the original loop limit is available from the
// pre-loop Opaque1 node.
Node *orig_limit = pre_opaq->original_loop_limit();
if (orig_limit == NULL || _igvn.type(orig_limit) == Type::TOP)
return;
// Must know if its a count-up or count-down loop
int stride_con = cl->stride_con();
Node* zero = _igvn.longcon(0);
Node* one = _igvn.longcon(1);
// Use symmetrical int range [-max_jint,max_jint]
Node* mini = _igvn.longcon(-max_jint);
set_ctrl(zero, C->root());
set_ctrl(one, C->root());
set_ctrl(mini, C->root());
// Check loop body for tests of trip-counter plus loop-invariant vs
// loop-invariant.
for( uint i = 0; i < loop->_body.size(); i++ ) {
Node *iff = loop->_body[i];
if( iff->Opcode() == Op_If ) { // Test?
// Test is an IfNode, has 2 projections. If BOTH are in the loop
// we need loop unswitching instead of iteration splitting.
Node *exit = loop->is_loop_exit(iff);
if( !exit ) continue;
int flip = (exit->Opcode() == Op_IfTrue) ? 1 : 0;
// Get boolean condition to test
Node *i1 = iff->in(1);
if( !i1->is_Bool() ) continue;
BoolNode *bol = i1->as_Bool();
BoolTest b_test = bol->_test;
// Flip sense of test if exit condition is flipped
if( flip )
b_test = b_test.negate();
// Get compare
Node *cmp = bol->in(1);
// Look for trip_counter + offset vs limit
Node *rc_exp = cmp->in(1);
Node *limit = cmp->in(2);
jint scale_con= 1; // Assume trip counter not scaled
Node *limit_c = get_ctrl(limit);
if( loop->is_member(get_loop(limit_c) ) ) {
// Compare might have operands swapped; commute them
b_test = b_test.commute();
rc_exp = cmp->in(2);
limit = cmp->in(1);
limit_c = get_ctrl(limit);
if( loop->is_member(get_loop(limit_c) ) )
continue; // Both inputs are loop varying; cannot RCE
}
// Here we know 'limit' is loop invariant
// 'limit' maybe pinned below the zero trip test (probably from a
// previous round of rce), in which case, it can't be used in the
// zero trip test expression which must occur before the zero test's if.
if( limit_c == ctrl ) {
continue; // Don't rce this check but continue looking for other candidates.
}
// Check for scaled induction variable plus an offset
Node *offset = NULL;
if (!is_scaled_iv_plus_offset(rc_exp, trip_counter, &scale_con, &offset)) {
continue;
}
Node *offset_c = get_ctrl(offset);
if( loop->is_member( get_loop(offset_c) ) )
continue; // Offset is not really loop invariant
// Here we know 'offset' is loop invariant.
// As above for the 'limit', the 'offset' maybe pinned below the
// zero trip test.
if( offset_c == ctrl ) {
continue; // Don't rce this check but continue looking for other candidates.
}
#ifdef ASSERT
if (TraceRangeLimitCheck) {
tty->print_cr("RC bool node%s", flip ? " flipped:" : ":");
bol->dump(2);
}
#endif
// At this point we have the expression as:
// scale_con * trip_counter + offset :: limit
// where scale_con, offset and limit are loop invariant. Trip_counter
// monotonically increases by stride_con, a constant. Both (or either)
// stride_con and scale_con can be negative which will flip about the
// sense of the test.
// Perform the limit computations in jlong to avoid overflow
jlong lscale_con = scale_con;
Node* int_offset = offset;
offset = new (C) ConvI2LNode(offset);
register_new_node(offset, pre_ctrl);
Node* int_limit = limit;
limit = new (C) ConvI2LNode(limit);
register_new_node(limit, pre_ctrl);
// Adjust pre and main loop limits to guard the correct iteration set
if( cmp->Opcode() == Op_CmpU ) {// Unsigned compare is really 2 tests
if( b_test._test == BoolTest::lt ) { // Range checks always use lt
// The underflow and overflow limits: 0 <= scale*I+offset < limit
add_constraint(stride_con, lscale_con, offset, zero, limit, pre_ctrl, &pre_limit, &main_limit);
} else {
#ifndef PRODUCT
if( PrintOpto )
tty->print_cr("missed RCE opportunity");
#endif
continue; // In release mode, ignore it
}
} else { // Otherwise work on normal compares
switch( b_test._test ) {
case BoolTest::gt:
// Fall into GE case
case BoolTest::ge:
// Convert (I*scale+offset) >= Limit to (I*(-scale)+(-offset)) <= -Limit
lscale_con = -lscale_con;
offset = new (C) SubLNode(zero, offset);
register_new_node( offset, pre_ctrl );
limit = new (C) SubLNode(zero, limit);
register_new_node( limit, pre_ctrl );
// Fall into LE case
case BoolTest::le:
if (b_test._test != BoolTest::gt) {
// Convert X <= Y to X < Y+1
limit = new (C) AddLNode(limit, one);
register_new_node( limit, pre_ctrl );
}
// Fall into LT case
case BoolTest::lt:
// The underflow and overflow limits: MIN_INT <= scale*I+offset < limit
// Note: (MIN_INT+1 == -MAX_INT) is used instead of MIN_INT here
// to avoid problem with scale == -1: MIN_INT/(-1) == MIN_INT.
add_constraint(stride_con, lscale_con, offset, mini, limit, pre_ctrl, &pre_limit, &main_limit);
break;
default:
#ifndef PRODUCT
if( PrintOpto )
tty->print_cr("missed RCE opportunity");
#endif
continue; // Unhandled case
}
}
// Kill the eliminated test
C->set_major_progress();
Node *kill_con = _igvn.intcon( 1-flip );
set_ctrl(kill_con, C->root());
_igvn.replace_input_of(iff, 1, kill_con);
// Find surviving projection
assert(iff->is_If(), "");
ProjNode* dp = ((IfNode*)iff)->proj_out(1-flip);
// Find loads off the surviving projection; remove their control edge
for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) {
Node* cd = dp->fast_out(i); // Control-dependent node
if (cd->is_Load() && cd->depends_only_on_test()) { // Loads can now float around in the loop
// Allow the load to float around in the loop, or before it
// but NOT before the pre-loop.
_igvn.replace_input_of(cd, 0, ctrl); // ctrl, not NULL
--i;
--imax;
}
}
} // End of is IF
}
// Update loop limits
if (pre_limit != orig_limit) {
// Computed pre-loop limit can be outside of loop iterations range.
pre_limit = (stride_con > 0) ? (Node*)new (C) MinINode(pre_limit, orig_limit)
: (Node*)new (C) MaxINode(pre_limit, orig_limit);
register_new_node(pre_limit, pre_ctrl);
}
_igvn.hash_delete(pre_opaq);
pre_opaq->set_req(1, pre_limit);
// Note:: we are making the main loop limit no longer precise;
// need to round up based on stride.
cl->set_nonexact_trip_count();
if (!LoopLimitCheck && stride_con != 1 && stride_con != -1) { // Cutout for common case
// "Standard" round-up logic: ([main_limit-init+(y-1)]/y)*y+init
// Hopefully, compiler will optimize for powers of 2.
Node *ctrl = get_ctrl(main_limit);
Node *stride = cl->stride();
Node *init = cl->init_trip();
Node *span = new (C) SubINode(main_limit,init);
register_new_node(span,ctrl);
Node *rndup = _igvn.intcon(stride_con + ((stride_con>0)?-1:1));
Node *add = new (C) AddINode(span,rndup);
register_new_node(add,ctrl);
Node *div = new (C) DivINode(0,add,stride);
register_new_node(div,ctrl);
Node *mul = new (C) MulINode(div,stride);
register_new_node(mul,ctrl);
Node *newlim = new (C) AddINode(mul,init);
register_new_node(newlim,ctrl);
main_limit = newlim;
}
Node *main_cle = cl->loopexit();
Node *main_bol = main_cle->in(1);
// Hacking loop bounds; need private copies of exit test
if( main_bol->outcnt() > 1 ) {// BoolNode shared?
_igvn.hash_delete(main_cle);
main_bol = main_bol->clone();// Clone a private BoolNode
register_new_node( main_bol, main_cle->in(0) );
main_cle->set_req(1,main_bol);
}
Node *main_cmp = main_bol->in(1);
if( main_cmp->outcnt() > 1 ) { // CmpNode shared?
_igvn.hash_delete(main_bol);
main_cmp = main_cmp->clone();// Clone a private CmpNode
register_new_node( main_cmp, main_cle->in(0) );
main_bol->set_req(1,main_cmp);
}
// Hack the now-private loop bounds
_igvn.replace_input_of(main_cmp, 2, main_limit);
// The OpaqueNode is unshared by design
assert( opqzm->outcnt() == 1, "cannot hack shared node" );
_igvn.replace_input_of(opqzm, 1, main_limit);
}
//------------------------------DCE_loop_body----------------------------------
// Remove simplistic dead code from loop body
void IdealLoopTree::DCE_loop_body() {
for( uint i = 0; i < _body.size(); i++ )
if( _body.at(i)->outcnt() == 0 )
_body.map( i--, _body.pop() );
}
//------------------------------adjust_loop_exit_prob--------------------------
// Look for loop-exit tests with the 50/50 (or worse) guesses from the parsing stage.
// Replace with a 1-in-10 exit guess.
void IdealLoopTree::adjust_loop_exit_prob( PhaseIdealLoop *phase ) {
Node *test = tail();
while( test != _head ) {
uint top = test->Opcode();
if( top == Op_IfTrue || top == Op_IfFalse ) {
int test_con = ((ProjNode*)test)->_con;
assert(top == (uint)(test_con? Op_IfTrue: Op_IfFalse), "sanity");
IfNode *iff = test->in(0)->as_If();
if( iff->outcnt() == 2 ) { // Ignore dead tests
Node *bol = iff->in(1);
if( bol && bol->req() > 1 && bol->in(1) &&
((bol->in(1)->Opcode() == Op_StorePConditional ) ||
(bol->in(1)->Opcode() == Op_StoreIConditional ) ||
(bol->in(1)->Opcode() == Op_StoreLConditional ) ||
(bol->in(1)->Opcode() == Op_CompareAndSwapI ) ||
(bol->in(1)->Opcode() == Op_CompareAndSwapL ) ||
(bol->in(1)->Opcode() == Op_CompareAndSwapP ) ||
(bol->in(1)->Opcode() == Op_CompareAndSwapN )))
return; // Allocation loops RARELY take backedge
// Find the OTHER exit path from the IF
Node* ex = iff->proj_out(1-test_con);
float p = iff->_prob;
if( !phase->is_member( this, ex ) && iff->_fcnt == COUNT_UNKNOWN ) {
if( top == Op_IfTrue ) {
if( p < (PROB_FAIR + PROB_UNLIKELY_MAG(3))) {
iff->_prob = PROB_STATIC_FREQUENT;
}
} else {
if( p > (PROB_FAIR - PROB_UNLIKELY_MAG(3))) {
iff->_prob = PROB_STATIC_INFREQUENT;
}
}
}
}
}
test = phase->idom(test);
}
}
//------------------------------policy_do_remove_empty_loop--------------------
// Micro-benchmark spamming. Policy is to always remove empty loops.
// The 'DO' part is to replace the trip counter with the value it will
// have on the last iteration. This will break the loop.
bool IdealLoopTree::policy_do_remove_empty_loop( PhaseIdealLoop *phase ) {
// Minimum size must be empty loop
if (_body.size() > EMPTY_LOOP_SIZE)
return false;
if (!_head->is_CountedLoop())
return false; // Dead loop
CountedLoopNode *cl = _head->as_CountedLoop();
if (!cl->is_valid_counted_loop())
return false; // Malformed loop
if (!phase->is_member(this, phase->get_ctrl(cl->loopexit()->in(CountedLoopEndNode::TestValue))))
return false; // Infinite loop
#ifdef ASSERT
// Ensure only one phi which is the iv.
Node* iv = NULL;
for (DUIterator_Fast imax, i = cl->fast_outs(imax); i < imax; i++) {
Node* n = cl->fast_out(i);
if (n->Opcode() == Op_Phi) {
assert(iv == NULL, "Too many phis" );
iv = n;
}
}
assert(iv == cl->phi(), "Wrong phi" );
#endif
// main and post loops have explicitly created zero trip guard
bool needs_guard = !cl->is_main_loop() && !cl->is_post_loop();
if (needs_guard) {
// Skip guard if values not overlap.
const TypeInt* init_t = phase->_igvn.type(cl->init_trip())->is_int();
const TypeInt* limit_t = phase->_igvn.type(cl->limit())->is_int();
int stride_con = cl->stride_con();
if (stride_con > 0) {
needs_guard = (init_t->_hi >= limit_t->_lo);
} else {
needs_guard = (init_t->_lo <= limit_t->_hi);
}
}
if (needs_guard) {
// Check for an obvious zero trip guard.
Node* inctrl = PhaseIdealLoop::skip_loop_predicates(cl->in(LoopNode::EntryControl));
if (inctrl->Opcode() == Op_IfTrue) {
// The test should look like just the backedge of a CountedLoop
Node* iff = inctrl->in(0);
if (iff->is_If()) {
Node* bol = iff->in(1);
if (bol->is_Bool() && bol->as_Bool()->_test._test == cl->loopexit()->test_trip()) {
Node* cmp = bol->in(1);
if (cmp->is_Cmp() && cmp->in(1) == cl->init_trip() && cmp->in(2) == cl->limit()) {
needs_guard = false;
}
}
}
}
}
#ifndef PRODUCT
if (PrintOpto) {
tty->print("Removing empty loop with%s zero trip guard", needs_guard ? "out" : "");
this->dump_head();
} else if (TraceLoopOpts) {
tty->print("Empty with%s zero trip guard ", needs_guard ? "out" : "");
this->dump_head();
}
#endif
if (needs_guard) {
// Peel the loop to ensure there's a zero trip guard
Node_List old_new;
phase->do_peeling(this, old_new);
}
// Replace the phi at loop head with the final value of the last
// iteration. Then the CountedLoopEnd will collapse (backedge never
// taken) and all loop-invariant uses of the exit values will be correct.
Node *phi = cl->phi();
Node *exact_limit = phase->exact_limit(this);
if (exact_limit != cl->limit()) {
// We also need to replace the original limit to collapse loop exit.
Node* cmp = cl->loopexit()->cmp_node();
assert(cl->limit() == cmp->in(2), "sanity");
// Duplicate cmp node if it has other users
if (cmp->outcnt() > 1) {
cmp = cmp->clone();
cmp = phase->_igvn.register_new_node_with_optimizer(cmp);
BoolNode *bol = cl->loopexit()->in(CountedLoopEndNode::TestValue)->as_Bool();
phase->_igvn.replace_input_of(bol, 1, cmp); // put bol on worklist
}
phase->_igvn._worklist.push(cmp->in(2)); // put limit on worklist
phase->_igvn.replace_input_of(cmp, 2, exact_limit); // put cmp on worklist
}
// Note: the final value after increment should not overflow since
// counted loop has limit check predicate.
Node *final = new (phase->C) SubINode( exact_limit, cl->stride() );
phase->register_new_node(final,cl->in(LoopNode::EntryControl));
phase->_igvn.replace_node(phi,final);
phase->C->set_major_progress();
return true;
}
//------------------------------policy_do_one_iteration_loop-------------------
// Convert one iteration loop into normal code.
bool IdealLoopTree::policy_do_one_iteration_loop( PhaseIdealLoop *phase ) {
if (!_head->as_Loop()->is_valid_counted_loop())
return false; // Only for counted loop
CountedLoopNode *cl = _head->as_CountedLoop();
if (!cl->has_exact_trip_count() || cl->trip_count() != 1) {
return false;
}
#ifndef PRODUCT
if(TraceLoopOpts) {
tty->print("OneIteration ");
this->dump_head();
}
#endif
Node *init_n = cl->init_trip();
#ifdef ASSERT
// Loop boundaries should be constant since trip count is exact.
assert(init_n->get_int() + cl->stride_con() >= cl->limit()->get_int(), "should be one iteration");
#endif
// Replace the phi at loop head with the value of the init_trip.
// Then the CountedLoopEnd will collapse (backedge will not be taken)
// and all loop-invariant uses of the exit values will be correct.
phase->_igvn.replace_node(cl->phi(), cl->init_trip());
phase->C->set_major_progress();
return true;
}
//=============================================================================
//------------------------------iteration_split_impl---------------------------
bool IdealLoopTree::iteration_split_impl( PhaseIdealLoop *phase, Node_List &old_new ) {
// Compute exact loop trip count if possible.
compute_exact_trip_count(phase);
// Convert one iteration loop into normal code.
if (policy_do_one_iteration_loop(phase))
return true;
// Check and remove empty loops (spam micro-benchmarks)
if (policy_do_remove_empty_loop(phase))
return true; // Here we removed an empty loop
bool should_peel = policy_peeling(phase); // Should we peel?
bool should_unswitch = policy_unswitching(phase);
// Non-counted loops may be peeled; exactly 1 iteration is peeled.
// This removes loop-invariant tests (usually null checks).
if (!_head->is_CountedLoop()) { // Non-counted loop
if (PartialPeelLoop && phase->partial_peel(this, old_new)) {
// Partial peel succeeded so terminate this round of loop opts
return false;
}
if (should_peel) { // Should we peel?
#ifndef PRODUCT
if (PrintOpto) tty->print_cr("should_peel");
#endif
phase->do_peeling(this,old_new);
} else if (should_unswitch) {
phase->do_unswitching(this, old_new);
}
return true;
}
CountedLoopNode *cl = _head->as_CountedLoop();
if (!cl->is_valid_counted_loop()) return true; // Ignore various kinds of broken loops
// Do nothing special to pre- and post- loops
if (cl->is_pre_loop() || cl->is_post_loop()) return true;
// Compute loop trip count from profile data
compute_profile_trip_cnt(phase);
// Before attempting fancy unrolling, RCE or alignment, see if we want
// to completely unroll this loop or do loop unswitching.
if (cl->is_normal_loop()) {
if (should_unswitch) {
phase->do_unswitching(this, old_new);
return true;
}
bool should_maximally_unroll = policy_maximally_unroll(phase);
if (should_maximally_unroll) {
// Here we did some unrolling and peeling. Eventually we will
// completely unroll this loop and it will no longer be a loop.
phase->do_maximally_unroll(this,old_new);
return true;
}
}
// Skip next optimizations if running low on nodes. Note that
// policy_unswitching and policy_maximally_unroll have this check.
int nodes_left = phase->C->max_node_limit() - phase->C->live_nodes();
if ((int)(2 * _body.size()) > nodes_left) {
return true;
}
// Counted loops may be peeled, may need some iterations run up
// front for RCE, and may want to align loop refs to a cache
// line. Thus we clone a full loop up front whose trip count is
// at least 1 (if peeling), but may be several more.
// The main loop will start cache-line aligned with at least 1
// iteration of the unrolled body (zero-trip test required) and
// will have some range checks removed.
// A post-loop will finish any odd iterations (leftover after
// unrolling), plus any needed for RCE purposes.
bool should_unroll = policy_unroll(phase);
bool should_rce = policy_range_check(phase);
bool should_align = policy_align(phase);
// If not RCE'ing (iteration splitting) or Aligning, then we do not
// need a pre-loop. We may still need to peel an initial iteration but
// we will not be needing an unknown number of pre-iterations.
//
// Basically, if may_rce_align reports FALSE first time through,
// we will not be able to later do RCE or Aligning on this loop.
bool may_rce_align = !policy_peel_only(phase) || should_rce || should_align;
// If we have any of these conditions (RCE, alignment, unrolling) met, then
// we switch to the pre-/main-/post-loop model. This model also covers
// peeling.
if (should_rce || should_align || should_unroll) {
if (cl->is_normal_loop()) // Convert to 'pre/main/post' loops
phase->insert_pre_post_loops(this,old_new, !may_rce_align);
// Adjust the pre- and main-loop limits to let the pre and post loops run
// with full checks, but the main-loop with no checks. Remove said
// checks from the main body.
if (should_rce)
phase->do_range_check(this,old_new);
// Double loop body for unrolling. Adjust the minimum-trip test (will do
// twice as many iterations as before) and the main body limit (only do
// an even number of trips). If we are peeling, we might enable some RCE
// and we'd rather unroll the post-RCE'd loop SO... do not unroll if
// peeling.
if (should_unroll && !should_peel)
phase->do_unroll(this,old_new, true);
// Adjust the pre-loop limits to align the main body
// iterations.
if (should_align)
Unimplemented();
} else { // Else we have an unchanged counted loop
if (should_peel) // Might want to peel but do nothing else
phase->do_peeling(this,old_new);
}
return true;
}
//=============================================================================
//------------------------------iteration_split--------------------------------
bool IdealLoopTree::iteration_split( PhaseIdealLoop *phase, Node_List &old_new ) {
// Recursively iteration split nested loops
if (_child && !_child->iteration_split(phase, old_new))
return false;
// Clean out prior deadwood
DCE_loop_body();
// Look for loop-exit tests with my 50/50 guesses from the Parsing stage.
// Replace with a 1-in-10 exit guess.
if (_parent /*not the root loop*/ &&
!_irreducible &&
// Also ignore the occasional dead backedge
!tail()->is_top()) {
adjust_loop_exit_prob(phase);
}
// Gate unrolling, RCE and peeling efforts.
if (!_child && // If not an inner loop, do not split
!_irreducible &&
_allow_optimizations &&
!tail()->is_top()) { // Also ignore the occasional dead backedge
if (!_has_call) {
if (!iteration_split_impl(phase, old_new)) {
return false;
}
} else if (policy_unswitching(phase)) {
phase->do_unswitching(this, old_new);
}
}
// Minor offset re-organization to remove loop-fallout uses of
// trip counter when there was no major reshaping.
phase->reorg_offsets(this);
if (_next && !_next->iteration_split(phase, old_new))
return false;
return true;
}
//=============================================================================
// Process all the loops in the loop tree and replace any fill
// patterns with an intrinsic version.
bool PhaseIdealLoop::do_intrinsify_fill() {
bool changed = false;
for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
IdealLoopTree* lpt = iter.current();
changed |= intrinsify_fill(lpt);
}
return changed;
}
// Examine an inner loop looking for a a single store of an invariant
// value in a unit stride loop,
bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& store_value,
Node*& shift, Node*& con) {
const char* msg = NULL;
Node* msg_node = NULL;
store_value = NULL;
con = NULL;
shift = NULL;
// Process the loop looking for stores. If there are multiple
// stores or extra control flow give at this point.
CountedLoopNode* head = lpt->_head->as_CountedLoop();
for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
Node* n = lpt->_body.at(i);
if (n->outcnt() == 0) continue; // Ignore dead
if (n->is_Store()) {
if (store != NULL) {
msg = "multiple stores";
break;
}
int opc = n->Opcode();
if (opc == Op_StoreP || opc == Op_StoreN || opc == Op_StoreNKlass || opc == Op_StoreCM) {
msg = "oop fills not handled";
break;
}
Node* value = n->in(MemNode::ValueIn);
if (!lpt->is_invariant(value)) {
msg = "variant store value";
} else if (!_igvn.type(n->in(MemNode::Address))->isa_aryptr()) {
msg = "not array address";
}
store = n;
store_value = value;
} else if (n->is_If() && n != head->loopexit()) {
msg = "extra control flow";
msg_node = n;
}
}
if (store == NULL) {
// No store in loop
return false;
}
if (msg == NULL && head->stride_con() != 1) {
// could handle negative strides too
if (head->stride_con() < 0) {
msg = "negative stride";
} else {
msg = "non-unit stride";
}
}
if (msg == NULL && !store->in(MemNode::Address)->is_AddP()) {
msg = "can't handle store address";
msg_node = store->in(MemNode::Address);
}
if (msg == NULL &&
(!store->in(MemNode::Memory)->is_Phi() ||
store->in(MemNode::Memory)->in(LoopNode::LoopBackControl) != store)) {
msg = "store memory isn't proper phi";
msg_node = store->in(MemNode::Memory);
}
// Make sure there is an appropriate fill routine
BasicType t = store->as_Mem()->memory_type();
const char* fill_name;
if (msg == NULL &&
StubRoutines::select_fill_function(t, false, fill_name) == NULL) {
msg = "unsupported store";
msg_node = store;
}
if (msg != NULL) {
#ifndef PRODUCT
if (TraceOptimizeFill) {
tty->print_cr("not fill intrinsic candidate: %s", msg);
if (msg_node != NULL) msg_node->dump();
}
#endif
return false;
}
// Make sure the address expression can be handled. It should be
// head->phi * elsize + con. head->phi might have a ConvI2L(CastII()).
Node* elements[4];
Node* cast = NULL;
Node* conv = NULL;
bool found_index = false;
int count = store->in(MemNode::Address)->as_AddP()->unpack_offsets(elements, ARRAY_SIZE(elements));
for (int e = 0; e < count; e++) {
Node* n = elements[e];
if (n->is_Con() && con == NULL) {
con = n;
} else if (n->Opcode() == Op_LShiftX && shift == NULL) {
Node* value = n->in(1);
#ifdef _LP64
if (value->Opcode() == Op_ConvI2L) {
conv = value;
value = value->in(1);
}
if (value->Opcode() == Op_CastII &&
value->as_CastII()->has_range_check()) {
// Skip range check dependent CastII nodes
cast = value;
value = value->in(1);
}
#endif
if (value != head->phi()) {
msg = "unhandled shift in address";
} else {
if (type2aelembytes(store->as_Mem()->memory_type(), true) != (1 << n->in(2)->get_int())) {
msg = "scale doesn't match";
} else {
found_index = true;
shift = n;
}
}
} else if (n->Opcode() == Op_ConvI2L && conv == NULL) {
conv = n;
n = n->in(1);
if (n->Opcode() == Op_CastII &&
n->as_CastII()->has_range_check()) {
// Skip range check dependent CastII nodes
cast = n;
n = n->in(1);
}
if (n == head->phi()) {
found_index = true;
} else {
msg = "unhandled input to ConvI2L";
}
} else if (n == head->phi()) {
// no shift, check below for allowed cases
found_index = true;
} else {
msg = "unhandled node in address";
msg_node = n;
}
}
if (count == -1) {
msg = "malformed address expression";
msg_node = store;
}
if (!found_index) {
msg = "missing use of index";
}
// byte sized items won't have a shift
if (msg == NULL && shift == NULL && t != T_BYTE && t != T_BOOLEAN) {
msg = "can't find shift";
msg_node = store;
}
if (msg != NULL) {
#ifndef PRODUCT
if (TraceOptimizeFill) {
tty->print_cr("not fill intrinsic: %s", msg);
if (msg_node != NULL) msg_node->dump();
}
#endif
return false;
}
// No make sure all the other nodes in the loop can be handled
VectorSet ok(Thread::current()->resource_area());
// store related values are ok
ok.set(store->_idx);
ok.set(store->in(MemNode::Memory)->_idx);
CountedLoopEndNode* loop_exit = head->loopexit();
guarantee(loop_exit != NULL, "no loop exit node");
// Loop structure is ok
ok.set(head->_idx);
ok.set(loop_exit->_idx);
ok.set(head->phi()->_idx);
ok.set(head->incr()->_idx);
ok.set(loop_exit->cmp_node()->_idx);
ok.set(loop_exit->in(1)->_idx);
// Address elements are ok
if (con) ok.set(con->_idx);
if (shift) ok.set(shift->_idx);
if (cast) ok.set(cast->_idx);
if (conv) ok.set(conv->_idx);
for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
Node* n = lpt->_body.at(i);
if (n->outcnt() == 0) continue; // Ignore dead
if (ok.test(n->_idx)) continue;
// Backedge projection is ok
if (n->is_IfTrue() && n->in(0) == loop_exit) continue;
if (!n->is_AddP()) {
msg = "unhandled node";
msg_node = n;
break;
}
}
// Make sure no unexpected values are used outside the loop
for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
Node* n = lpt->_body.at(i);
// These values can be replaced with other nodes if they are used
// outside the loop.
if (n == store || n == loop_exit || n == head->incr() || n == store->in(MemNode::Memory)) continue;
for (SimpleDUIterator iter(n); iter.has_next(); iter.next()) {
Node* use = iter.get();
if (!lpt->_body.contains(use)) {
msg = "node is used outside loop";
// lpt->_body.dump();
msg_node = n;
break;
}
}
}
#ifdef ASSERT
if (TraceOptimizeFill) {
if (msg != NULL) {
tty->print_cr("no fill intrinsic: %s", msg);
if (msg_node != NULL) msg_node->dump();
} else {
tty->print_cr("fill intrinsic for:");
}
store->dump();
if (Verbose) {
lpt->_body.dump();
}
}
#endif
return msg == NULL;
}
bool PhaseIdealLoop::intrinsify_fill(IdealLoopTree* lpt) {
// Only for counted inner loops
if (!lpt->is_counted() || !lpt->is_inner()) {
return false;
}
// Must have constant stride
CountedLoopNode* head = lpt->_head->as_CountedLoop();
if (!head->is_valid_counted_loop() || !head->is_normal_loop()) {
return false;
}
// Check that the body only contains a store of a loop invariant
// value that is indexed by the loop phi.
Node* store = NULL;
Node* store_value = NULL;
Node* shift = NULL;
Node* offset = NULL;
if (!match_fill_loop(lpt, store, store_value, shift, offset)) {
return false;
}
Node* exit = head->loopexit()->proj_out(0);
if (exit == NULL) {
return false;
}
#ifndef PRODUCT
if (TraceLoopOpts) {
tty->print("ArrayFill ");
lpt->dump_head();
}
#endif
// Now replace the whole loop body by a call to a fill routine that
// covers the same region as the loop.
Node* base = store->in(MemNode::Address)->as_AddP()->in(AddPNode::Base);
// Build an expression for the beginning of the copy region
Node* index = head->init_trip();
#ifdef _LP64
index = new (C) ConvI2LNode(index);
_igvn.register_new_node_with_optimizer(index);
#endif
if (shift != NULL) {
// byte arrays don't require a shift but others do.
index = new (C) LShiftXNode(index, shift->in(2));
_igvn.register_new_node_with_optimizer(index);
}
index = new (C) AddPNode(base, base, index);
_igvn.register_new_node_with_optimizer(index);
Node* from = new (C) AddPNode(base, index, offset);
_igvn.register_new_node_with_optimizer(from);
// Compute the number of elements to copy
Node* len = new (C) SubINode(head->limit(), head->init_trip());
_igvn.register_new_node_with_optimizer(len);
BasicType t = store->as_Mem()->memory_type();
bool aligned = false;
if (offset != NULL && head->init_trip()->is_Con()) {
int element_size = type2aelembytes(t);
aligned = (offset->find_intptr_t_type()->get_con() + head->init_trip()->get_int() * element_size) % HeapWordSize == 0;
}
// Build a call to the fill routine
const char* fill_name;
address fill = StubRoutines::select_fill_function(t, aligned, fill_name);
assert(fill != NULL, "what?");
// Convert float/double to int/long for fill routines
if (t == T_FLOAT) {
store_value = new (C) MoveF2INode(store_value);
_igvn.register_new_node_with_optimizer(store_value);
} else if (t == T_DOUBLE) {
store_value = new (C) MoveD2LNode(store_value);
_igvn.register_new_node_with_optimizer(store_value);
}
if (CCallingConventionRequiresIntsAsLongs &&
// See StubRoutines::select_fill_function for types. FLOAT has been converted to INT.
(t == T_FLOAT || t == T_INT || is_subword_type(t))) {
store_value = new (C) ConvI2LNode(store_value);
_igvn.register_new_node_with_optimizer(store_value);
}
Node* mem_phi = store->in(MemNode::Memory);
Node* result_ctrl;
Node* result_mem;
const TypeFunc* call_type = OptoRuntime::array_fill_Type();
CallLeafNode *call = new (C) CallLeafNoFPNode(call_type, fill,
fill_name, TypeAryPtr::get_array_body_type(t));
uint cnt = 0;
call->init_req(TypeFunc::Parms + cnt++, from);
call->init_req(TypeFunc::Parms + cnt++, store_value);
if (CCallingConventionRequiresIntsAsLongs) {
call->init_req(TypeFunc::Parms + cnt++, C->top());
}
#ifdef _LP64
len = new (C) ConvI2LNode(len);
_igvn.register_new_node_with_optimizer(len);
#endif
call->init_req(TypeFunc::Parms + cnt++, len);
#ifdef _LP64
call->init_req(TypeFunc::Parms + cnt++, C->top());
#endif
call->init_req(TypeFunc::Control, head->init_control());
call->init_req(TypeFunc::I_O, C->top()); // Does no I/O.
call->init_req(TypeFunc::Memory, mem_phi->in(LoopNode::EntryControl));
call->init_req(TypeFunc::ReturnAdr, C->start()->proj_out(TypeFunc::ReturnAdr));
call->init_req(TypeFunc::FramePtr, C->start()->proj_out(TypeFunc::FramePtr));
_igvn.register_new_node_with_optimizer(call);
result_ctrl = new (C) ProjNode(call,TypeFunc::Control);
_igvn.register_new_node_with_optimizer(result_ctrl);
result_mem = new (C) ProjNode(call,TypeFunc::Memory);
_igvn.register_new_node_with_optimizer(result_mem);
/* Disable following optimization until proper fix (add missing checks).
// If this fill is tightly coupled to an allocation and overwrites
// the whole body, allow it to take over the zeroing.
AllocateNode* alloc = AllocateNode::Ideal_allocation(base, this);
if (alloc != NULL && alloc->is_AllocateArray()) {
Node* length = alloc->as_AllocateArray()->Ideal_length();
if (head->limit() == length &&
head->init_trip() == _igvn.intcon(0)) {
if (TraceOptimizeFill) {
tty->print_cr("Eliminated zeroing in allocation");
}
alloc->maybe_set_complete(&_igvn);
} else {
#ifdef ASSERT
if (TraceOptimizeFill) {
tty->print_cr("filling array but bounds don't match");
alloc->dump();
head->init_trip()->dump();
head->limit()->dump();
length->dump();
}
#endif
}
}
*/
// Redirect the old control and memory edges that are outside the loop.
// Sometimes the memory phi of the head is used as the outgoing
// state of the loop. It's safe in this case to replace it with the
// result_mem.
_igvn.replace_node(store->in(MemNode::Memory), result_mem);
lazy_replace(exit, result_ctrl);
_igvn.replace_node(store, result_mem);
// Any uses the increment outside of the loop become the loop limit.
_igvn.replace_node(head->incr(), head->limit());
// Disconnect the head from the loop.
for (uint i = 0; i < lpt->_body.size(); i++) {
Node* n = lpt->_body.at(i);
_igvn.replace_node(n, C->top());
}
return true;
}
C:\hotspot-69087d08d473\src\share\vm/opto/loopUnswitch.cpp
/*
* Copyright (c) 2006, 2012, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "memory/allocation.inline.hpp"
#include "opto/connode.hpp"
#include "opto/loopnode.hpp"
#include "opto/rootnode.hpp"
//================= Loop Unswitching =====================
//
// orig: transformed:
// if (invariant-test) then
// predicate predicate
// loop loop
// stmt1 stmt1
// if (invariant-test) then stmt2
// stmt2 stmt4
// else endloop
// stmt3 else
// endif predicate [clone]
// stmt4 loop [clone]
// endloop stmt1 [clone]
// stmt3
// stmt4 [clone]
// endloop
// endif
//
// Note: the "else" clause may be empty
//------------------------------policy_unswitching-----------------------------
// Return TRUE or FALSE if the loop should be unswitched
// (ie. clone loop with an invariant test that does not exit the loop)
bool IdealLoopTree::policy_unswitching( PhaseIdealLoop *phase ) const {
if( !LoopUnswitching ) {
return false;
}
if (!_head->is_Loop()) {
return false;
}
int nodes_left = phase->C->max_node_limit() - phase->C->live_nodes();
if ((int)(2 * _body.size()) > nodes_left) {
return false; // Too speculative if running low on nodes.
}
LoopNode* head = _head->as_Loop();
if (head->unswitch_count() + 1 > head->unswitch_max()) {
return false;
}
return phase->find_unswitching_candidate(this) != NULL;
}
//------------------------------find_unswitching_candidate-----------------------------
// Find candidate "if" for unswitching
IfNode* PhaseIdealLoop::find_unswitching_candidate(const IdealLoopTree *loop) const {
// Find first invariant test that doesn't exit the loop
LoopNode *head = loop->_head->as_Loop();
IfNode* unswitch_iff = NULL;
Node* n = head->in(LoopNode::LoopBackControl);
while (n != head) {
Node* n_dom = idom(n);
if (n->is_Region()) {
if (n_dom->is_If()) {
IfNode* iff = n_dom->as_If();
if (iff->in(1)->is_Bool()) {
BoolNode* bol = iff->in(1)->as_Bool();
if (bol->in(1)->is_Cmp()) {
// If condition is invariant and not a loop exit,
// then found reason to unswitch.
if (loop->is_invariant(bol) && !loop->is_loop_exit(iff)) {
unswitch_iff = iff;
}
}
}
}
}
n = n_dom;
}
return unswitch_iff;
}
//------------------------------do_unswitching-----------------------------
// Clone loop with an invariant test (that does not exit) and
// insert a clone of the test that selects which version to
// execute.
void PhaseIdealLoop::do_unswitching (IdealLoopTree *loop, Node_List &old_new) {
// Find first invariant test that doesn't exit the loop
LoopNode *head = loop->_head->as_Loop();
IfNode* unswitch_iff = find_unswitching_candidate((const IdealLoopTree *)loop);
assert(unswitch_iff != NULL, "should be at least one");
#ifndef PRODUCT
if (TraceLoopOpts) {
tty->print("Unswitch %d ", head->unswitch_count()+1);
loop->dump_head();
}
#endif
// Need to revert back to normal loop
if (head->is_CountedLoop() && !head->as_CountedLoop()->is_normal_loop()) {
head->as_CountedLoop()->set_normal_loop();
}
ProjNode* proj_true = create_slow_version_of_loop(loop, old_new);
#ifdef ASSERT
Node* uniqc = proj_true->unique_ctrl_out();
Node* entry = head->in(LoopNode::EntryControl);
Node* predicate = find_predicate(entry);
if (predicate != NULL && LoopLimitCheck && UseLoopPredicate) {
// We may have two predicates, find first.
entry = find_predicate(entry->in(0)->in(0));
if (entry != NULL) predicate = entry;
}
if (predicate != NULL) predicate = predicate->in(0);
assert(proj_true->is_IfTrue() &&
(predicate == NULL && uniqc == head ||
predicate != NULL && uniqc == predicate), "by construction");
#endif
// Increment unswitch count
LoopNode* head_clone = old_new[head->_idx]->as_Loop();
int nct = head->unswitch_count() + 1;
head->set_unswitch_count(nct);
head_clone->set_unswitch_count(nct);
// Add test to new "if" outside of loop
IfNode* invar_iff = proj_true->in(0)->as_If();
Node* invar_iff_c = invar_iff->in(0);
BoolNode* bol = unswitch_iff->in(1)->as_Bool();
invar_iff->set_req(1, bol);
invar_iff->_prob = unswitch_iff->_prob;
ProjNode* proj_false = invar_iff->proj_out(0)->as_Proj();
// Hoist invariant casts out of each loop to the appropriate
// control projection.
Node_List worklist;
for (DUIterator_Fast imax, i = unswitch_iff->fast_outs(imax); i < imax; i++) {
ProjNode* proj= unswitch_iff->fast_out(i)->as_Proj();
// Copy to a worklist for easier manipulation
for (DUIterator_Fast jmax, j = proj->fast_outs(jmax); j < jmax; j++) {
Node* use = proj->fast_out(j);
if (use->Opcode() == Op_CheckCastPP && loop->is_invariant(use->in(1))) {
worklist.push(use);
}
}
ProjNode* invar_proj = invar_iff->proj_out(proj->_con)->as_Proj();
while (worklist.size() > 0) {
Node* use = worklist.pop();
Node* nuse = use->clone();
nuse->set_req(0, invar_proj);
_igvn.replace_input_of(use, 1, nuse);
register_new_node(nuse, invar_proj);
// Same for the clone
Node* use_clone = old_new[use->_idx];
_igvn.replace_input_of(use_clone, 1, nuse);
}
}
// Hardwire the control paths in the loops into if(true) and if(false)
_igvn.rehash_node_delayed(unswitch_iff);
short_circuit_if(unswitch_iff, proj_true);
IfNode* unswitch_iff_clone = old_new[unswitch_iff->_idx]->as_If();
_igvn.rehash_node_delayed(unswitch_iff_clone);
short_circuit_if(unswitch_iff_clone, proj_false);
// Reoptimize loops
loop->record_for_igvn();
for(int i = loop->_body.size() - 1; i >= 0 ; i--) {
Node *n = loop->_body[i];
Node *n_clone = old_new[n->_idx];
_igvn._worklist.push(n_clone);
}
#ifndef PRODUCT
if (TraceLoopUnswitching) {
tty->print_cr("Loop unswitching orig: %d @ %d new: %d @ %d",
head->_idx, unswitch_iff->_idx,
old_new[head->_idx]->_idx, unswitch_iff_clone->_idx);
}
#endif
C->set_major_progress();
}
//-------------------------create_slow_version_of_loop------------------------
// Create a slow version of the loop by cloning the loop
// and inserting an if to select fast-slow versions.
// Return control projection of the entry to the fast version.
ProjNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop,
Node_List &old_new) {
LoopNode* head = loop->_head->as_Loop();
bool counted_loop = head->is_CountedLoop();
Node* entry = head->in(LoopNode::EntryControl);
_igvn.rehash_node_delayed(entry);
IdealLoopTree* outer_loop = loop->_parent;
Node *cont = _igvn.intcon(1);
set_ctrl(cont, C->root());
Node* opq = new (C) Opaque1Node(C, cont);
register_node(opq, outer_loop, entry, dom_depth(entry));
Node *bol = new (C) Conv2BNode(opq);
register_node(bol, outer_loop, entry, dom_depth(entry));
IfNode* iff = new (C) IfNode(entry, bol, PROB_MAX, COUNT_UNKNOWN);
register_node(iff, outer_loop, entry, dom_depth(entry));
ProjNode* iffast = new (C) IfTrueNode(iff);
register_node(iffast, outer_loop, iff, dom_depth(iff));
ProjNode* ifslow = new (C) IfFalseNode(iff);
register_node(ifslow, outer_loop, iff, dom_depth(iff));
// Clone the loop body. The clone becomes the fast loop. The
// original pre-header will (illegally) have 3 control users
// (old & new loops & new if).
clone_loop(loop, old_new, dom_depth(head), iff);
assert(old_new[head->_idx]->is_Loop(), "" );
// Fast (true) control
Node* iffast_pred = clone_loop_predicates(entry, iffast, !counted_loop);
_igvn.replace_input_of(head, LoopNode::EntryControl, iffast_pred);
set_idom(head, iffast_pred, dom_depth(head));
// Slow (false) control
Node* ifslow_pred = clone_loop_predicates(entry, ifslow, !counted_loop);
LoopNode* slow_head = old_new[head->_idx]->as_Loop();
_igvn.replace_input_of(slow_head, LoopNode::EntryControl, ifslow_pred);
set_idom(slow_head, ifslow_pred, dom_depth(slow_head));
recompute_dom_depth();
return iffast;
}
C:\hotspot-69087d08d473\src\share\vm/opto/machnode.cpp
/*
* Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "gc_interface/collectedHeap.hpp"
#include "opto/machnode.hpp"
#include "opto/regalloc.hpp"
//=============================================================================
// Return the value requested
// result register lookup, corresponding to int_format
int MachOper::reg(PhaseRegAlloc *ra_, const Node *node) const {
return (int)ra_->get_encode(node);
}
// input register lookup, corresponding to ext_format
int MachOper::reg(PhaseRegAlloc *ra_, const Node *node, int idx) const {
return (int)(ra_->get_encode(node->in(idx)));
}
intptr_t MachOper::constant() const { return 0x00; }
relocInfo::relocType MachOper::constant_reloc() const { return relocInfo::none; }
jdouble MachOper::constantD() const { ShouldNotReachHere(); return 0.0; }
jfloat MachOper::constantF() const { ShouldNotReachHere(); return 0.0; }
jlong MachOper::constantL() const { ShouldNotReachHere(); return CONST64(0) ; }
TypeOopPtr *MachOper::oop() const { return NULL; }
int MachOper::ccode() const { return 0x00; }
// A zero, default, indicates this value is not needed.
// May need to lookup the base register, as done in int_ and ext_format
int MachOper::base (PhaseRegAlloc *ra_, const Node *node, int idx) const { return 0x00; }
int MachOper::index(PhaseRegAlloc *ra_, const Node *node, int idx) const { return 0x00; }
int MachOper::scale() const { return 0x00; }
int MachOper::disp (PhaseRegAlloc *ra_, const Node *node, int idx) const { return 0x00; }
int MachOper::constant_disp() const { return 0; }
int MachOper::base_position() const { return -1; } // no base input
int MachOper::index_position() const { return -1; } // no index input
// Check for PC-Relative displacement
relocInfo::relocType MachOper::disp_reloc() const { return relocInfo::none; }
// Return the label
Label* MachOper::label() const { ShouldNotReachHere(); return 0; }
intptr_t MachOper::method() const { ShouldNotReachHere(); return 0; }
//------------------------------negate-----------------------------------------
// Negate conditional branches. Error for non-branch operands
void MachOper::negate() {
ShouldNotCallThis();
}
//-----------------------------type--------------------------------------------
const Type *MachOper::type() const {
return Type::BOTTOM;
}
//------------------------------in_RegMask-------------------------------------
const RegMask *MachOper::in_RegMask(int index) const {
ShouldNotReachHere();
return NULL;
}
//------------------------------dump_spec--------------------------------------
// Print any per-operand special info
#ifndef PRODUCT
void MachOper::dump_spec(outputStream *st) const { }
#endif
//------------------------------hash-------------------------------------------
// Print any per-operand special info
uint MachOper::hash() const {
ShouldNotCallThis();
return 5;
}
//------------------------------cmp--------------------------------------------
// Print any per-operand special info
uint MachOper::cmp( const MachOper &oper ) const {
ShouldNotCallThis();
return opcode() == oper.opcode();
}
//------------------------------hash-------------------------------------------
// Print any per-operand special info
uint labelOper::hash() const {
return _block_num;
}
//------------------------------cmp--------------------------------------------
// Print any per-operand special info
uint labelOper::cmp( const MachOper &oper ) const {
return (opcode() == oper.opcode()) && (_label == oper.label());
}
//------------------------------hash-------------------------------------------
// Print any per-operand special info
uint methodOper::hash() const {
return (uint)_method;
}
//------------------------------cmp--------------------------------------------
// Print any per-operand special info
uint methodOper::cmp( const MachOper &oper ) const {
return (opcode() == oper.opcode()) && (_method == oper.method());
}
//=============================================================================
//------------------------------MachNode---------------------------------------
//------------------------------emit-------------------------------------------
void MachNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
#ifdef ASSERT
tty->print("missing MachNode emit function: ");
dump();
#endif
ShouldNotCallThis();
}
//---------------------------postalloc_expand----------------------------------
// Expand node after register allocation.
void MachNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {}
//------------------------------size-------------------------------------------
// Size of instruction in bytes
uint MachNode::size(PhaseRegAlloc *ra_) const {
// If a virtual was not defined for this specific instruction,
// Call the helper which finds the size by emitting the bits.
return MachNode::emit_size(ra_);
}
//------------------------------size-------------------------------------------
// Helper function that computes size by emitting code
uint MachNode::emit_size(PhaseRegAlloc *ra_) const {
// Emit into a trash buffer and count bytes emitted.
assert(ra_ == ra_->C->regalloc(), "sanity");
return ra_->C->scratch_emit_size(this);
}
//------------------------------hash-------------------------------------------
uint MachNode::hash() const {
uint no = num_opnds();
uint sum = rule();
for( uint i=0; i<no; i++ )
sum += _opnds[i]->hash();
return sum+Node::hash();
}
//-----------------------------cmp---------------------------------------------
uint MachNode::cmp( const Node &node ) const {
MachNode& n = *((Node&)node).as_Mach();
uint no = num_opnds();
if( no != n.num_opnds() ) return 0;
if( rule() != n.rule() ) return 0;
for( uint i=0; i<no; i++ ) // All operands must match
if( !_opnds[i]->cmp( *n._opnds[i] ) )
return 0; // mis-matched operands
return 1; // match
}
// Return an equivalent instruction using memory for cisc_operand position
MachNode *MachNode::cisc_version(int offset, Compile* C) {
ShouldNotCallThis();
return NULL;
}
void MachNode::use_cisc_RegMask() {
ShouldNotReachHere();
}
//-----------------------------in_RegMask--------------------------------------
const RegMask &MachNode::in_RegMask( uint idx ) const {
uint numopnds = num_opnds(); // Virtual call for number of operands
uint skipped = oper_input_base(); // Sum of leaves skipped so far
if( idx < skipped ) {
assert( ideal_Opcode() == Op_AddP, "expected base ptr here" );
assert( idx == 1, "expected base ptr here" );
// debug info can be anywhere
return *Compile::current()->matcher()->idealreg2spillmask[Op_RegP];
}
uint opcnt = 1; // First operand
uint num_edges = _opnds[1]->num_edges(); // leaves for first operand
while( idx >= skipped+num_edges ) {
skipped += num_edges;
opcnt++; // Bump operand count
assert( opcnt < numopnds, "Accessing non-existent operand" );
num_edges = _opnds[opcnt]->num_edges(); // leaves for next operand
}
const RegMask *rm = cisc_RegMask();
if( rm == NULL || (int)opcnt != cisc_operand() ) {
rm = _opnds[opcnt]->in_RegMask(idx-skipped);
}
return *rm;
}
//-----------------------------memory_inputs--------------------------------
const MachOper* MachNode::memory_inputs(Node* &base, Node* &index) const {
const MachOper* oper = memory_operand();
if (oper == (MachOper*)-1) {
base = NodeSentinel;
index = NodeSentinel;
} else {
base = NULL;
index = NULL;
if (oper != NULL) {
// It has a unique memory operand. Find its index.
int oper_idx = num_opnds();
while (--oper_idx >= 0) {
if (_opnds[oper_idx] == oper) break;
}
int oper_pos = operand_index(oper_idx);
int base_pos = oper->base_position();
if (base_pos >= 0) {
base = _in[oper_pos+base_pos];
}
int index_pos = oper->index_position();
if (index_pos >= 0) {
index = _in[oper_pos+index_pos];
}
}
}
return oper;
}
//-----------------------------get_base_and_disp----------------------------
const Node* MachNode::get_base_and_disp(intptr_t &offset, const TypePtr* &adr_type) const {
// Find the memory inputs using our helper function
Node* base;
Node* index;
const MachOper* oper = memory_inputs(base, index);
if (oper == NULL) {
// Base has been set to NULL
offset = 0;
} else if (oper == (MachOper*)-1) {
// Base has been set to NodeSentinel
// There is not a unique memory use here. We will fall to AliasIdxBot.
offset = Type::OffsetBot;
} else {
// Base may be NULL, even if offset turns out to be != 0
intptr_t disp = oper->constant_disp();
int scale = oper->scale();
// Now we have collected every part of the ADLC MEMORY_INTER.
// See if it adds up to a base + offset.
if (index != NULL) {
const Type* t_index = index->bottom_type();
if (t_index->isa_narrowoop() || t_index->isa_narrowklass()) { // EncodeN, LoadN, LoadConN, LoadNKlass,
// EncodeNKlass, LoadConNklass.
// Memory references through narrow oops have a
// funny base so grab the type from the index:
// [R12 + narrow_oop_reg<<3 + offset]
assert(base == NULL, "Memory references through narrow oops have no base");
offset = disp;
adr_type = t_index->make_ptr()->add_offset(offset);
return NULL;
} else if (!index->is_Con()) {
disp = Type::OffsetBot;
} else if (disp != Type::OffsetBot) {
const TypeX* ti = t_index->isa_intptr_t();
if (ti == NULL) {
disp = Type::OffsetBot; // a random constant??
} else {
disp += ti->get_con() << scale;
}
}
}
offset = disp;
// In i486.ad, indOffset32X uses base==RegI and disp==RegP,
// this will prevent alias analysis without the following support:
// Lookup the TypePtr used by indOffset32X, a compile-time constant oop,
// Add the offset determined by the "base", or use Type::OffsetBot.
if( adr_type == TYPE_PTR_SENTINAL ) {
const TypePtr *t_disp = oper->disp_as_type(); // only !NULL for indOffset32X
if (t_disp != NULL) {
offset = Type::OffsetBot;
const Type* t_base = base->bottom_type();
if (t_base->isa_intptr_t()) {
const TypeX *t_offset = t_base->is_intptr_t();
if( t_offset->is_con() ) {
offset = t_offset->get_con();
}
}
adr_type = t_disp->add_offset(offset);
} else if( base == NULL && offset != 0 && offset != Type::OffsetBot ) {
// Use ideal type if it is oop ptr.
const TypePtr *tp = oper->type()->isa_ptr();
if( tp != NULL) {
adr_type = tp;
}
}
}
}
return base;
}
//---------------------------------adr_type---------------------------------
const class TypePtr *MachNode::adr_type() const {
intptr_t offset = 0;
const TypePtr *adr_type = TYPE_PTR_SENTINAL; // attempt computing adr_type
const Node *base = get_base_and_disp(offset, adr_type);
if( adr_type != TYPE_PTR_SENTINAL ) {
return adr_type; // get_base_and_disp has the answer
}
// Direct addressing modes have no base node, simply an indirect
// offset, which is always to raw memory.
// %%%%% Someday we'd like to allow constant oop offsets which
// would let Intel load from static globals in 1 instruction.
// Currently Intel requires 2 instructions and a register temp.
if (base == NULL) {
// NULL base, zero offset means no memory at all (a null pointer!)
if (offset == 0) {
return NULL;
}
// NULL base, any offset means any pointer whatever
if (offset == Type::OffsetBot) {
return TypePtr::BOTTOM;
}
// %%% make offset be intptr_t
assert(!Universe::heap()->is_in_reserved(cast_to_oop(offset)), "must be a raw ptr");
return TypeRawPtr::BOTTOM;
}
// base of -1 with no particular offset means all of memory
if (base == NodeSentinel) return TypePtr::BOTTOM;
const Type* t = base->bottom_type();
if (t->isa_narrowoop() && Universe::narrow_oop_shift() == 0) {
// 32-bit unscaled narrow oop can be the base of any address expression
t = t->make_ptr();
}
if (t->isa_narrowklass() && Universe::narrow_klass_shift() == 0) {
// 32-bit unscaled narrow oop can be the base of any address expression
t = t->make_ptr();
}
if (t->isa_intptr_t() && offset != 0 && offset != Type::OffsetBot) {
// We cannot assert that the offset does not look oop-ish here.
// Depending on the heap layout the cardmark base could land
// inside some oopish region. It definitely does for Win2K.
// The sum of cardmark-base plus shift-by-9-oop lands outside
// the oop-ish area but we can't assert for that statically.
return TypeRawPtr::BOTTOM;
}
const TypePtr *tp = t->isa_ptr();
// be conservative if we do not recognize the type
if (tp == NULL) {
assert(false, "this path may produce not optimal code");
return TypePtr::BOTTOM;
}
assert(tp->base() != Type::AnyPtr, "not a bare pointer");
return tp->add_offset(offset);
}
//-----------------------------operand_index---------------------------------
int MachNode::operand_index( uint operand ) const {
if( operand < 1 ) return -1;
assert(operand < num_opnds(), "oob");
if( _opnds[operand]->num_edges() == 0 ) return -1;
uint skipped = oper_input_base(); // Sum of leaves skipped so far
for (uint opcnt = 1; opcnt < operand; opcnt++) {
uint num_edges = _opnds[opcnt]->num_edges(); // leaves for operand
skipped += num_edges;
}
return skipped;
}
int MachNode::operand_index(const MachOper *oper) const {
uint skipped = oper_input_base(); // Sum of leaves skipped so far
uint opcnt;
for (opcnt = 1; opcnt < num_opnds(); opcnt++) {
if (_opnds[opcnt] == oper) break;
uint num_edges = _opnds[opcnt]->num_edges(); // leaves for operand
skipped += num_edges;
}
if (_opnds[opcnt] != oper) return -1;
return skipped;
}
//------------------------------peephole---------------------------------------
// Apply peephole rule(s) to this instruction
MachNode *MachNode::peephole( Block *block, int block_index, PhaseRegAlloc *ra_, int &deleted, Compile* C ) {
return NULL;
}
//------------------------------add_case_label---------------------------------
// Adds the label for the case
void MachNode::add_case_label( int index_num, Label* blockLabel) {
ShouldNotCallThis();
}
//------------------------------method_set-------------------------------------
// Set the absolute address of a method
void MachNode::method_set( intptr_t addr ) {
ShouldNotCallThis();
}
//------------------------------rematerialize----------------------------------
bool MachNode::rematerialize() const {
// Temps are always rematerializable
if (is_MachTemp()) return true;
uint r = rule(); // Match rule
if( r < Matcher::_begin_rematerialize ||
r >= Matcher::_end_rematerialize )
return false;
// For 2-address instructions, the input live range is also the output
// live range. Remateralizing does not make progress on the that live range.
if( two_adr() ) return false;
// Check for rematerializing float constants, or not
if( !Matcher::rematerialize_float_constants ) {
int op = ideal_Opcode();
if( op == Op_ConF || op == Op_ConD )
return false;
}
// Defining flags - can't spill these! Must remateralize.
if( ideal_reg() == Op_RegFlags )
return true;
// Stretching lots of inputs - don't do it.
if( req() > 2 )
return false;
// Don't remateralize somebody with bound inputs - it stretches a
// fixed register lifetime.
uint idx = oper_input_base();
if (req() > idx) {
const RegMask &rm = in_RegMask(idx);
if (rm.is_bound(ideal_reg()))
return false;
}
return true;
}
#ifndef PRODUCT
//------------------------------dump_spec--------------------------------------
// Print any per-operand special info
void MachNode::dump_spec(outputStream *st) const {
uint cnt = num_opnds();
for( uint i=0; i<cnt; i++ )
_opnds[i]->dump_spec(st);
const TypePtr *t = adr_type();
if( t ) {
Compile* C = Compile::current();
if( C->alias_type(t)->is_volatile() )
st->print(" Volatile!");
}
}
//------------------------------dump_format------------------------------------
// access to virtual
void MachNode::dump_format(PhaseRegAlloc *ra, outputStream *st) const {
format(ra, st); // access to virtual
}
#endif
//=============================================================================
#ifndef PRODUCT
void MachTypeNode::dump_spec(outputStream *st) const {
_bottom_type->dump_on(st);
}
#endif
//=============================================================================
int MachConstantNode::constant_offset() {
// Bind the offset lazily.
if (_constant.offset() == -1) {
Compile::ConstantTable& constant_table = Compile::current()->constant_table();
int offset = constant_table.find_offset(_constant);
// If called from Compile::scratch_emit_size return the
// pre-calculated offset.
// NOTE: If the AD file does some table base offset optimizations
// later the AD file needs to take care of this fact.
if (Compile::current()->in_scratch_emit_size()) {
return constant_table.calculate_table_base_offset() + offset;
}
_constant.set_offset(constant_table.table_base_offset() + offset);
}
return _constant.offset();
}
int MachConstantNode::constant_offset_unchecked() const {
return _constant.offset();
}
//=============================================================================
#ifndef PRODUCT
void MachNullCheckNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
int reg = ra_->get_reg_first(in(1)->in(_vidx));
st->print("%s %s", Name(), Matcher::regName[reg]);
}
#endif
void MachNullCheckNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
// only emits entries in the null-pointer exception handler table
}
void MachNullCheckNode::label_set(Label* label, uint block_num) {
// Nothing to emit
}
void MachNullCheckNode::save_label( Label** label, uint* block_num ) {
// Nothing to emit
}
const RegMask &MachNullCheckNode::in_RegMask( uint idx ) const {
if( idx == 0 ) return RegMask::Empty;
else return in(1)->as_Mach()->out_RegMask();
}
//=============================================================================
const Type *MachProjNode::bottom_type() const {
if( _ideal_reg == fat_proj ) return Type::BOTTOM;
// Try the normal mechanism first
const Type *t = in(0)->bottom_type();
if( t->base() == Type::Tuple ) {
const TypeTuple *tt = t->is_tuple();
if (_con < tt->cnt())
return tt->field_at(_con);
}
// Else use generic type from ideal register set
assert((uint)_ideal_reg < (uint)_last_machine_leaf && Type::mreg2type[_ideal_reg], "in bounds");
return Type::mreg2type[_ideal_reg];
}
const TypePtr *MachProjNode::adr_type() const {
if (bottom_type() == Type::MEMORY) {
// in(0) might be a narrow MemBar; otherwise we will report TypePtr::BOTTOM
const TypePtr* adr_type = in(0)->adr_type();
#ifdef ASSERT
if (!is_error_reported() && !Node::in_dump())
assert(adr_type != NULL, "source must have adr_type");
#endif
return adr_type;
}
assert(bottom_type()->base() != Type::Memory, "no other memories?");
return NULL;
}
#ifndef PRODUCT
void MachProjNode::dump_spec(outputStream *st) const {
ProjNode::dump_spec(st);
switch (_ideal_reg) {
case unmatched_proj: st->print("/unmatched"); break;
case fat_proj: st->print("/fat"); if (WizardMode) _rout.dump(); break;
}
}
#endif
//=============================================================================
#ifndef PRODUCT
void MachIfNode::dump_spec(outputStream *st) const {
st->print("P=%f, C=%f",_prob, _fcnt);
}
#endif
//=============================================================================
uint MachReturnNode::size_of() const { return sizeof(*this); }
//------------------------------Registers--------------------------------------
const RegMask &MachReturnNode::in_RegMask( uint idx ) const {
return _in_rms[idx];
}
const TypePtr *MachReturnNode::adr_type() const {
// most returns and calls are assumed to consume & modify all of memory
// the matcher will copy non-wide adr_types from ideal originals
return _adr_type;
}
//=============================================================================
const Type *MachSafePointNode::bottom_type() const { return TypeTuple::MEMBAR; }
//------------------------------Registers--------------------------------------
const RegMask &MachSafePointNode::in_RegMask( uint idx ) const {
// Values in the domain use the users calling convention, embodied in the
// _in_rms array of RegMasks.
if( idx < TypeFunc::Parms ) return _in_rms[idx];
if (SafePointNode::needs_polling_address_input() &&
idx == TypeFunc::Parms &&
ideal_Opcode() == Op_SafePoint) {
return MachNode::in_RegMask(idx);
}
// Values outside the domain represent debug info
assert(in(idx)->ideal_reg() != Op_RegFlags, "flags register is not spillable");
return *Compile::current()->matcher()->idealreg2spillmask[in(idx)->ideal_reg()];
}
//=============================================================================
uint MachCallNode::cmp( const Node &n ) const
{ return _tf == ((MachCallNode&)n)._tf; }
const Type *MachCallNode::bottom_type() const { return tf()->range(); }
const Type *MachCallNode::Value(PhaseTransform *phase) const { return tf()->range(); }
#ifndef PRODUCT
void MachCallNode::dump_spec(outputStream *st) const {
st->print("# ");
tf()->dump_on(st);
if (_cnt != COUNT_UNKNOWN) st->print(" C=%f",_cnt);
if (jvms() != NULL) jvms()->dump_spec(st);
}
#endif
bool MachCallNode::return_value_is_used() const {
if (tf()->range()->cnt() == TypeFunc::Parms) {
// void return
return false;
}
// find the projection corresponding to the return value
for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
Node *use = fast_out(i);
if (!use->is_Proj()) continue;
if (use->as_Proj()->_con == TypeFunc::Parms) {
return true;
}
}
return false;
}
// Similar to cousin class CallNode::returns_pointer
// Because this is used in deoptimization, we want the type info, not the data
// flow info; the interpreter will "use" things that are dead to the optimizer.
bool MachCallNode::returns_pointer() const {
const TypeTuple *r = tf()->range();
return (r->cnt() > TypeFunc::Parms &&
r->field_at(TypeFunc::Parms)->isa_ptr());
}
//------------------------------Registers--------------------------------------
const RegMask &MachCallNode::in_RegMask(uint idx) const {
// Values in the domain use the users calling convention, embodied in the
// _in_rms array of RegMasks.
if (idx < tf()->domain()->cnt()) {
return _in_rms[idx];
}
if (idx == mach_constant_base_node_input()) {
return MachConstantBaseNode::static_out_RegMask();
}
// Values outside the domain represent debug info
return *Compile::current()->matcher()->idealreg2debugmask[in(idx)->ideal_reg()];
}
//=============================================================================
uint MachCallJavaNode::size_of() const { return sizeof(*this); }
uint MachCallJavaNode::cmp( const Node &n ) const {
MachCallJavaNode &call = (MachCallJavaNode&)n;
return MachCallNode::cmp(call) && _method->equals(call._method);
}
#ifndef PRODUCT
void MachCallJavaNode::dump_spec(outputStream *st) const {
if (_method_handle_invoke)
st->print("MethodHandle ");
if (_method) {
_method->print_short_name(st);
st->print(" ");
}
MachCallNode::dump_spec(st);
}
#endif
//------------------------------Registers--------------------------------------
const RegMask &MachCallJavaNode::in_RegMask(uint idx) const {
// Values in the domain use the users calling convention, embodied in the
// _in_rms array of RegMasks.
if (idx < tf()->domain()->cnt()) {
return _in_rms[idx];
}
if (idx == mach_constant_base_node_input()) {
return MachConstantBaseNode::static_out_RegMask();
}
// Values outside the domain represent debug info
Matcher* m = Compile::current()->matcher();
// If this call is a MethodHandle invoke we have to use a different
// debugmask which does not include the register we use to save the
// SP over MH invokes.
RegMask** debugmask = _method_handle_invoke ? m->idealreg2mhdebugmask : m->idealreg2debugmask;
return *debugmask[in(idx)->ideal_reg()];
}
//=============================================================================
uint MachCallStaticJavaNode::size_of() const { return sizeof(*this); }
uint MachCallStaticJavaNode::cmp( const Node &n ) const {
MachCallStaticJavaNode &call = (MachCallStaticJavaNode&)n;
return MachCallJavaNode::cmp(call) && _name == call._name;
}
//----------------------------uncommon_trap_request----------------------------
// If this is an uncommon trap, return the request code, else zero.
int MachCallStaticJavaNode::uncommon_trap_request() const {
if (_name != NULL && !strcmp(_name, "uncommon_trap")) {
return CallStaticJavaNode::extract_uncommon_trap_request(this);
}
return 0;
}
#ifndef PRODUCT
// Helper for summarizing uncommon_trap arguments.
void MachCallStaticJavaNode::dump_trap_args(outputStream *st) const {
int trap_req = uncommon_trap_request();
if (trap_req != 0) {
char buf[100];
st->print("(%s)",
Deoptimization::format_trap_request(buf, sizeof(buf),
trap_req));
}
}
void MachCallStaticJavaNode::dump_spec(outputStream *st) const {
st->print("Static ");
if (_name != NULL) {
st->print("wrapper for: %s", _name );
dump_trap_args(st);
st->print(" ");
}
MachCallJavaNode::dump_spec(st);
}
#endif
//=============================================================================
#ifndef PRODUCT
void MachCallDynamicJavaNode::dump_spec(outputStream *st) const {
st->print("Dynamic ");
MachCallJavaNode::dump_spec(st);
}
#endif
//=============================================================================
uint MachCallRuntimeNode::size_of() const { return sizeof(*this); }
uint MachCallRuntimeNode::cmp( const Node &n ) const {
MachCallRuntimeNode &call = (MachCallRuntimeNode&)n;
return MachCallNode::cmp(call) && !strcmp(_name,call._name);
}
#ifndef PRODUCT
void MachCallRuntimeNode::dump_spec(outputStream *st) const {
st->print("%s ",_name);
MachCallNode::dump_spec(st);
}
#endif
//=============================================================================
// A shared JVMState for all HaltNodes. Indicates the start of debug info
// is at TypeFunc::Parms. Only required for SOE register spill handling -
// to indicate where the stack-slot-only debug info inputs begin.
// There is no other JVM state needed here.
JVMState jvms_for_throw(0);
JVMState *MachHaltNode::jvms() const {
return &jvms_for_throw;
}
//=============================================================================
#ifndef PRODUCT
void labelOper::int_format(PhaseRegAlloc *ra, const MachNode *node, outputStream *st) const {
st->print("B%d", _block_num);
}
#endif // PRODUCT
//=============================================================================
#ifndef PRODUCT
void methodOper::int_format(PhaseRegAlloc *ra, const MachNode *node, outputStream *st) const {
st->print(INTPTR_FORMAT, _method);
}
#endif // PRODUCT
C:\hotspot-69087d08d473\src\share\vm/opto/machnode.hpp
/*
* Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_VM_OPTO_MACHNODE_HPP
#define SHARE_VM_OPTO_MACHNODE_HPP
#include "opto/callnode.hpp"
#include "opto/matcher.hpp"
#include "opto/multnode.hpp"
#include "opto/node.hpp"
#include "opto/regmask.hpp"
class BiasedLockingCounters;
class BufferBlob;
class CodeBuffer;
class JVMState;
class MachCallDynamicJavaNode;
class MachCallJavaNode;
class MachCallLeafNode;
class MachCallNode;
class MachCallRuntimeNode;
class MachCallStaticJavaNode;
class MachEpilogNode;
class MachIfNode;
class MachNullCheckNode;
class MachOper;
class MachProjNode;
class MachPrologNode;
class MachReturnNode;
class MachSafePointNode;
class MachSpillCopyNode;
class Matcher;
class PhaseRegAlloc;
class RegMask;
class RTMLockingCounters;
class State;
//---------------------------MachOper------------------------------------------
class MachOper : public ResourceObj {
public:
// Allocate right next to the MachNodes in the same arena
void *operator new( size_t x, Compile* C ) throw() { return C->node_arena()->Amalloc_D(x); }
// Opcode
virtual uint opcode() const = 0;
// Number of input edges.
// Generally at least 1
virtual uint num_edges() const { return 1; }
// Array of Register masks
virtual const RegMask *in_RegMask(int index) const;
// Methods to output the encoding of the operand
// Negate conditional branches. Error for non-branch Nodes
virtual void negate();
// Return the value requested
// result register lookup, corresponding to int_format
virtual int reg(PhaseRegAlloc *ra_, const Node *node) const;
// input register lookup, corresponding to ext_format
virtual int reg(PhaseRegAlloc *ra_, const Node *node, int idx) const;
// helpers for MacroAssembler generation from ADLC
Register as_Register(PhaseRegAlloc *ra_, const Node *node) const {
return ::as_Register(reg(ra_, node));
}
Register as_Register(PhaseRegAlloc *ra_, const Node *node, int idx) const {
return ::as_Register(reg(ra_, node, idx));
}
FloatRegister as_FloatRegister(PhaseRegAlloc *ra_, const Node *node) const {
return ::as_FloatRegister(reg(ra_, node));
}
FloatRegister as_FloatRegister(PhaseRegAlloc *ra_, const Node *node, int idx) const {
return ::as_FloatRegister(reg(ra_, node, idx));
}
#if defined(IA32) || defined(AMD64)
XMMRegister as_XMMRegister(PhaseRegAlloc *ra_, const Node *node) const {
return ::as_XMMRegister(reg(ra_, node));
}
XMMRegister as_XMMRegister(PhaseRegAlloc *ra_, const Node *node, int idx) const {
return ::as_XMMRegister(reg(ra_, node, idx));
}
#endif
// CondRegister reg converter
#if defined(PPC64)
ConditionRegister as_ConditionRegister(PhaseRegAlloc *ra_, const Node *node) const {
return ::as_ConditionRegister(reg(ra_, node));
}
ConditionRegister as_ConditionRegister(PhaseRegAlloc *ra_, const Node *node, int idx) const {
return ::as_ConditionRegister(reg(ra_, node, idx));
}
#endif
virtual intptr_t constant() const;
virtual relocInfo::relocType constant_reloc() const;
virtual jdouble constantD() const;
virtual jfloat constantF() const;
virtual jlong constantL() const;
virtual TypeOopPtr *oop() const;
virtual int ccode() const;
// A zero, default, indicates this value is not needed.
// May need to lookup the base register, as done in int_ and ext_format
virtual int base (PhaseRegAlloc *ra_, const Node *node, int idx) const;
virtual int index(PhaseRegAlloc *ra_, const Node *node, int idx) const;
virtual int scale() const;
// Parameters needed to support MEMORY_INTERFACE access to stackSlot
virtual int disp (PhaseRegAlloc *ra_, const Node *node, int idx) const;
// Check for PC-Relative displacement
virtual relocInfo::relocType disp_reloc() const;
virtual int constant_disp() const; // usu. 0, may return Type::OffsetBot
virtual int base_position() const; // base edge position, or -1
virtual int index_position() const; // index edge position, or -1
// Access the TypeKlassPtr of operands with a base==RegI and disp==RegP
// Only returns non-null value for i486.ad's indOffset32X
virtual const TypePtr *disp_as_type() const { return NULL; }
// Return the label
virtual Label *label() const;
// Return the method's address
virtual intptr_t method() const;
// Hash and compare over operands are currently identical
virtual uint hash() const;
virtual uint cmp( const MachOper &oper ) const;
// Virtual clone, since I do not know how big the MachOper is.
virtual MachOper *clone(Compile* C) const = 0;
// Return ideal Type from simple operands. Fail for complex operands.
virtual const Type *type() const;
// Set an integer offset if we have one, or error otherwise
virtual void set_con( jint c0 ) { ShouldNotReachHere(); }
#ifndef PRODUCT
// Return name of operand
virtual const char *Name() const { return "???";}
// Methods to output the text version of the operand
virtual void int_format(PhaseRegAlloc *,const MachNode *node, outputStream *st) const = 0;
virtual void ext_format(PhaseRegAlloc *,const MachNode *node,int idx, outputStream *st) const=0;
virtual void dump_spec(outputStream *st) const; // Print per-operand info
// Check whether o is a valid oper.
static bool notAnOper(const MachOper *o) {
if (o == NULL) return true;
if (((intptr_t)o & 1) != 0) return true;
if (*(address*)o == badAddress) return true; // kill by Node::destruct
return false;
}
#endif // !PRODUCT
};
//------------------------------MachNode---------------------------------------
// Base type for all machine specific nodes. All node classes generated by the
// ADLC inherit from this class.
class MachNode : public Node {
public:
MachNode() : Node((uint)0), _num_opnds(0), _opnds(NULL) {
init_class_id(Class_Mach);
}
// Required boilerplate
virtual uint size_of() const { return sizeof(MachNode); }
virtual int Opcode() const; // Always equal to MachNode
virtual uint rule() const = 0; // Machine-specific opcode
// Number of inputs which come before the first operand.
// Generally at least 1, to skip the Control input
virtual uint oper_input_base() const { return 1; }
// Position of constant base node in node's inputs. -1 if
// no constant base node input.
virtual uint mach_constant_base_node_input() const { return (uint)-1; }
// Copy inputs and operands to new node of instruction.
// Called from cisc_version() and short_branch_version().
// !!!! The method's body is defined in ad_<arch>.cpp file.
void fill_new_machnode(MachNode *n, Compile* C) const;
// Return an equivalent instruction using memory for cisc_operand position
virtual MachNode *cisc_version(int offset, Compile* C);
// Modify this instruction's register mask to use stack version for cisc_operand
virtual void use_cisc_RegMask();
// Support for short branches
bool may_be_short_branch() const { return (flags() & Flag_may_be_short_branch) != 0; }
// Avoid back to back some instructions on some CPUs.
enum AvoidBackToBackFlag { AVOID_NONE = 0,
AVOID_BEFORE = Flag_avoid_back_to_back_before,
AVOID_AFTER = Flag_avoid_back_to_back_after,
AVOID_BEFORE_AND_AFTER = AVOID_BEFORE | AVOID_AFTER };
bool avoid_back_to_back(AvoidBackToBackFlag flag_value) const {
return (flags() & flag_value) == flag_value;
}
// instruction implemented with a call
bool has_call() const { return (flags() & Flag_has_call) != 0; }
// First index in _in[] corresponding to operand, or -1 if there is none
int operand_index(uint operand) const;
int operand_index(const MachOper *oper) const;
// Register class input is expected in
virtual const RegMask &in_RegMask(uint) const;
// cisc-spillable instructions redefine for use by in_RegMask
virtual const RegMask *cisc_RegMask() const { return NULL; }
// If this instruction is a 2-address instruction, then return the
// index of the input which must match the output. Not nessecary
// for instructions which bind the input and output register to the
// same singleton regiser (e.g., Intel IDIV which binds AX to be
// both an input and an output). It is nessecary when the input and
// output have choices - but they must use the same choice.
virtual uint two_adr( ) const { return 0; }
// Array of complex operand pointers. Each corresponds to zero or
// more leafs. Must be set by MachNode constructor to point to an
// internal array of MachOpers. The MachOper array is sized by
// specific MachNodes described in the ADL.
uint _num_opnds;
MachOper **_opnds;
uint num_opnds() const { return _num_opnds; }
// Emit bytes into cbuf
virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const;
// Expand node after register allocation.
// Node is replaced by several nodes in the postalloc expand phase.
// Corresponding methods are generated for nodes if they specify
// postalloc_expand. See block.cpp for more documentation.
virtual bool requires_postalloc_expand() const { return false; }
virtual void postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_);
// Size of instruction in bytes
virtual uint size(PhaseRegAlloc *ra_) const;
// Helper function that computes size by emitting code
virtual uint emit_size(PhaseRegAlloc *ra_) const;
// Return the alignment required (in units of relocInfo::addr_unit())
// for this instruction (must be a power of 2)
virtual int alignment_required() const { return 1; }
// Return the padding (in bytes) to be emitted before this
// instruction to properly align it.
virtual int compute_padding(int current_offset) const { return 0; }
// Return number of relocatable values contained in this instruction
virtual int reloc() const { return 0; }
// Return number of words used for double constants in this instruction
virtual int ins_num_consts() const { return 0; }
// Hash and compare over operands. Used to do GVN on machine Nodes.
virtual uint hash() const;
virtual uint cmp( const Node &n ) const;
// Expand method for MachNode, replaces nodes representing pseudo
// instructions with a set of nodes which represent real machine
// instructions and compute the same value.
virtual MachNode *Expand( State *, Node_List &proj_list, Node* mem ) { return this; }
// Bottom_type call; value comes from operand0
virtual const class Type *bottom_type() const { return _opnds[0]->type(); }
virtual uint ideal_reg() const { const Type *t = _opnds[0]->type(); return t == TypeInt::CC ? Op_RegFlags : t->ideal_reg(); }
// If this is a memory op, return the base pointer and fixed offset.
// If there are no such, return NULL. If there are multiple addresses
// or the address is indeterminate (rare cases) then return (Node*)-1,
// which serves as node bottom.
// If the offset is not statically determined, set it to Type::OffsetBot.
// This method is free to ignore stack slots if that helps.
#define TYPE_PTR_SENTINAL ((const TypePtr*)-1)
// Passing TYPE_PTR_SENTINAL as adr_type asks for computation of the adr_type if possible
const Node* get_base_and_disp(intptr_t &offset, const TypePtr* &adr_type) const;
// Helper for get_base_and_disp: find the base and index input nodes.
// Returns the MachOper as determined by memory_operand(), for use, if
// needed by the caller. If (MachOper *)-1 is returned, base and index
// are set to NodeSentinel. If (MachOper *) NULL is returned, base and
// index are set to NULL.
const MachOper* memory_inputs(Node* &base, Node* &index) const;
// Helper for memory_inputs: Which operand carries the necessary info?
// By default, returns NULL, which means there is no such operand.
// If it returns (MachOper*)-1, this means there are multiple memories.
virtual const MachOper* memory_operand() const { return NULL; }
// Call "get_base_and_disp" to decide which category of memory is used here.
virtual const class TypePtr *adr_type() const;
// Apply peephole rule(s) to this instruction
virtual MachNode *peephole( Block *block, int block_index, PhaseRegAlloc *ra_, int &deleted, Compile* C );
// Top-level ideal Opcode matched
virtual int ideal_Opcode() const { return Op_Node; }
// Adds the label for the case
virtual void add_case_label( int switch_val, Label* blockLabel);
// Set the absolute address for methods
virtual void method_set( intptr_t addr );
// Should we clone rather than spill this instruction?
bool rematerialize() const;
// Get the pipeline info
static const Pipeline *pipeline_class();
virtual const Pipeline *pipeline() const;
// Returns true if this node is a check that can be implemented with a trap.
virtual bool is_TrapBasedCheckNode() const { return false; }
#ifndef PRODUCT
virtual const char *Name() const = 0; // Machine-specific name
virtual void dump_spec(outputStream *st) const; // Print per-node info
void dump_format(PhaseRegAlloc *ra, outputStream *st) const; // access to virtual
#endif
};
//------------------------------MachIdealNode----------------------------
// Machine specific versions of nodes that must be defined by user.
// These are not converted by matcher from ideal nodes to machine nodes
// but are inserted into the code by the compiler.
class MachIdealNode : public MachNode {
public:
MachIdealNode( ) {}
// Define the following defaults for non-matched machine nodes
virtual uint oper_input_base() const { return 0; }
virtual uint rule() const { return 9999999; }
virtual const class Type *bottom_type() const { return _opnds == NULL ? Type::CONTROL : MachNode::bottom_type(); }
};
//------------------------------MachTypeNode----------------------------
// Machine Nodes that need to retain a known Type.
class MachTypeNode : public MachNode {
virtual uint size_of() const { return sizeof(*this); } // Size is bigger
public:
MachTypeNode( ) {}
const Type *_bottom_type;
virtual const class Type *bottom_type() const { return _bottom_type; }
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
//------------------------------MachBreakpointNode----------------------------
// Machine breakpoint or interrupt Node
class MachBreakpointNode : public MachIdealNode {
public:
MachBreakpointNode( ) {}
virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const;
virtual uint size(PhaseRegAlloc *ra_) const;
#ifndef PRODUCT
virtual const char *Name() const { return "Breakpoint"; }
virtual void format( PhaseRegAlloc *, outputStream *st ) const;
#endif
};
//------------------------------MachConstantBaseNode--------------------------
// Machine node that represents the base address of the constant table.
class MachConstantBaseNode : public MachIdealNode {
public:
static const RegMask& _out_RegMask; // We need the out_RegMask statically in MachConstantNode::in_RegMask().
public:
MachConstantBaseNode() : MachIdealNode() {
init_class_id(Class_MachConstantBase);
}
virtual const class Type* bottom_type() const { return TypeRawPtr::NOTNULL; }
virtual uint ideal_reg() const { return Op_RegP; }
virtual uint oper_input_base() const { return 1; }
virtual bool requires_postalloc_expand() const;
virtual void postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_);
virtual void emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const;
virtual uint size(PhaseRegAlloc* ra_) const;
virtual bool pinned() const { return UseRDPCForConstantTableBase; }
static const RegMask& static_out_RegMask() { return _out_RegMask; }
virtual const RegMask& out_RegMask() const { return static_out_RegMask(); }
#ifndef PRODUCT
virtual const char* Name() const { return "MachConstantBaseNode"; }
virtual void format(PhaseRegAlloc*, outputStream* st) const;
#endif
};
//------------------------------MachConstantNode-------------------------------
// Machine node that holds a constant which is stored in the constant table.
class MachConstantNode : public MachTypeNode {
protected:
Compile::Constant _constant; // This node's constant.
public:
MachConstantNode() : MachTypeNode() {
init_class_id(Class_MachConstant);
}
virtual void eval_constant(Compile* C) {
#ifdef ASSERT
tty->print("missing MachConstantNode eval_constant function: ");
dump();
#endif
ShouldNotCallThis();
}
virtual const RegMask &in_RegMask(uint idx) const {
if (idx == mach_constant_base_node_input())
return MachConstantBaseNode::static_out_RegMask();
return MachNode::in_RegMask(idx);
}
// Input edge of MachConstantBaseNode.
virtual uint mach_constant_base_node_input() const { return req() - 1; }
int constant_offset();
int constant_offset() const { return ((MachConstantNode*) this)->constant_offset(); }
// Unchecked version to avoid assertions in debug output.
int constant_offset_unchecked() const;
};
//------------------------------MachUEPNode-----------------------------------
// Machine Unvalidated Entry Point Node
class MachUEPNode : public MachIdealNode {
public:
MachUEPNode( ) {}
virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const;
virtual uint size(PhaseRegAlloc *ra_) const;
#ifndef PRODUCT
virtual const char *Name() const { return "Unvalidated-Entry-Point"; }
virtual void format( PhaseRegAlloc *, outputStream *st ) const;
#endif
};
//------------------------------MachPrologNode--------------------------------
// Machine function Prolog Node
class MachPrologNode : public MachIdealNode {
public:
MachPrologNode( ) {}
virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const;
virtual uint size(PhaseRegAlloc *ra_) const;
virtual int reloc() const;
#ifndef PRODUCT
virtual const char *Name() const { return "Prolog"; }
virtual void format( PhaseRegAlloc *, outputStream *st ) const;
#endif
};
//------------------------------MachEpilogNode--------------------------------
// Machine function Epilog Node
class MachEpilogNode : public MachIdealNode {
public:
MachEpilogNode(bool do_poll = false) : _do_polling(do_poll) {}
virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const;
virtual uint size(PhaseRegAlloc *ra_) const;
virtual int reloc() const;
virtual const Pipeline *pipeline() const;
private:
bool _do_polling;
public:
bool do_polling() const { return _do_polling; }
// Offset of safepoint from the beginning of the node
int safepoint_offset() const;
#ifndef PRODUCT
virtual const char *Name() const { return "Epilog"; }
virtual void format( PhaseRegAlloc *, outputStream *st ) const;
#endif
};
//------------------------------MachNopNode-----------------------------------
// Machine function Nop Node
class MachNopNode : public MachIdealNode {
private:
int _count;
public:
MachNopNode( ) : _count(1) {}
MachNopNode( int count ) : _count(count) {}
virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const;
virtual uint size(PhaseRegAlloc *ra_) const;
virtual const class Type *bottom_type() const { return Type::CONTROL; }
virtual int ideal_Opcode() const { return Op_Con; } // bogus; see output.cpp
virtual const Pipeline *pipeline() const;
#ifndef PRODUCT
virtual const char *Name() const { return "Nop"; }
virtual void format( PhaseRegAlloc *, outputStream *st ) const;
virtual void dump_spec(outputStream *st) const { } // No per-operand info
#endif
};
//------------------------------MachSpillCopyNode------------------------------
// Machine SpillCopy Node. Copies 1 or 2 words from any location to any
// location (stack or register).
class MachSpillCopyNode : public MachIdealNode {
const RegMask *_in; // RegMask for input
const RegMask *_out; // RegMask for output
const Type *_type;
public:
MachSpillCopyNode( Node *n, const RegMask &in, const RegMask &out ) :
MachIdealNode(), _in(&in), _out(&out), _type(n->bottom_type()) {
init_class_id(Class_MachSpillCopy);
init_flags(Flag_is_Copy);
add_req(NULL);
add_req(n);
}
virtual uint size_of() const { return sizeof(*this); }
void set_out_RegMask(const RegMask &out) { _out = &out; }
void set_in_RegMask(const RegMask &in) { _in = ∈ }
virtual const RegMask &out_RegMask() const { return *_out; }
virtual const RegMask &in_RegMask(uint) const { return *_in; }
virtual const class Type *bottom_type() const { return _type; }
virtual uint ideal_reg() const { return _type->ideal_reg(); }
virtual uint oper_input_base() const { return 1; }
uint implementation( CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream* st ) const;
virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const;
virtual uint size(PhaseRegAlloc *ra_) const;
#ifndef PRODUCT
virtual const char *Name() const { return "MachSpillCopy"; }
virtual void format( PhaseRegAlloc *, outputStream *st ) const;
#endif
};
// MachMergeNode is similar to a PhiNode in a sense it merges multiple values,
// however it doesn't have a control input and is more like a MergeMem.
// It is inserted after the register allocation is done to ensure that nodes use single
// definition of a multidef lrg in a block.
class MachMergeNode : public MachIdealNode {
public:
MachMergeNode(Node *n1) {
init_class_id(Class_MachMerge);
add_req(NULL);
add_req(n1);
}
virtual const RegMask &out_RegMask() const { return in(1)->out_RegMask(); }
virtual const RegMask &in_RegMask(uint idx) const { return in(1)->in_RegMask(idx); }
virtual const class Type *bottom_type() const { return in(1)->bottom_type(); }
virtual uint ideal_reg() const { return bottom_type()->ideal_reg(); }
virtual uint oper_input_base() const { return 1; }
virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { }
virtual uint size(PhaseRegAlloc *ra_) const { return 0; }
#ifndef PRODUCT
virtual const char *Name() const { return "MachMerge"; }
#endif
};
//------------------------------MachBranchNode--------------------------------
// Abstract machine branch Node
class MachBranchNode : public MachIdealNode {
public:
MachBranchNode() : MachIdealNode() {
init_class_id(Class_MachBranch);
}
virtual void label_set(Label* label, uint block_num) = 0;
virtual void save_label(Label** label, uint* block_num) = 0;
// Support for short branches
virtual MachNode *short_branch_version(Compile* C) { return NULL; }
virtual bool pinned() const { return true; };
};
//------------------------------MachNullChkNode--------------------------------
// Machine-dependent null-pointer-check Node. Points a real MachNode that is
// also some kind of memory op. Turns the indicated MachNode into a
// conditional branch with good latency on the ptr-not-null path and awful
// latency on the pointer-is-null path.
class MachNullCheckNode : public MachBranchNode {
public:
const uint _vidx; // Index of memop being tested
MachNullCheckNode( Node *ctrl, Node *memop, uint vidx ) : MachBranchNode(), _vidx(vidx) {
init_class_id(Class_MachNullCheck);
add_req(ctrl);
add_req(memop);
}
virtual uint size_of() const { return sizeof(*this); }
virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const;
virtual void label_set(Label* label, uint block_num);
virtual void save_label(Label** label, uint* block_num);
virtual void negate() { }
virtual const class Type *bottom_type() const { return TypeTuple::IFBOTH; }
virtual uint ideal_reg() const { return NotAMachineReg; }
virtual const RegMask &in_RegMask(uint) const;
virtual const RegMask &out_RegMask() const { return RegMask::Empty; }
#ifndef PRODUCT
virtual const char *Name() const { return "NullCheck"; }
virtual void format( PhaseRegAlloc *, outputStream *st ) const;
#endif
};
//------------------------------MachProjNode----------------------------------
// Machine-dependent Ideal projections (how is that for an oxymoron). Really
// just MachNodes made by the Ideal world that replicate simple projections
// but with machine-dependent input & output register masks. Generally
// produced as part of calling conventions. Normally I make MachNodes as part
// of the Matcher process, but the Matcher is ill suited to issues involving
// frame handling, so frame handling is all done in the Ideal world with
// occasional callbacks to the machine model for important info.
class MachProjNode : public ProjNode {
public:
MachProjNode( Node *multi, uint con, const RegMask &out, uint ideal_reg ) : ProjNode(multi,con), _rout(out), _ideal_reg(ideal_reg) {
init_class_id(Class_MachProj);
}
RegMask _rout;
const uint _ideal_reg;
enum projType {
unmatched_proj = 0, // Projs for Control, I/O, memory not matched
fat_proj = 999 // Projs killing many regs, defined by _rout
};
virtual int Opcode() const;
virtual const Type *bottom_type() const;
virtual const TypePtr *adr_type() const;
virtual const RegMask &in_RegMask(uint) const { return RegMask::Empty; }
virtual const RegMask &out_RegMask() const { return _rout; }
virtual uint ideal_reg() const { return _ideal_reg; }
// Need size_of() for virtual ProjNode::clone()
virtual uint size_of() const { return sizeof(MachProjNode); }
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
//------------------------------MachIfNode-------------------------------------
// Machine-specific versions of IfNodes
class MachIfNode : public MachBranchNode {
virtual uint size_of() const { return sizeof(*this); } // Size is bigger
public:
float _prob; // Probability branch goes either way
float _fcnt; // Frequency counter
MachIfNode() : MachBranchNode() {
init_class_id(Class_MachIf);
}
// Negate conditional branches.
virtual void negate() = 0;
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
//------------------------------MachGotoNode-----------------------------------
// Machine-specific versions of GotoNodes
class MachGotoNode : public MachBranchNode {
public:
MachGotoNode() : MachBranchNode() {
init_class_id(Class_MachGoto);
}
};
//------------------------------MachFastLockNode-------------------------------------
// Machine-specific versions of FastLockNodes
class MachFastLockNode : public MachNode {
virtual uint size_of() const { return sizeof(*this); } // Size is bigger
public:
BiasedLockingCounters* _counters;
RTMLockingCounters* _rtm_counters; // RTM lock counters for inflated locks
RTMLockingCounters* _stack_rtm_counters; // RTM lock counters for stack locks
MachFastLockNode() : MachNode() {}
};
//------------------------------MachReturnNode--------------------------------
// Machine-specific versions of subroutine returns
class MachReturnNode : public MachNode {
virtual uint size_of() const; // Size is bigger
public:
RegMask *_in_rms; // Input register masks, set during allocation
ReallocMark _nesting; // assertion check for reallocations
const TypePtr* _adr_type; // memory effects of call or return
MachReturnNode() : MachNode() {
init_class_id(Class_MachReturn);
_adr_type = TypePtr::BOTTOM; // the default: all of memory
}
void set_adr_type(const TypePtr* atp) { _adr_type = atp; }
virtual const RegMask &in_RegMask(uint) const;
virtual bool pinned() const { return true; };
virtual const TypePtr *adr_type() const;
};
//------------------------------MachSafePointNode-----------------------------
// Machine-specific versions of safepoints
class MachSafePointNode : public MachReturnNode {
public:
OopMap* _oop_map; // Array of OopMap info (8-bit char) for GC
JVMState* _jvms; // Pointer to list of JVM State Objects
uint _jvmadj; // Extra delta to jvms indexes (mach. args)
OopMap* oop_map() const { return _oop_map; }
void set_oop_map(OopMap* om) { _oop_map = om; }
MachSafePointNode() : MachReturnNode(), _oop_map(NULL), _jvms(NULL), _jvmadj(0) {
init_class_id(Class_MachSafePoint);
}
virtual JVMState* jvms() const { return _jvms; }
void set_jvms(JVMState* s) {
_jvms = s;
}
virtual const Type *bottom_type() const;
virtual const RegMask &in_RegMask(uint) const;
// Functionality from old debug nodes
Node *returnadr() const { return in(TypeFunc::ReturnAdr); }
Node *frameptr () const { return in(TypeFunc::FramePtr); }
Node *local(const JVMState* jvms, uint idx) const {
assert(verify_jvms(jvms), "jvms must match");
return in(_jvmadj + jvms->locoff() + idx);
}
Node *stack(const JVMState* jvms, uint idx) const {
assert(verify_jvms(jvms), "jvms must match");
return in(_jvmadj + jvms->stkoff() + idx);
}
Node *monitor_obj(const JVMState* jvms, uint idx) const {
assert(verify_jvms(jvms), "jvms must match");
return in(_jvmadj + jvms->monitor_obj_offset(idx));
}
Node *monitor_box(const JVMState* jvms, uint idx) const {
assert(verify_jvms(jvms), "jvms must match");
return in(_jvmadj + jvms->monitor_box_offset(idx));
}
void set_local(const JVMState* jvms, uint idx, Node *c) {
assert(verify_jvms(jvms), "jvms must match");
set_req(_jvmadj + jvms->locoff() + idx, c);
}
void set_stack(const JVMState* jvms, uint idx, Node *c) {
assert(verify_jvms(jvms), "jvms must match");
set_req(_jvmadj + jvms->stkoff() + idx, c);
}
void set_monitor(const JVMState* jvms, uint idx, Node *c) {
assert(verify_jvms(jvms), "jvms must match");
set_req(_jvmadj + jvms->monoff() + idx, c);
}
};
//------------------------------MachCallNode----------------------------------
// Machine-specific versions of subroutine calls
class MachCallNode : public MachSafePointNode {
protected:
virtual uint hash() const { return NO_HASH; } // CFG nodes do not hash
virtual uint cmp( const Node &n ) const;
virtual uint size_of() const = 0; // Size is bigger
public:
const TypeFunc *_tf; // Function type
address _entry_point; // Address of the method being called
float _cnt; // Estimate of number of times called
uint _argsize; // Size of argument block on stack
const TypeFunc* tf() const { return _tf; }
const address entry_point() const { return _entry_point; }
const float cnt() const { return _cnt; }
uint argsize() const { return _argsize; }
void set_tf(const TypeFunc* tf) { _tf = tf; }
void set_entry_point(address p) { _entry_point = p; }
void set_cnt(float c) { _cnt = c; }
void set_argsize(int s) { _argsize = s; }
MachCallNode() : MachSafePointNode() {
init_class_id(Class_MachCall);
}
virtual const Type *bottom_type() const;
virtual bool pinned() const { return false; }
virtual const Type *Value( PhaseTransform *phase ) const;
virtual const RegMask &in_RegMask(uint) const;
virtual int ret_addr_offset() { return 0; }
bool returns_long() const { return tf()->return_type() == T_LONG; }
bool return_value_is_used() const;
// Similar to cousin class CallNode::returns_pointer
bool returns_pointer() const;
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
//------------------------------MachCallJavaNode------------------------------
// "Base" class for machine-specific versions of subroutine calls
class MachCallJavaNode : public MachCallNode {
protected:
virtual uint cmp( const Node &n ) const;
virtual uint size_of() const; // Size is bigger
public:
ciMethod* _method; // Method being direct called
int _bci; // Byte Code index of call byte code
bool _optimized_virtual; // Tells if node is a static call or an optimized virtual
bool _method_handle_invoke; // Tells if the call has to preserve SP
MachCallJavaNode() : MachCallNode() {
init_class_id(Class_MachCallJava);
}
virtual const RegMask &in_RegMask(uint) const;
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
//------------------------------MachCallStaticJavaNode------------------------
// Machine-specific versions of monomorphic subroutine calls
class MachCallStaticJavaNode : public MachCallJavaNode {
virtual uint cmp( const Node &n ) const;
virtual uint size_of() const; // Size is bigger
public:
const char *_name; // Runtime wrapper name
MachCallStaticJavaNode() : MachCallJavaNode() {
init_class_id(Class_MachCallStaticJava);
}
// If this is an uncommon trap, return the request code, else zero.
int uncommon_trap_request() const;
virtual int ret_addr_offset();
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
void dump_trap_args(outputStream *st) const;
#endif
};
//------------------------------MachCallDynamicJavaNode------------------------
// Machine-specific versions of possibly megamorphic subroutine calls
class MachCallDynamicJavaNode : public MachCallJavaNode {
public:
int _vtable_index;
MachCallDynamicJavaNode() : MachCallJavaNode() {
init_class_id(Class_MachCallDynamicJava);
DEBUG_ONLY(_vtable_index = -99); // throw an assert if uninitialized
}
virtual int ret_addr_offset();
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
//------------------------------MachCallRuntimeNode----------------------------
// Machine-specific versions of subroutine calls
class MachCallRuntimeNode : public MachCallNode {
virtual uint cmp( const Node &n ) const;
virtual uint size_of() const; // Size is bigger
public:
const char *_name; // Printable name, if _method is NULL
MachCallRuntimeNode() : MachCallNode() {
init_class_id(Class_MachCallRuntime);
}
virtual int ret_addr_offset();
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
class MachCallLeafNode: public MachCallRuntimeNode {
public:
MachCallLeafNode() : MachCallRuntimeNode() {
init_class_id(Class_MachCallLeaf);
}
};
//------------------------------MachHaltNode-----------------------------------
// Machine-specific versions of halt nodes
class MachHaltNode : public MachReturnNode {
public:
virtual JVMState* jvms() const;
};
//------------------------------MachTempNode-----------------------------------
// Node used by the adlc to construct inputs to represent temporary registers
class MachTempNode : public MachNode {
private:
MachOper *_opnd_array[1];
public:
virtual const RegMask &out_RegMask() const { return *_opnds[0]->in_RegMask(0); }
virtual uint rule() const { return 9999999; }
virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {}
MachTempNode(MachOper* oper) {
init_class_id(Class_MachTemp);
_num_opnds = 1;
_opnds = _opnd_array;
add_req(NULL);
_opnds[0] = oper;
}
virtual uint size_of() const { return sizeof(MachTempNode); }
#ifndef PRODUCT
virtual void format(PhaseRegAlloc *, outputStream *st ) const {}
virtual const char *Name() const { return "MachTemp";}
#endif
};
//------------------------------labelOper--------------------------------------
// Machine-independent version of label operand
class labelOper : public MachOper {
private:
virtual uint num_edges() const { return 0; }
public:
// Supported for fixed size branches
Label* _label; // Label for branch(es)
uint _block_num;
labelOper() : _block_num(0), _label(0) {}
labelOper(Label* label, uint block_num) : _label(label), _block_num(block_num) {}
labelOper(labelOper* l) : _label(l->_label) , _block_num(l->_block_num) {}
virtual MachOper *clone(Compile* C) const;
virtual Label *label() const { assert(_label != NULL, "need Label"); return _label; }
virtual uint opcode() const;
virtual uint hash() const;
virtual uint cmp( const MachOper &oper ) const;
#ifndef PRODUCT
virtual const char *Name() const { return "Label";}
virtual void int_format(PhaseRegAlloc *ra, const MachNode *node, outputStream *st) const;
virtual void ext_format(PhaseRegAlloc *ra, const MachNode *node, int idx, outputStream *st) const { int_format( ra, node, st ); }
#endif
};
//------------------------------methodOper--------------------------------------
// Machine-independent version of method operand
class methodOper : public MachOper {
private:
virtual uint num_edges() const { return 0; }
public:
intptr_t _method; // Address of method
methodOper() : _method(0) {}
methodOper(intptr_t method) : _method(method) {}
virtual MachOper *clone(Compile* C) const;
virtual intptr_t method() const { return _method; }
virtual uint opcode() const;
virtual uint hash() const;
virtual uint cmp( const MachOper &oper ) const;
#ifndef PRODUCT
virtual const char *Name() const { return "Method";}
virtual void int_format(PhaseRegAlloc *ra, const MachNode *node, outputStream *st) const;
virtual void ext_format(PhaseRegAlloc *ra, const MachNode *node, int idx, outputStream *st) const { int_format( ra, node, st ); }
#endif
};
#endif // SHARE_VM_OPTO_MACHNODE_HPP
C:\hotspot-69087d08d473\src\share\vm/opto/macro.cpp
/*
* Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "compiler/compileLog.hpp"
#include "libadt/vectset.hpp"
#include "opto/addnode.hpp"
#include "opto/callnode.hpp"
#include "opto/cfgnode.hpp"
#include "opto/compile.hpp"
#include "opto/connode.hpp"
#include "opto/locknode.hpp"
#include "opto/loopnode.hpp"
#include "opto/macro.hpp"
#include "opto/memnode.hpp"
#include "opto/node.hpp"
#include "opto/phaseX.hpp"
#include "opto/rootnode.hpp"
#include "opto/runtime.hpp"
#include "opto/subnode.hpp"
#include "opto/type.hpp"
#include "runtime/sharedRuntime.hpp"
//
// Replace any references to "oldref" in inputs to "use" with "newref".
// Returns the number of replacements made.
//
int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
int nreplacements = 0;
uint req = use->req();
for (uint j = 0; j < use->len(); j++) {
Node *uin = use->in(j);
if (uin == oldref) {
if (j < req)
use->set_req(j, newref);
else
use->set_prec(j, newref);
nreplacements++;
} else if (j >= req && uin == NULL) {
break;
}
}
return nreplacements;
}
void PhaseMacroExpand::copy_call_debug_info(CallNode *oldcall, CallNode * newcall) {
// Copy debug information and adjust JVMState information
uint old_dbg_start = oldcall->tf()->domain()->cnt();
uint new_dbg_start = newcall->tf()->domain()->cnt();
int jvms_adj = new_dbg_start - old_dbg_start;
assert (new_dbg_start == newcall->req(), "argument count mismatch");
// SafePointScalarObject node could be referenced several times in debug info.
// Use Dict to record cloned nodes.
Dict* sosn_map = new Dict(cmpkey,hashkey);
for (uint i = old_dbg_start; i < oldcall->req(); i++) {
Node* old_in = oldcall->in(i);
// Clone old SafePointScalarObjectNodes, adjusting their field contents.
if (old_in != NULL && old_in->is_SafePointScalarObject()) {
SafePointScalarObjectNode* old_sosn = old_in->as_SafePointScalarObject();
uint old_unique = C->unique();
Node* new_in = old_sosn->clone(sosn_map);
if (old_unique != C->unique()) { // New node?
new_in->set_req(0, C->root()); // reset control edge
new_in = transform_later(new_in); // Register new node.
}
old_in = new_in;
}
newcall->add_req(old_in);
}
newcall->set_jvms(oldcall->jvms());
for (JVMState *jvms = newcall->jvms(); jvms != NULL; jvms = jvms->caller()) {
jvms->set_map(newcall);
jvms->set_locoff(jvms->locoff()+jvms_adj);
jvms->set_stkoff(jvms->stkoff()+jvms_adj);
jvms->set_monoff(jvms->monoff()+jvms_adj);
jvms->set_scloff(jvms->scloff()+jvms_adj);
jvms->set_endoff(jvms->endoff()+jvms_adj);
}
}
Node* PhaseMacroExpand::opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path) {
Node* cmp;
if (mask != 0) {
Node* and_node = transform_later(new (C) AndXNode(word, MakeConX(mask)));
cmp = transform_later(new (C) CmpXNode(and_node, MakeConX(bits)));
} else {
cmp = word;
}
Node* bol = transform_later(new (C) BoolNode(cmp, BoolTest::ne));
IfNode* iff = new (C) IfNode( ctrl, bol, PROB_MIN, COUNT_UNKNOWN );
transform_later(iff);
// Fast path taken.
Node *fast_taken = transform_later( new (C) IfFalseNode(iff) );
// Fast path not-taken, i.e. slow path
Node *slow_taken = transform_later( new (C) IfTrueNode(iff) );
if (return_fast_path) {
region->init_req(edge, slow_taken); // Capture slow-control
return fast_taken;
} else {
region->init_req(edge, fast_taken); // Capture fast-control
return slow_taken;
}
}
//--------------------copy_predefined_input_for_runtime_call--------------------
void PhaseMacroExpand::copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call) {
// Set fixed predefined input arguments
call->init_req( TypeFunc::Control, ctrl );
call->init_req( TypeFunc::I_O , oldcall->in( TypeFunc::I_O) );
call->init_req( TypeFunc::Memory , oldcall->in( TypeFunc::Memory ) ); // ?????
call->init_req( TypeFunc::ReturnAdr, oldcall->in( TypeFunc::ReturnAdr ) );
call->init_req( TypeFunc::FramePtr, oldcall->in( TypeFunc::FramePtr ) );
}
//------------------------------make_slow_call---------------------------------
CallNode* PhaseMacroExpand::make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call, const char* leaf_name, Node* slow_path, Node* parm0, Node* parm1) {
// Slow-path call
CallNode *call = leaf_name
? (CallNode*)new (C) CallLeafNode ( slow_call_type, slow_call, leaf_name, TypeRawPtr::BOTTOM )
: (CallNode*)new (C) CallStaticJavaNode( slow_call_type, slow_call, OptoRuntime::stub_name(slow_call), oldcall->jvms()->bci(), TypeRawPtr::BOTTOM );
// Slow path call has no side-effects, uses few values
copy_predefined_input_for_runtime_call(slow_path, oldcall, call );
if (parm0 != NULL) call->init_req(TypeFunc::Parms+0, parm0);
if (parm1 != NULL) call->init_req(TypeFunc::Parms+1, parm1);
copy_call_debug_info(oldcall, call);
call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON.
_igvn.replace_node(oldcall, call);
transform_later(call);
return call;
}
void PhaseMacroExpand::extract_call_projections(CallNode *call) {
_fallthroughproj = NULL;
_fallthroughcatchproj = NULL;
_ioproj_fallthrough = NULL;
_ioproj_catchall = NULL;
_catchallcatchproj = NULL;
_memproj_fallthrough = NULL;
_memproj_catchall = NULL;
_resproj = NULL;
for (DUIterator_Fast imax, i = call->fast_outs(imax); i < imax; i++) {
ProjNode *pn = call->fast_out(i)->as_Proj();
switch (pn->_con) {
case TypeFunc::Control:
{
// For Control (fallthrough) and I_O (catch_all_index) we have CatchProj -> Catch -> Proj
_fallthroughproj = pn;
DUIterator_Fast jmax, j = pn->fast_outs(jmax);
const Node *cn = pn->fast_out(j);
if (cn->is_Catch()) {
ProjNode *cpn = NULL;
for (DUIterator_Fast kmax, k = cn->fast_outs(kmax); k < kmax; k++) {
cpn = cn->fast_out(k)->as_Proj();
assert(cpn->is_CatchProj(), "must be a CatchProjNode");
if (cpn->_con == CatchProjNode::fall_through_index)
_fallthroughcatchproj = cpn;
else {
assert(cpn->_con == CatchProjNode::catch_all_index, "must be correct index.");
_catchallcatchproj = cpn;
}
}
}
break;
}
case TypeFunc::I_O:
if (pn->_is_io_use)
_ioproj_catchall = pn;
else
_ioproj_fallthrough = pn;
break;
case TypeFunc::Memory:
if (pn->_is_io_use)
_memproj_catchall = pn;
else
_memproj_fallthrough = pn;
break;
case TypeFunc::Parms:
_resproj = pn;
break;
default:
assert(false, "unexpected projection from allocation node.");
}
}
}
// Eliminate a card mark sequence. p2x is a ConvP2XNode
void PhaseMacroExpand::eliminate_card_mark(Node* p2x) {
assert(p2x->Opcode() == Op_CastP2X, "ConvP2XNode required");
if (!UseG1GC) {
// vanilla/CMS post barrier
Node *shift = p2x->unique_out();
Node *addp = shift->unique_out();
for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
Node *mem = addp->last_out(j);
if (UseCondCardMark && mem->is_Load()) {
assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
// The load is checking if the card has been written so
// replace it with zero to fold the test.
_igvn.replace_node(mem, intcon(0));
continue;
}
assert(mem->is_Store(), "store required");
_igvn.replace_node(mem, mem->in(MemNode::Memory));
}
} else {
// G1 pre/post barriers
assert(p2x->outcnt() <= 2, "expects 1 or 2 users: Xor and URShift nodes");
// It could be only one user, URShift node, in Object.clone() instrinsic
// but the new allocation is passed to arraycopy stub and it could not
// be scalar replaced. So we don't check the case.
// An other case of only one user (Xor) is when the value check for NULL
// in G1 post barrier is folded after CCP so the code which used URShift
// is removed.
// Take Region node before eliminating post barrier since it also
// eliminates CastP2X node when it has only one user.
Node* this_region = p2x->in(0);
assert(this_region != NULL, "");
// Remove G1 post barrier.
// Search for CastP2X->Xor->URShift->Cmp path which
// checks if the store done to a different from the value's region.
// And replace Cmp with #0 (false) to collapse G1 post barrier.
Node* xorx = NULL;
for (DUIterator_Fast imax, i = p2x->fast_outs(imax); i < imax; i++) {
Node* u = p2x->fast_out(i);
if (u->Opcode() == Op_XorX) {
xorx = u;
break;
}
}
assert(xorx != NULL, "missing G1 post barrier");
Node* shift = xorx->unique_out();
Node* cmpx = shift->unique_out();
assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
"missing region check in G1 post barrier");
_igvn.replace_node(cmpx, makecon(TypeInt::CC_EQ));
// Remove G1 pre barrier.
// Search "if (marking != 0)" check and set it to "false".
// There is no G1 pre barrier if previous stored value is NULL
// (for example, after initialization).
if (this_region->is_Region() && this_region->req() == 3) {
int ind = 1;
if (!this_region->in(ind)->is_IfFalse()) {
ind = 2;
}
if (this_region->in(ind)->is_IfFalse()) {
Node* bol = this_region->in(ind)->in(0)->in(1);
assert(bol->is_Bool(), "");
cmpx = bol->in(1);
if (bol->as_Bool()->_test._test == BoolTest::ne &&
cmpx->is_Cmp() && cmpx->in(2) == intcon(0) &&
cmpx->in(1)->is_Load()) {
Node* adr = cmpx->in(1)->as_Load()->in(MemNode::Address);
const int marking_offset = in_bytes(JavaThread::satb_mark_queue_offset() +
PtrQueue::byte_offset_of_active());
if (adr->is_AddP() && adr->in(AddPNode::Base) == top() &&
adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
adr->in(AddPNode::Offset) == MakeConX(marking_offset)) {
_igvn.replace_node(cmpx, makecon(TypeInt::CC_EQ));
}
}
}
}
// Now CastP2X can be removed since it is used only on dead path
// which currently still alive until igvn optimize it.
assert(p2x->outcnt() == 0 || p2x->unique_out()->Opcode() == Op_URShiftX, "");
_igvn.replace_node(p2x, top());
}
}
// Search for a memory operation for the specified memory slice.
static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_mem, Node *alloc, PhaseGVN *phase) {
Node *orig_mem = mem;
Node *alloc_mem = alloc->in(TypeFunc::Memory);
const TypeOopPtr *tinst = phase->C->get_adr_type(alias_idx)->isa_oopptr();
while (true) {
if (mem == alloc_mem || mem == start_mem ) {
return mem; // hit one of our sentinels
} else if (mem->is_MergeMem()) {
mem = mem->as_MergeMem()->memory_at(alias_idx);
} else if (mem->is_Proj() && mem->as_Proj()->_con == TypeFunc::Memory) {
Node *in = mem->in(0);
// we can safely skip over safepoints, calls, locks and membars because we
// already know that the object is safe to eliminate.
if (in->is_Initialize() && in->as_Initialize()->allocation() == alloc) {
return in;
} else if (in->is_Call()) {
CallNode *call = in->as_Call();
if (!call->may_modify(tinst, phase)) {
mem = call->in(TypeFunc::Memory);
}
mem = in->in(TypeFunc::Memory);
} else if (in->is_MemBar()) {
mem = in->in(TypeFunc::Memory);
} else {
assert(false, "unexpected projection");
}
} else if (mem->is_Store()) {
const TypePtr* atype = mem->as_Store()->adr_type();
int adr_idx = Compile::current()->get_alias_index(atype);
if (adr_idx == alias_idx) {
assert(atype->isa_oopptr(), "address type must be oopptr");
int adr_offset = atype->offset();
uint adr_iid = atype->is_oopptr()->instance_id();
// Array elements references have the same alias_idx
// but different offset and different instance_id.
if (adr_offset == offset && adr_iid == alloc->_idx)
return mem;
} else {
assert(adr_idx == Compile::AliasIdxRaw, "address must match or be raw");
}
mem = mem->in(MemNode::Memory);
} else if (mem->is_ClearArray()) {
if (!ClearArrayNode::step_through(&mem, alloc->_idx, phase)) {
// Can not bypass initialization of the instance
// we are looking.
debug_only(intptr_t offset;)
assert(alloc == AllocateNode::Ideal_allocation(mem->in(3), phase, offset), "sanity");
InitializeNode* init = alloc->as_Allocate()->initialization();
// We are looking for stored value, return Initialize node
// or memory edge from Allocate node.
if (init != NULL)
return init;
else
return alloc->in(TypeFunc::Memory); // It will produce zero value (see callers).
}
// Otherwise skip it (the call updated 'mem' value).
} else if (mem->Opcode() == Op_SCMemProj) {
mem = mem->in(0);
Node* adr = NULL;
if (mem->is_LoadStore()) {
adr = mem->in(MemNode::Address);
} else {
assert(mem->Opcode() == Op_EncodeISOArray, "sanity");
adr = mem->in(3); // Destination array
}
const TypePtr* atype = adr->bottom_type()->is_ptr();
int adr_idx = Compile::current()->get_alias_index(atype);
if (adr_idx == alias_idx) {
assert(false, "Object is not scalar replaceable if a LoadStore node access its field");
return NULL;
}
mem = mem->in(MemNode::Memory);
} else {
return mem;
}
assert(mem != orig_mem, "dead memory loop");
}
}
//
// Given a Memory Phi, compute a value Phi containing the values from stores
// on the input paths.
// Note: this function is recursive, its depth is limied by the "level" argument
// Returns the computed Phi, or NULL if it cannot compute it.
Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *phi_type, const TypeOopPtr *adr_t, Node *alloc, Node_Stack *value_phis, int level) {
assert(mem->is_Phi(), "sanity");
int alias_idx = C->get_alias_index(adr_t);
int offset = adr_t->offset();
int instance_id = adr_t->instance_id();
// Check if an appropriate value phi already exists.
Node* region = mem->in(0);
for (DUIterator_Fast kmax, k = region->fast_outs(kmax); k < kmax; k++) {
Node* phi = region->fast_out(k);
if (phi->is_Phi() && phi != mem &&
phi->as_Phi()->is_same_inst_field(phi_type, (int)mem->_idx, instance_id, alias_idx, offset)) {
return phi;
}
}
// Check if an appropriate new value phi already exists.
Node* new_phi = value_phis->find(mem->_idx);
if (new_phi != NULL)
return new_phi;
if (level <= 0) {
return NULL; // Give up: phi tree too deep
}
Node *start_mem = C->start()->proj_out(TypeFunc::Memory);
Node *alloc_mem = alloc->in(TypeFunc::Memory);
uint length = mem->req();
GrowableArray <Node *> values(length, length, NULL, false);
// create a new Phi for the value
PhiNode *phi = new (C) PhiNode(mem->in(0), phi_type, NULL, mem->_idx, instance_id, alias_idx, offset);
transform_later(phi);
value_phis->push(phi, mem->_idx);
for (uint j = 1; j < length; j++) {
Node *in = mem->in(j);
if (in == NULL || in->is_top()) {
values.at_put(j, in);
} else {
Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn);
if (val == start_mem || val == alloc_mem) {
// hit a sentinel, return appropriate 0 value
values.at_put(j, _igvn.zerocon(ft));
continue;
}
if (val->is_Initialize()) {
val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
}
if (val == NULL) {
return NULL; // can't find a value on this path
}
if (val == mem) {
values.at_put(j, mem);
} else if (val->is_Store()) {
values.at_put(j, val->in(MemNode::ValueIn));
} else if(val->is_Proj() && val->in(0) == alloc) {
values.at_put(j, _igvn.zerocon(ft));
} else if (val->is_Phi()) {
val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1);
if (val == NULL) {
return NULL;
}
values.at_put(j, val);
} else if (val->Opcode() == Op_SCMemProj) {
assert(val->in(0)->is_LoadStore() || val->in(0)->Opcode() == Op_EncodeISOArray, "sanity");
assert(false, "Object is not scalar replaceable if a LoadStore node access its field");
return NULL;
} else {
#ifdef ASSERT
val->dump();
assert(false, "unknown node on this path");
#endif
return NULL; // unknown node on this path
}
}
}
// Set Phi's inputs
for (uint j = 1; j < length; j++) {
if (values.at(j) == mem) {
phi->init_req(j, phi);
} else {
phi->init_req(j, values.at(j));
}
}
return phi;
}
// Search the last value stored into the object's field.
Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, Node *alloc) {
assert(adr_t->is_known_instance_field(), "instance required");
int instance_id = adr_t->instance_id();
assert((uint)instance_id == alloc->_idx, "wrong allocation");
int alias_idx = C->get_alias_index(adr_t);
int offset = adr_t->offset();
Node *start_mem = C->start()->proj_out(TypeFunc::Memory);
Node *alloc_ctrl = alloc->in(TypeFunc::Control);
Node *alloc_mem = alloc->in(TypeFunc::Memory);
Arena *a = Thread::current()->resource_area();
VectorSet visited(a);
bool done = sfpt_mem == alloc_mem;
Node *mem = sfpt_mem;
while (!done) {
if (visited.test_set(mem->_idx)) {
return NULL; // found a loop, give up
}
mem = scan_mem_chain(mem, alias_idx, offset, start_mem, alloc, &_igvn);
if (mem == start_mem || mem == alloc_mem) {
done = true; // hit a sentinel, return appropriate 0 value
} else if (mem->is_Initialize()) {
mem = mem->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
if (mem == NULL) {
done = true; // Something go wrong.
} else if (mem->is_Store()) {
const TypePtr* atype = mem->as_Store()->adr_type();
assert(C->get_alias_index(atype) == Compile::AliasIdxRaw, "store is correct memory slice");
done = true;
}
} else if (mem->is_Store()) {
const TypeOopPtr* atype = mem->as_Store()->adr_type()->isa_oopptr();
assert(atype != NULL, "address type must be oopptr");
assert(C->get_alias_index(atype) == alias_idx &&
atype->is_known_instance_field() && atype->offset() == offset &&
atype->instance_id() == instance_id, "store is correct memory slice");
done = true;
} else if (mem->is_Phi()) {
// try to find a phi's unique input
Node *unique_input = NULL;
Node *top = C->top();
for (uint i = 1; i < mem->req(); i++) {
Node *n = scan_mem_chain(mem->in(i), alias_idx, offset, start_mem, alloc, &_igvn);
if (n == NULL || n == top || n == mem) {
continue;
} else if (unique_input == NULL) {
unique_input = n;
} else if (unique_input != n) {
unique_input = top;
break;
}
}
if (unique_input != NULL && unique_input != top) {
mem = unique_input;
} else {
done = true;
}
} else {
assert(false, "unexpected node");
}
}
if (mem != NULL) {
if (mem == start_mem || mem == alloc_mem) {
// hit a sentinel, return appropriate 0 value
return _igvn.zerocon(ft);
} else if (mem->is_Store()) {
return mem->in(MemNode::ValueIn);
} else if (mem->is_Phi()) {
// attempt to produce a Phi reflecting the values on the input paths of the Phi
Node_Stack value_phis(a, 8);
Node * phi = value_from_mem_phi(mem, ft, ftype, adr_t, alloc, &value_phis, ValueSearchLimit);
if (phi != NULL) {
return phi;
} else {
// Kill all new Phis
while(value_phis.is_nonempty()) {
Node* n = value_phis.node();
_igvn.replace_node(n, C->top());
value_phis.pop();
}
}
}
}
// Something go wrong.
return NULL;
}
// Check the possibility of scalar replacement.
bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints) {
// Scan the uses of the allocation to check for anything that would
// prevent us from eliminating it.
NOT_PRODUCT( const char* fail_eliminate = NULL; )
DEBUG_ONLY( Node* disq_node = NULL; )
bool can_eliminate = true;
Node* res = alloc->result_cast();
const TypeOopPtr* res_type = NULL;
if (res == NULL) {
// All users were eliminated.
} else if (!res->is_CheckCastPP()) {
NOT_PRODUCT(fail_eliminate = "Allocation does not have unique CheckCastPP";)
can_eliminate = false;
} else {
res_type = _igvn.type(res)->isa_oopptr();
if (res_type == NULL) {
NOT_PRODUCT(fail_eliminate = "Neither instance or array allocation";)
can_eliminate = false;
} else if (res_type->isa_aryptr()) {
int length = alloc->in(AllocateNode::ALength)->find_int_con(-1);
if (length < 0) {
NOT_PRODUCT(fail_eliminate = "Array's size is not constant";)
can_eliminate = false;
}
}
}
if (can_eliminate && res != NULL) {
for (DUIterator_Fast jmax, j = res->fast_outs(jmax);
j < jmax && can_eliminate; j++) {
Node* use = res->fast_out(j);
if (use->is_AddP()) {
const TypePtr* addp_type = _igvn.type(use)->is_ptr();
int offset = addp_type->offset();
if (offset == Type::OffsetTop || offset == Type::OffsetBot) {
NOT_PRODUCT(fail_eliminate = "Undefined field referrence";)
can_eliminate = false;
break;
}
for (DUIterator_Fast kmax, k = use->fast_outs(kmax);
k < kmax && can_eliminate; k++) {
Node* n = use->fast_out(k);
if (!n->is_Store() && n->Opcode() != Op_CastP2X) {
DEBUG_ONLY(disq_node = n;)
if (n->is_Load() || n->is_LoadStore()) {
NOT_PRODUCT(fail_eliminate = "Field load";)
} else {
NOT_PRODUCT(fail_eliminate = "Not store field referrence";)
}
can_eliminate = false;
}
}
} else if (use->is_SafePoint()) {
SafePointNode* sfpt = use->as_SafePoint();
if (sfpt->is_Call() && sfpt->as_Call()->has_non_debug_use(res)) {
// Object is passed as argument.
DEBUG_ONLY(disq_node = use;)
NOT_PRODUCT(fail_eliminate = "Object is passed as argument";)
can_eliminate = false;
}
Node* sfptMem = sfpt->memory();
if (sfptMem == NULL || sfptMem->is_top()) {
DEBUG_ONLY(disq_node = use;)
NOT_PRODUCT(fail_eliminate = "NULL or TOP memory";)
can_eliminate = false;
} else {
safepoints.append_if_missing(sfpt);
}
} else if (use->Opcode() != Op_CastP2X) { // CastP2X is used by card mark
if (use->is_Phi()) {
if (use->outcnt() == 1 && use->unique_out()->Opcode() == Op_Return) {
NOT_PRODUCT(fail_eliminate = "Object is return value";)
} else {
NOT_PRODUCT(fail_eliminate = "Object is referenced by Phi";)
}
DEBUG_ONLY(disq_node = use;)
} else {
if (use->Opcode() == Op_Return) {
NOT_PRODUCT(fail_eliminate = "Object is return value";)
}else {
NOT_PRODUCT(fail_eliminate = "Object is referenced by node";)
}
DEBUG_ONLY(disq_node = use;)
}
can_eliminate = false;
}
}
}
#ifndef PRODUCT
if (PrintEliminateAllocations) {
if (can_eliminate) {
tty->print("Scalar ");
if (res == NULL)
alloc->dump();
else
res->dump();
} else if (alloc->_is_scalar_replaceable) {
tty->print("NotScalar (%s)", fail_eliminate);
if (res == NULL)
alloc->dump();
else
res->dump();
#ifdef ASSERT
if (disq_node != NULL) {
tty->print(" >>>> ");
disq_node->dump();
}
#endif /*ASSERT*/
}
}
#endif
return can_eliminate;
}
// Do scalar replacement.
bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints) {
GrowableArray <SafePointNode *> safepoints_done;
ciKlass* klass = NULL;
ciInstanceKlass* iklass = NULL;
int nfields = 0;
int array_base = 0;
int element_size = 0;
BasicType basic_elem_type = T_ILLEGAL;
ciType* elem_type = NULL;
Node* res = alloc->result_cast();
assert(res == NULL || res->is_CheckCastPP(), "unexpected AllocateNode result");
const TypeOopPtr* res_type = NULL;
if (res != NULL) { // Could be NULL when there are no users
res_type = _igvn.type(res)->isa_oopptr();
}
if (res != NULL) {
klass = res_type->klass();
if (res_type->isa_instptr()) {
// find the fields of the class which will be needed for safepoint debug information
assert(klass->is_instance_klass(), "must be an instance klass.");
iklass = klass->as_instance_klass();
nfields = iklass->nof_nonstatic_fields();
} else {
// find the array's elements which will be needed for safepoint debug information
nfields = alloc->in(AllocateNode::ALength)->find_int_con(-1);
assert(klass->is_array_klass() && nfields >= 0, "must be an array klass.");
elem_type = klass->as_array_klass()->element_type();
basic_elem_type = elem_type->basic_type();
array_base = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
element_size = type2aelembytes(basic_elem_type);
}
}
//
// Process the safepoint uses
//
while (safepoints.length() > 0) {
SafePointNode* sfpt = safepoints.pop();
Node* mem = sfpt->memory();
assert(sfpt->jvms() != NULL, "missed JVMS");
// Fields of scalar objs are referenced only at the end
// of regular debuginfo at the last (youngest) JVMS.
// Record relative start index.
uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
SafePointScalarObjectNode* sobj = new (C) SafePointScalarObjectNode(res_type,
#ifdef ASSERT
alloc,
#endif
first_ind, nfields);
sobj->init_req(0, C->root());
transform_later(sobj);
// Scan object's fields adding an input to the safepoint for each field.
for (int j = 0; j < nfields; j++) {
intptr_t offset;
ciField* field = NULL;
if (iklass != NULL) {
field = iklass->nonstatic_field_at(j);
offset = field->offset();
elem_type = field->type();
basic_elem_type = field->layout_type();
} else {
offset = array_base + j * (intptr_t)element_size;
}
const Type *field_type;
// The next code is taken from Parse::do_get_xxx().
if (basic_elem_type == T_OBJECT || basic_elem_type == T_ARRAY) {
if (!elem_type->is_loaded()) {
field_type = TypeInstPtr::BOTTOM;
} else if (field != NULL && field->is_constant() && field->is_static()) {
// This can happen if the constant oop is non-perm.
ciObject* con = field->constant_value().as_object();
// Do not "join" in the previous type; it doesn't add value,
// and may yield a vacuous result if the field is of interface type.
field_type = TypeOopPtr::make_from_constant(con)->isa_oopptr();
assert(field_type != NULL, "field singleton type must be consistent");
} else {
field_type = TypeOopPtr::make_from_klass(elem_type->as_klass());
}
if (UseCompressedOops) {
field_type = field_type->make_narrowoop();
basic_elem_type = T_NARROWOOP;
}
} else {
field_type = Type::get_const_basic_type(basic_elem_type);
}
const TypeOopPtr *field_addr_type = res_type->add_offset(offset)->isa_oopptr();
Node *field_val = value_from_mem(mem, basic_elem_type, field_type, field_addr_type, alloc);
if (field_val == NULL) {
// We weren't able to find a value for this field,
// give up on eliminating this allocation.
// Remove any extra entries we added to the safepoint.
uint last = sfpt->req() - 1;
for (int k = 0; k < j; k++) {
sfpt->del_req(last--);
}
// rollback processed safepoints
while (safepoints_done.length() > 0) {
SafePointNode* sfpt_done = safepoints_done.pop();
// remove any extra entries we added to the safepoint
last = sfpt_done->req() - 1;
for (int k = 0; k < nfields; k++) {
sfpt_done->del_req(last--);
}
JVMState *jvms = sfpt_done->jvms();
jvms->set_endoff(sfpt_done->req());
// Now make a pass over the debug information replacing any references
// to SafePointScalarObjectNode with the allocated object.
int start = jvms->debug_start();
int end = jvms->debug_end();
for (int i = start; i < end; i++) {
if (sfpt_done->in(i)->is_SafePointScalarObject()) {
SafePointScalarObjectNode* scobj = sfpt_done->in(i)->as_SafePointScalarObject();
if (scobj->first_index(jvms) == sfpt_done->req() &&
scobj->n_fields() == (uint)nfields) {
assert(scobj->alloc() == alloc, "sanity");
sfpt_done->set_req(i, res);
}
}
}
}
#ifndef PRODUCT
if (PrintEliminateAllocations) {
if (field != NULL) {
tty->print("=== At SafePoint node %d can't find value of Field: ",
sfpt->_idx);
field->print();
int field_idx = C->get_alias_index(field_addr_type);
tty->print(" (alias_idx=%d)", field_idx);
} else { // Array's element
tty->print("=== At SafePoint node %d can't find value of array element [%d]",
sfpt->_idx, j);
}
tty->print(", which prevents elimination of: ");
if (res == NULL)
alloc->dump();
else
res->dump();
}
#endif
return false;
}
if (UseCompressedOops && field_type->isa_narrowoop()) {
// Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
// to be able scalar replace the allocation.
if (field_val->is_EncodeP()) {
field_val = field_val->in(1);
} else {
field_val = transform_later(new (C) DecodeNNode(field_val, field_val->get_ptr_type()));
}
}
sfpt->add_req(field_val);
}
JVMState *jvms = sfpt->jvms();
jvms->set_endoff(sfpt->req());
// Now make a pass over the debug information replacing any references
// to the allocated object with "sobj"
int start = jvms->debug_start();
int end = jvms->debug_end();
sfpt->replace_edges_in_range(res, sobj, start, end);
safepoints_done.append_if_missing(sfpt); // keep it for rollback
}
return true;
}
// Process users of eliminated allocation.
void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
Node* res = alloc->result_cast();
if (res != NULL) {
for (DUIterator_Last jmin, j = res->last_outs(jmin); j >= jmin; ) {
Node *use = res->last_out(j);
uint oc1 = res->outcnt();
if (use->is_AddP()) {
for (DUIterator_Last kmin, k = use->last_outs(kmin); k >= kmin; ) {
Node *n = use->last_out(k);
uint oc2 = use->outcnt();
if (n->is_Store()) {
#ifdef ASSERT
// Verify that there is no dependent MemBarVolatile nodes,
// they should be removed during IGVN, see MemBarNode::Ideal().
for (DUIterator_Fast pmax, p = n->fast_outs(pmax);
p < pmax; p++) {
Node* mb = n->fast_out(p);
assert(mb->is_Initialize() || !mb->is_MemBar() ||
mb->req() <= MemBarNode::Precedent ||
mb->in(MemBarNode::Precedent) != n,
"MemBarVolatile should be eliminated for non-escaping object");
}
#endif
_igvn.replace_node(n, n->in(MemNode::Memory));
} else {
eliminate_card_mark(n);
}
k -= (oc2 - use->outcnt());
}
} else {
eliminate_card_mark(use);
}
j -= (oc1 - res->outcnt());
}
assert(res->outcnt() == 0, "all uses of allocated objects must be deleted");
_igvn.remove_dead_node(res);
}
//
// Process other users of allocation's projections
//
if (_resproj != NULL && _resproj->outcnt() != 0) {
// First disconnect stores captured by Initialize node.
// If Initialize node is eliminated first in the following code,
// it will kill such stores and DUIterator_Last will assert.
for (DUIterator_Fast jmax, j = _resproj->fast_outs(jmax); j < jmax; j++) {
Node *use = _resproj->fast_out(j);
if (use->is_AddP()) {
// raw memory addresses used only by the initialization
_igvn.replace_node(use, C->top());
--j; --jmax;
}
}
for (DUIterator_Last jmin, j = _resproj->last_outs(jmin); j >= jmin; ) {
Node *use = _resproj->last_out(j);
uint oc1 = _resproj->outcnt();
if (use->is_Initialize()) {
// Eliminate Initialize node.
InitializeNode *init = use->as_Initialize();
assert(init->outcnt() <= 2, "only a control and memory projection expected");
Node *ctrl_proj = init->proj_out(TypeFunc::Control);
if (ctrl_proj != NULL) {
assert(init->in(TypeFunc::Control) == _fallthroughcatchproj, "allocation control projection");
_igvn.replace_node(ctrl_proj, _fallthroughcatchproj);
}
Node *mem_proj = init->proj_out(TypeFunc::Memory);
if (mem_proj != NULL) {
Node *mem = init->in(TypeFunc::Memory);
#ifdef ASSERT
if (mem->is_MergeMem()) {
assert(mem->in(TypeFunc::Memory) == _memproj_fallthrough, "allocation memory projection");
} else {
assert(mem == _memproj_fallthrough, "allocation memory projection");
}
#endif
_igvn.replace_node(mem_proj, mem);
}
} else {
assert(false, "only Initialize or AddP expected");
}
j -= (oc1 - _resproj->outcnt());
}
}
if (_fallthroughcatchproj != NULL) {
_igvn.replace_node(_fallthroughcatchproj, alloc->in(TypeFunc::Control));
}
if (_memproj_fallthrough != NULL) {
_igvn.replace_node(_memproj_fallthrough, alloc->in(TypeFunc::Memory));
}
if (_memproj_catchall != NULL) {
_igvn.replace_node(_memproj_catchall, C->top());
}
if (_ioproj_fallthrough != NULL) {
_igvn.replace_node(_ioproj_fallthrough, alloc->in(TypeFunc::I_O));
}
if (_ioproj_catchall != NULL) {
_igvn.replace_node(_ioproj_catchall, C->top());
}
if (_catchallcatchproj != NULL) {
_igvn.replace_node(_catchallcatchproj, C->top());
}
}
bool PhaseMacroExpand::eliminate_allocate_node(AllocateNode *alloc) {
// Don't do scalar replacement if the frame can be popped by JVMTI:
// if reallocation fails during deoptimization we'll pop all
// interpreter frames for this compiled frame and that won't play
// nice with JVMTI popframe.
if (!EliminateAllocations || JvmtiExport::can_pop_frame() || !alloc->_is_non_escaping) {
return false;
}
Node* klass = alloc->in(AllocateNode::KlassNode);
const TypeKlassPtr* tklass = _igvn.type(klass)->is_klassptr();
Node* res = alloc->result_cast();
// Eliminate boxing allocations which are not used
// regardless scalar replacable status.
bool boxing_alloc = C->eliminate_boxing() &&
tklass->klass()->is_instance_klass() &&
tklass->klass()->as_instance_klass()->is_box_klass();
if (!alloc->_is_scalar_replaceable && (!boxing_alloc || (res != NULL))) {
return false;
}
extract_call_projections(alloc);
GrowableArray <SafePointNode *> safepoints;
if (!can_eliminate_allocation(alloc, safepoints)) {
return false;
}
if (!alloc->_is_scalar_replaceable) {
assert(res == NULL, "sanity");
// We can only eliminate allocation if all debug info references
// are already replaced with SafePointScalarObject because
// we can't search for a fields value without instance_id.
if (safepoints.length() > 0) {
return false;
}
}
if (!scalar_replacement(alloc, safepoints)) {
return false;
}
CompileLog* log = C->log();
if (log != NULL) {
log->head("eliminate_allocation type='%d'",
log->identify(tklass->klass()));
JVMState* p = alloc->jvms();
while (p != NULL) {
log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method()));
p = p->caller();
}
log->tail("eliminate_allocation");
}
process_users_of_allocation(alloc);
#ifndef PRODUCT
if (PrintEliminateAllocations) {
if (alloc->is_AllocateArray())
tty->print_cr("++++ Eliminated: %d AllocateArray", alloc->_idx);
else
tty->print_cr("++++ Eliminated: %d Allocate", alloc->_idx);
}
#endif
return true;
}
bool PhaseMacroExpand::eliminate_boxing_node(CallStaticJavaNode *boxing) {
// EA should remove all uses of non-escaping boxing node.
if (!C->eliminate_boxing() || boxing->proj_out(TypeFunc::Parms) != NULL) {
return false;
}
assert(boxing->result_cast() == NULL, "unexpected boxing node result");
extract_call_projections(boxing);
const TypeTuple* r = boxing->tf()->range();
assert(r->cnt() > TypeFunc::Parms, "sanity");
const TypeInstPtr* t = r->field_at(TypeFunc::Parms)->isa_instptr();
assert(t != NULL, "sanity");
CompileLog* log = C->log();
if (log != NULL) {
log->head("eliminate_boxing type='%d'",
log->identify(t->klass()));
JVMState* p = boxing->jvms();
while (p != NULL) {
log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method()));
p = p->caller();
}
log->tail("eliminate_boxing");
}
process_users_of_allocation(boxing);
#ifndef PRODUCT
if (PrintEliminateAllocations) {
tty->print("++++ Eliminated: %d ", boxing->_idx);
boxing->method()->print_short_name(tty);
tty->cr();
}
#endif
return true;
}
//---------------------------set_eden_pointers-------------------------
void PhaseMacroExpand::set_eden_pointers(Node* &eden_top_adr, Node* &eden_end_adr) {
if (UseTLAB) { // Private allocation: load from TLS
Node* thread = transform_later(new (C) ThreadLocalNode());
int tlab_top_offset = in_bytes(JavaThread::tlab_top_offset());
int tlab_end_offset = in_bytes(JavaThread::tlab_end_offset());
eden_top_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_top_offset);
eden_end_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_end_offset);
} else { // Shared allocation: load from globals
CollectedHeap* ch = Universe::heap();
address top_adr = (address)ch->top_addr();
address end_adr = (address)ch->end_addr();
eden_top_adr = makecon(TypeRawPtr::make(top_adr));
eden_end_adr = basic_plus_adr(eden_top_adr, end_adr - top_adr);
}
}
Node* PhaseMacroExpand::make_load(Node* ctl, Node* mem, Node* base, int offset, const Type* value_type, BasicType bt) {
Node* adr = basic_plus_adr(base, offset);
const TypePtr* adr_type = adr->bottom_type()->is_ptr();
Node* value = LoadNode::make(_igvn, ctl, mem, adr, adr_type, value_type, bt, MemNode::unordered);
transform_later(value);
return value;
}
Node* PhaseMacroExpand::make_store(Node* ctl, Node* mem, Node* base, int offset, Node* value, BasicType bt) {
Node* adr = basic_plus_adr(base, offset);
mem = StoreNode::make(_igvn, ctl, mem, adr, NULL, value, bt, MemNode::unordered);
transform_later(mem);
return mem;
}
//=============================================================================
//
// A L L O C A T I O N
//
// Allocation attempts to be fast in the case of frequent small objects.
// It breaks down like this:
//
// 1) Size in doublewords is computed. This is a constant for objects and
// variable for most arrays. Doubleword units are used to avoid size
// overflow of huge doubleword arrays. We need doublewords in the end for
// rounding.
//
// 2) Size is checked for being 'too large'. Too-large allocations will go
// the slow path into the VM. The slow path can throw any required
// exceptions, and does all the special checks for very large arrays. The
// size test can constant-fold away for objects. For objects with
// finalizers it constant-folds the otherway: you always go slow with
// finalizers.
//
// 3) If NOT using TLABs, this is the contended loop-back point.
// Load-Locked the heap top. If using TLABs normal-load the heap top.
//
// 4) Check that heap top + size*8 < max. If we fail go the slow ` route.
// NOTE: "top+size*8" cannot wrap the 4Gig line! Here's why: for largish
// "size*8" we always enter the VM, where "largish" is a constant picked small
// enough that there's always space between the eden max and 4Gig (old space is
// there so it's quite large) and large enough that the cost of entering the VM
// is dwarfed by the cost to initialize the space.
//
// 5) If NOT using TLABs, Store-Conditional the adjusted heap top back
// down. If contended, repeat at step 3. If using TLABs normal-store
// adjusted heap top back down; there is no contention.
//
// 6) If !ZeroTLAB then Bulk-clear the object/array. Fill in klass & mark
// fields.
//
// 7) Merge with the slow-path; cast the raw memory pointer to the correct
// oop flavor.
//
//=============================================================================
// FastAllocateSizeLimit value is in DOUBLEWORDS.
// Allocations bigger than this always go the slow route.
// This value must be small enough that allocation attempts that need to
// trigger exceptions go the slow route. Also, it must be small enough so
// that heap_top + size_in_bytes does not wrap around the 4Gig limit.
//=============================================================================j//
// %%% Here is an old comment from parseHelper.cpp; is it outdated?
// The allocator will coalesce int->oop copies away. See comment in
// coalesce.cpp about how this works. It depends critically on the exact
// code shape produced here, so if you are changing this code shape
// make sure the GC info for the heap-top is correct in and around the
// slow-path call.
//
void PhaseMacroExpand::expand_allocate_common(
AllocateNode* alloc, // allocation node to be expanded
Node* length, // array length for an array allocation
const TypeFunc* slow_call_type, // Type of slow call
address slow_call_address // Address of slow call
)
{
Node* ctrl = alloc->in(TypeFunc::Control);
Node* mem = alloc->in(TypeFunc::Memory);
Node* i_o = alloc->in(TypeFunc::I_O);
Node* size_in_bytes = alloc->in(AllocateNode::AllocSize);
Node* klass_node = alloc->in(AllocateNode::KlassNode);
Node* initial_slow_test = alloc->in(AllocateNode::InitialTest);
assert(ctrl != NULL, "must have control");
// We need a Region and corresponding Phi's to merge the slow-path and fast-path results.
// they will not be used if "always_slow" is set
enum { slow_result_path = 1, fast_result_path = 2 };
Node *result_region = NULL;
Node *result_phi_rawmem = NULL;
Node *result_phi_rawoop = NULL;
Node *result_phi_i_o = NULL;
// The initial slow comparison is a size check, the comparison
// we want to do is a BoolTest::gt
bool always_slow = false;
int tv = _igvn.find_int_con(initial_slow_test, -1);
if (tv >= 0) {
always_slow = (tv == 1);
initial_slow_test = NULL;
} else {
initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn);
}
if (C->env()->dtrace_alloc_probes() ||
!UseTLAB && (!Universe::heap()->supports_inline_contig_alloc() ||
(UseConcMarkSweepGC && CMSIncrementalMode))) {
// Force slow-path allocation
always_slow = true;
initial_slow_test = NULL;
}
enum { too_big_or_final_path = 1, need_gc_path = 2 };
Node *slow_region = NULL;
Node *toobig_false = ctrl;
assert (initial_slow_test == NULL || !always_slow, "arguments must be consistent");
// generate the initial test if necessary
if (initial_slow_test != NULL ) {
slow_region = new (C) RegionNode(3);
// Now make the initial failure test. Usually a too-big test but
// might be a TRUE for finalizers or a fancy class check for
// newInstance0.
IfNode *toobig_iff = new (C) IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN);
transform_later(toobig_iff);
// Plug the failing-too-big test into the slow-path region
Node *toobig_true = new (C) IfTrueNode( toobig_iff );
transform_later(toobig_true);
slow_region ->init_req( too_big_or_final_path, toobig_true );
toobig_false = new (C) IfFalseNode( toobig_iff );
transform_later(toobig_false);
} else { // No initial test, just fall into next case
toobig_false = ctrl;
debug_only(slow_region = NodeSentinel);
}
Node *slow_mem = mem; // save the current memory state for slow path
// generate the fast allocation code unless we know that the initial test will always go slow
if (!always_slow) {
// Fast path modifies only raw memory.
if (mem->is_MergeMem()) {
mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
}
Node* eden_top_adr;
Node* eden_end_adr;
set_eden_pointers(eden_top_adr, eden_end_adr);
// Load Eden::end. Loop invariant and hoisted.
//
// Note: We set the control input on "eden_end" and "old_eden_top" when using
// a TLAB to work around a bug where these values were being moved across
// a safepoint. These are not oops, so they cannot be include in the oop
// map, but they can be changed by a GC. The proper way to fix this would
// be to set the raw memory state when generating a SafepointNode. However
// this will require extensive changes to the loop optimization in order to
// prevent a degradation of the optimization.
// See comment in memnode.hpp, around line 227 in class LoadPNode.
Node *eden_end = make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);
// allocate the Region and Phi nodes for the result
result_region = new (C) RegionNode(3);
result_phi_rawmem = new (C) PhiNode(result_region, Type::MEMORY, TypeRawPtr::BOTTOM);
result_phi_rawoop = new (C) PhiNode(result_region, TypeRawPtr::BOTTOM);
result_phi_i_o = new (C) PhiNode(result_region, Type::ABIO); // I/O is used for Prefetch
// We need a Region for the loop-back contended case.
enum { fall_in_path = 1, contended_loopback_path = 2 };
Node *contended_region;
Node *contended_phi_rawmem;
if (UseTLAB) {
contended_region = toobig_false;
contended_phi_rawmem = mem;
} else {
contended_region = new (C) RegionNode(3);
contended_phi_rawmem = new (C) PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
// Now handle the passing-too-big test. We fall into the contended
// loop-back merge point.
contended_region ->init_req(fall_in_path, toobig_false);
contended_phi_rawmem->init_req(fall_in_path, mem);
transform_later(contended_region);
transform_later(contended_phi_rawmem);
}
// Load(-locked) the heap top.
// See note above concerning the control input when using a TLAB
Node *old_eden_top = UseTLAB
? new (C) LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
: new (C) LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
transform_later(old_eden_top);
// Add to heap top to get a new heap top
Node *new_eden_top = new (C) AddPNode(top(), old_eden_top, size_in_bytes);
transform_later(new_eden_top);
// Check for needing a GC; compare against heap end
Node *needgc_cmp = new (C) CmpPNode(new_eden_top, eden_end);
transform_later(needgc_cmp);
Node *needgc_bol = new (C) BoolNode(needgc_cmp, BoolTest::ge);
transform_later(needgc_bol);
IfNode *needgc_iff = new (C) IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
transform_later(needgc_iff);
// Plug the failing-heap-space-need-gc test into the slow-path region
Node *needgc_true = new (C) IfTrueNode(needgc_iff);
transform_later(needgc_true);
if (initial_slow_test) {
slow_region->init_req(need_gc_path, needgc_true);
// This completes all paths into the slow merge point
transform_later(slow_region);
} else { // No initial slow path needed!
// Just fall from the need-GC path straight into the VM call.
slow_region = needgc_true;
}
// No need for a GC. Setup for the Store-Conditional
Node *needgc_false = new (C) IfFalseNode(needgc_iff);
transform_later(needgc_false);
// Grab regular I/O before optional prefetch may change it.
// Slow-path does no I/O so just set it to the original I/O.
result_phi_i_o->init_req(slow_result_path, i_o);
i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem,
old_eden_top, new_eden_top, length);
// Name successful fast-path variables
Node* fast_oop = old_eden_top;
Node* fast_oop_ctrl;
Node* fast_oop_rawmem;
// Store (-conditional) the modified eden top back down.
// StorePConditional produces flags for a test PLUS a modified raw
// memory state.
if (UseTLAB) {
Node* store_eden_top =
new (C) StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr,
TypeRawPtr::BOTTOM, new_eden_top, MemNode::unordered);
transform_later(store_eden_top);
fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
fast_oop_rawmem = store_eden_top;
} else {
Node* store_eden_top =
new (C) StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr,
new_eden_top, fast_oop/*old_eden_top*/);
transform_later(store_eden_top);
Node *contention_check = new (C) BoolNode(store_eden_top, BoolTest::ne);
transform_later(contention_check);
store_eden_top = new (C) SCMemProjNode(store_eden_top);
transform_later(store_eden_top);
// If not using TLABs, check to see if there was contention.
IfNode *contention_iff = new (C) IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN);
transform_later(contention_iff);
Node *contention_true = new (C) IfTrueNode(contention_iff);
transform_later(contention_true);
// If contention, loopback and try again.
contended_region->init_req(contended_loopback_path, contention_true);
contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top);
// Fast-path succeeded with no contention!
Node *contention_false = new (C) IfFalseNode(contention_iff);
transform_later(contention_false);
fast_oop_ctrl = contention_false;
// Bump total allocated bytes for this thread
Node* thread = new (C) ThreadLocalNode();
transform_later(thread);
Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread,
in_bytes(JavaThread::allocated_bytes_offset()));
Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
0, TypeLong::LONG, T_LONG);
#ifdef _LP64
Node* alloc_size = size_in_bytes;
#else
Node* alloc_size = new (C) ConvI2LNode(size_in_bytes);
transform_later(alloc_size);
#endif
Node* new_alloc_bytes = new (C) AddLNode(alloc_bytes, alloc_size);
transform_later(new_alloc_bytes);
fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
0, new_alloc_bytes, T_LONG);
}
InitializeNode* init = alloc->initialization();
fast_oop_rawmem = initialize_object(alloc,
fast_oop_ctrl, fast_oop_rawmem, fast_oop,
klass_node, length, size_in_bytes);
// If initialization is performed by an array copy, any required
// MemBarStoreStore was already added. If the object does not
// escape no need for a MemBarStoreStore. Otherwise we need a
// MemBarStoreStore so that stores that initialize this object
// can't be reordered with a subsequent store that makes this
// object accessible by other threads.
#ifndef AARCH64
if (init == NULL || (!init->is_complete_with_arraycopy() && !init->does_not_escape())) {
#else
if (!alloc->does_not_escape_thread() &&
(init == NULL || !init->is_complete_with_arraycopy())) {
#endif
if (init == NULL || init->req() < InitializeNode::RawStores) {
// No InitializeNode or no stores captured by zeroing
// elimination. Simply add the MemBarStoreStore after object
// initialization.
MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
transform_later(mb);
mb->init_req(TypeFunc::Memory, fast_oop_rawmem);
mb->init_req(TypeFunc::Control, fast_oop_ctrl);
fast_oop_ctrl = new (C) ProjNode(mb,TypeFunc::Control);
transform_later(fast_oop_ctrl);
fast_oop_rawmem = new (C) ProjNode(mb,TypeFunc::Memory);
transform_later(fast_oop_rawmem);
} else {
// Add the MemBarStoreStore after the InitializeNode so that
// all stores performing the initialization that were moved
// before the InitializeNode happen before the storestore
// barrier.
Node* init_ctrl = init->proj_out(TypeFunc::Control);
Node* init_mem = init->proj_out(TypeFunc::Memory);
MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
transform_later(mb);
Node* ctrl = new (C) ProjNode(init,TypeFunc::Control);
transform_later(ctrl);
Node* mem = new (C) ProjNode(init,TypeFunc::Memory);
transform_later(mem);
// The MemBarStoreStore depends on control and memory coming
// from the InitializeNode
mb->init_req(TypeFunc::Memory, mem);
mb->init_req(TypeFunc::Control, ctrl);
ctrl = new (C) ProjNode(mb,TypeFunc::Control);
transform_later(ctrl);
mem = new (C) ProjNode(mb,TypeFunc::Memory);
transform_later(mem);
// All nodes that depended on the InitializeNode for control
// and memory must now depend on the MemBarNode that itself
// depends on the InitializeNode
_igvn.replace_node(init_ctrl, ctrl);
_igvn.replace_node(init_mem, mem);
}
}
if (C->env()->dtrace_extended_probes()) {
// Slow-path call
int size = TypeFunc::Parms + 2;
CallLeafNode *call = new (C) CallLeafNode(OptoRuntime::dtrace_object_alloc_Type(),
CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc_base),
"dtrace_object_alloc",
TypeRawPtr::BOTTOM);
// Get base of thread-local storage area
Node* thread = new (C) ThreadLocalNode();
transform_later(thread);
call->init_req(TypeFunc::Parms+0, thread);
call->init_req(TypeFunc::Parms+1, fast_oop);
call->init_req(TypeFunc::Control, fast_oop_ctrl);
call->init_req(TypeFunc::I_O , top()); // does no i/o
call->init_req(TypeFunc::Memory , fast_oop_rawmem);
call->init_req(TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr));
call->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr));
transform_later(call);
fast_oop_ctrl = new (C) ProjNode(call,TypeFunc::Control);
transform_later(fast_oop_ctrl);
fast_oop_rawmem = new (C) ProjNode(call,TypeFunc::Memory);
transform_later(fast_oop_rawmem);
}
// Plug in the successful fast-path into the result merge point
result_region ->init_req(fast_result_path, fast_oop_ctrl);
result_phi_rawoop->init_req(fast_result_path, fast_oop);
result_phi_i_o ->init_req(fast_result_path, i_o);
result_phi_rawmem->init_req(fast_result_path, fast_oop_rawmem);
} else {
slow_region = ctrl;
result_phi_i_o = i_o; // Rename it to use in the following code.
}
// Generate slow-path call
CallNode *call = new (C) CallStaticJavaNode(slow_call_type, slow_call_address,
OptoRuntime::stub_name(slow_call_address),
alloc->jvms()->bci(),
TypePtr::BOTTOM);
call->init_req( TypeFunc::Control, slow_region );
call->init_req( TypeFunc::I_O , top() ) ; // does no i/o
call->init_req( TypeFunc::Memory , slow_mem ); // may gc ptrs
call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) );
call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) );
call->init_req(TypeFunc::Parms+0, klass_node);
if (length != NULL) {
call->init_req(TypeFunc::Parms+1, length);
}
// Copy debug information and adjust JVMState information, then replace
// allocate node with the call
copy_call_debug_info((CallNode *) alloc, call);
if (!always_slow) {
call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON.
} else {
// Hook i_o projection to avoid its elimination during allocation
// replacement (when only a slow call is generated).
call->set_req(TypeFunc::I_O, result_phi_i_o);
}
_igvn.replace_node(alloc, call);
transform_later(call);
// Identify the output projections from the allocate node and
// adjust any references to them.
// The control and io projections look like:
//
// v---Proj(ctrl) <-----+ v---CatchProj(ctrl)
// Allocate Catch
// ^---Proj(io) <-------+ ^---CatchProj(io)
//
// We are interested in the CatchProj nodes.
//
extract_call_projections(call);
// An allocate node has separate memory projections for the uses on
// the control and i_o paths. Replace the control memory projection with
// result_phi_rawmem (unless we are only generating a slow call when
// both memory projections are combined)
if (!always_slow && _memproj_fallthrough != NULL) {
for (DUIterator_Fast imax, i = _memproj_fallthrough->fast_outs(imax); i < imax; i++) {
Node *use = _memproj_fallthrough->fast_out(i);
_igvn.rehash_node_delayed(use);
imax -= replace_input(use, _memproj_fallthrough, result_phi_rawmem);
// back up iterator
--i;
}
}
// Now change uses of _memproj_catchall to use _memproj_fallthrough and delete
// _memproj_catchall so we end up with a call that has only 1 memory projection.
if (_memproj_catchall != NULL ) {
if (_memproj_fallthrough == NULL) {
_memproj_fallthrough = new (C) ProjNode(call, TypeFunc::Memory);
transform_later(_memproj_fallthrough);
}
for (DUIterator_Fast imax, i = _memproj_catchall->fast_outs(imax); i < imax; i++) {
Node *use = _memproj_catchall->fast_out(i);
_igvn.rehash_node_delayed(use);
imax -= replace_input(use, _memproj_catchall, _memproj_fallthrough);
// back up iterator
--i;
}
assert(_memproj_catchall->outcnt() == 0, "all uses must be deleted");
_igvn.remove_dead_node(_memproj_catchall);
}
// An allocate node has separate i_o projections for the uses on the control
// and i_o paths. Always replace the control i_o projection with result i_o
// otherwise incoming i_o become dead when only a slow call is generated
// (it is different from memory projections where both projections are
// combined in such case).
if (_ioproj_fallthrough != NULL) {
for (DUIterator_Fast imax, i = _ioproj_fallthrough->fast_outs(imax); i < imax; i++) {
Node *use = _ioproj_fallthrough->fast_out(i);
_igvn.rehash_node_delayed(use);
imax -= replace_input(use, _ioproj_fallthrough, result_phi_i_o);
// back up iterator
--i;
}
}
// Now change uses of _ioproj_catchall to use _ioproj_fallthrough and delete
// _ioproj_catchall so we end up with a call that has only 1 i_o projection.
if (_ioproj_catchall != NULL ) {
if (_ioproj_fallthrough == NULL) {
_ioproj_fallthrough = new (C) ProjNode(call, TypeFunc::I_O);
transform_later(_ioproj_fallthrough);
}
for (DUIterator_Fast imax, i = _ioproj_catchall->fast_outs(imax); i < imax; i++) {
Node *use = _ioproj_catchall->fast_out(i);
_igvn.rehash_node_delayed(use);
imax -= replace_input(use, _ioproj_catchall, _ioproj_fallthrough);
// back up iterator
--i;
}
assert(_ioproj_catchall->outcnt() == 0, "all uses must be deleted");
_igvn.remove_dead_node(_ioproj_catchall);
}
// if we generated only a slow call, we are done
if (always_slow) {
// Now we can unhook i_o.
if (result_phi_i_o->outcnt() > 1) {
call->set_req(TypeFunc::I_O, top());
} else {
assert(result_phi_i_o->unique_ctrl_out() == call, "");
// Case of new array with negative size known during compilation.
// AllocateArrayNode::Ideal() optimization disconnect unreachable
// following code since call to runtime will throw exception.
// As result there will be no users of i_o after the call.
// Leave i_o attached to this call to avoid problems in preceding graph.
}
return;
}
if (_fallthroughcatchproj != NULL) {
ctrl = _fallthroughcatchproj->clone();
transform_later(ctrl);
_igvn.replace_node(_fallthroughcatchproj, result_region);
} else {
ctrl = top();
}
Node *slow_result;
if (_resproj == NULL) {
// no uses of the allocation result
slow_result = top();
} else {
slow_result = _resproj->clone();
transform_later(slow_result);
_igvn.replace_node(_resproj, result_phi_rawoop);
}
// Plug slow-path into result merge point
result_region ->init_req( slow_result_path, ctrl );
result_phi_rawoop->init_req( slow_result_path, slow_result);
result_phi_rawmem->init_req( slow_result_path, _memproj_fallthrough );
transform_later(result_region);
transform_later(result_phi_rawoop);
transform_later(result_phi_rawmem);
transform_later(result_phi_i_o);
// This completes all paths into the result merge point
}
// Helper for PhaseMacroExpand::expand_allocate_common.
// Initializes the newly-allocated storage.
Node*
PhaseMacroExpand::initialize_object(AllocateNode* alloc,
Node* control, Node* rawmem, Node* object,
Node* klass_node, Node* length,
Node* size_in_bytes) {
InitializeNode* init = alloc->initialization();
// Store the klass & mark bits
Node* mark_node = NULL;
// For now only enable fast locking for non-array types
if (UseBiasedLocking && (length == NULL)) {
mark_node = make_load(control, rawmem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
} else {
mark_node = makecon(TypeRawPtr::make((address)markOopDesc::prototype()));
}
rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS);
rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
int header_size = alloc->minimum_header_size(); // conservatively small
// Array length
if (length != NULL) { // Arrays need length field
rawmem = make_store(control, rawmem, object, arrayOopDesc::length_offset_in_bytes(), length, T_INT);
// conservatively small header size:
header_size = arrayOopDesc::base_offset_in_bytes(T_BYTE);
ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass();
if (k->is_array_klass()) // we know the exact header size in most cases:
header_size = Klass::layout_helper_header_size(k->layout_helper());
}
// Clear the object body, if necessary.
if (init == NULL) {
// The init has somehow disappeared; be cautious and clear everything.
//
// This can happen if a node is allocated but an uncommon trap occurs
// immediately. In this case, the Initialize gets associated with the
// trap, and may be placed in a different (outer) loop, if the Allocate
// is in a loop. If (this is rare) the inner loop gets unrolled, then
// there can be two Allocates to one Initialize. The answer in all these
// edge cases is safety first. It is always safe to clear immediately
// within an Allocate, and then (maybe or maybe not) clear some more later.
if (!ZeroTLAB)
rawmem = ClearArrayNode::clear_memory(control, rawmem, object,
header_size, size_in_bytes,
&_igvn);
} else {
if (!init->is_complete()) {
// Try to win by zeroing only what the init does not store.
// We can also try to do some peephole optimizations,
// such as combining some adjacent subword stores.
rawmem = init->complete_stores(control, rawmem, object,
header_size, size_in_bytes, &_igvn);
}
// We have no more use for this link, since the AllocateNode goes away:
init->set_req(InitializeNode::RawAddress, top());
// (If we keep the link, it just confuses the register allocator,
// who thinks he sees a real use of the address by the membar.)
}
return rawmem;
}
// Generate prefetch instructions for next allocations.
Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false,
Node*& contended_phi_rawmem,
Node* old_eden_top, Node* new_eden_top,
Node* length) {
enum { fall_in_path = 1, pf_path = 2 };
if( UseTLAB && AllocatePrefetchStyle == 2 ) {
// Generate prefetch allocation with watermark check.
// As an allocation hits the watermark, we will prefetch starting
// at a "distance" away from watermark.
Node *pf_region = new (C) RegionNode(3);
Node *pf_phi_rawmem = new (C) PhiNode( pf_region, Type::MEMORY,
TypeRawPtr::BOTTOM );
// I/O is used for Prefetch
Node *pf_phi_abio = new (C) PhiNode( pf_region, Type::ABIO );
Node *thread = new (C) ThreadLocalNode();
transform_later(thread);
Node *eden_pf_adr = new (C) AddPNode( top()/*not oop*/, thread,
_igvn.MakeConX(in_bytes(JavaThread::tlab_pf_top_offset())) );
transform_later(eden_pf_adr);
Node *old_pf_wm = new (C) LoadPNode(needgc_false,
contended_phi_rawmem, eden_pf_adr,
TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM,
MemNode::unordered);
transform_later(old_pf_wm);
// check against new_eden_top
Node *need_pf_cmp = new (C) CmpPNode( new_eden_top, old_pf_wm );
transform_later(need_pf_cmp);
Node *need_pf_bol = new (C) BoolNode( need_pf_cmp, BoolTest::ge );
transform_later(need_pf_bol);
IfNode *need_pf_iff = new (C) IfNode( needgc_false, need_pf_bol,
PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN );
transform_later(need_pf_iff);
// true node, add prefetchdistance
Node *need_pf_true = new (C) IfTrueNode( need_pf_iff );
transform_later(need_pf_true);
Node *need_pf_false = new (C) IfFalseNode( need_pf_iff );
transform_later(need_pf_false);
Node *new_pf_wmt = new (C) AddPNode( top(), old_pf_wm,
_igvn.MakeConX(AllocatePrefetchDistance) );
transform_later(new_pf_wmt );
new_pf_wmt->set_req(0, need_pf_true);
Node *store_new_wmt = new (C) StorePNode(need_pf_true,
contended_phi_rawmem, eden_pf_adr,
TypeRawPtr::BOTTOM, new_pf_wmt,
MemNode::unordered);
transform_later(store_new_wmt);
// adding prefetches
pf_phi_abio->init_req( fall_in_path, i_o );
Node *prefetch_adr;
Node *prefetch;
uint lines = AllocatePrefetchDistance / AllocatePrefetchStepSize;
uint step_size = AllocatePrefetchStepSize;
uint distance = 0;
for ( uint i = 0; i < lines; i++ ) {
prefetch_adr = new (C) AddPNode( old_pf_wm, new_pf_wmt,
_igvn.MakeConX(distance) );
transform_later(prefetch_adr);
prefetch = new (C) PrefetchAllocationNode( i_o, prefetch_adr );
transform_later(prefetch);
distance += step_size;
i_o = prefetch;
}
pf_phi_abio->set_req( pf_path, i_o );
pf_region->init_req( fall_in_path, need_pf_false );
pf_region->init_req( pf_path, need_pf_true );
pf_phi_rawmem->init_req( fall_in_path, contended_phi_rawmem );
pf_phi_rawmem->init_req( pf_path, store_new_wmt );
transform_later(pf_region);
transform_later(pf_phi_rawmem);
transform_later(pf_phi_abio);
needgc_false = pf_region;
contended_phi_rawmem = pf_phi_rawmem;
i_o = pf_phi_abio;
} else if( UseTLAB && AllocatePrefetchStyle == 3 ) {
// Insert a prefetch for each allocation.
// This code is used to generate 1 prefetch instruction per cache line.
Node *pf_region = new (C) RegionNode(3);
Node *pf_phi_rawmem = new (C) PhiNode( pf_region, Type::MEMORY,
TypeRawPtr::BOTTOM );
// Generate several prefetch instructions.
uint lines = (length != NULL) ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
uint step_size = AllocatePrefetchStepSize;
uint distance = AllocatePrefetchDistance;
// Next cache address.
Node *cache_adr = new (C) AddPNode(old_eden_top, old_eden_top,
_igvn.MakeConX(distance));
transform_later(cache_adr);
cache_adr = new (C) CastP2XNode(needgc_false, cache_adr);
transform_later(cache_adr);
// Address is aligned to execute prefetch to the beginning of cache line size
// (it is important when BIS instruction is used on SPARC as prefetch).
Node* mask = _igvn.MakeConX(~(intptr_t)(step_size-1));
cache_adr = new (C) AndXNode(cache_adr, mask);
transform_later(cache_adr);
cache_adr = new (C) CastX2PNode(cache_adr);
transform_later(cache_adr);
// Prefetch
Node *prefetch = new (C) PrefetchAllocationNode( contended_phi_rawmem, cache_adr );
prefetch->set_req(0, needgc_false);
transform_later(prefetch);
contended_phi_rawmem = prefetch;
Node *prefetch_adr;
distance = step_size;
for ( uint i = 1; i < lines; i++ ) {
prefetch_adr = new (C) AddPNode( cache_adr, cache_adr,
_igvn.MakeConX(distance) );
transform_later(prefetch_adr);
prefetch = new (C) PrefetchAllocationNode( contended_phi_rawmem, prefetch_adr );
transform_later(prefetch);
distance += step_size;
contended_phi_rawmem = prefetch;
}
} else if( AllocatePrefetchStyle > 0 ) {
// Insert a prefetch for each allocation only on the fast-path
Node *prefetch_adr;
Node *prefetch;
// Generate several prefetch instructions.
uint lines = (length != NULL) ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
uint step_size = AllocatePrefetchStepSize;
uint distance = AllocatePrefetchDistance;
for ( uint i = 0; i < lines; i++ ) {
prefetch_adr = new (C) AddPNode( old_eden_top, new_eden_top,
_igvn.MakeConX(distance) );
transform_later(prefetch_adr);
prefetch = new (C) PrefetchAllocationNode( i_o, prefetch_adr );
// Do not let it float too high, since if eden_top == eden_end,
// both might be null.
if( i == 0 ) { // Set control for first prefetch, next follows it
prefetch->init_req(0, needgc_false);
}
transform_later(prefetch);
distance += step_size;
i_o = prefetch;
}
}
return i_o;
}
void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) {
expand_allocate_common(alloc, NULL,
OptoRuntime::new_instance_Type(),
OptoRuntime::new_instance_Java());
}
void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) {
Node* length = alloc->in(AllocateNode::ALength);
InitializeNode* init = alloc->initialization();
Node* klass_node = alloc->in(AllocateNode::KlassNode);
ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass();
address slow_call_address; // Address of slow call
if (init != NULL && init->is_complete_with_arraycopy() &&
k->is_type_array_klass()) {
// Don't zero type array during slow allocation in VM since
// it will be initialized later by arraycopy in compiled code.
slow_call_address = OptoRuntime::new_array_nozero_Java();
} else {
slow_call_address = OptoRuntime::new_array_Java();
}
expand_allocate_common(alloc, length,
OptoRuntime::new_array_Type(),
slow_call_address);
}
//-------------------mark_eliminated_box----------------------------------
//
// During EA obj may point to several objects but after few ideal graph
// transformations (CCP) it may point to only one non escaping object
// (but still using phi), corresponding locks and unlocks will be marked
// for elimination. Later obj could be replaced with a new node (new phi)
// and which does not have escape information. And later after some graph
// reshape other locks and unlocks (which were not marked for elimination
// before) are connected to this new obj (phi) but they still will not be
// marked for elimination since new obj has no escape information.
// Mark all associated (same box and obj) lock and unlock nodes for
// elimination if some of them marked already.
void PhaseMacroExpand::mark_eliminated_box(Node* oldbox, Node* obj) {
if (oldbox->as_BoxLock()->is_eliminated())
return; // This BoxLock node was processed already.
// New implementation (EliminateNestedLocks) has separate BoxLock
// node for each locked region so mark all associated locks/unlocks as
// eliminated even if different objects are referenced in one locked region
// (for example, OSR compilation of nested loop inside locked scope).
if (EliminateNestedLocks ||
oldbox->as_BoxLock()->is_simple_lock_region(NULL, obj)) {
// Box is used only in one lock region. Mark this box as eliminated.
_igvn.hash_delete(oldbox);
oldbox->as_BoxLock()->set_eliminated(); // This changes box's hash value
_igvn.hash_insert(oldbox);
for (uint i = 0; i < oldbox->outcnt(); i++) {
Node* u = oldbox->raw_out(i);
if (u->is_AbstractLock() && !u->as_AbstractLock()->is_non_esc_obj()) {
AbstractLockNode* alock = u->as_AbstractLock();
// Check lock's box since box could be referenced by Lock's debug info.
if (alock->box_node() == oldbox) {
// Mark eliminated all related locks and unlocks.
#ifdef ASSERT
alock->log_lock_optimization(C, "eliminate_lock_set_non_esc4");
#endif
alock->set_non_esc_obj();
}
}
}
return;
}
// Create new "eliminated" BoxLock node and use it in monitor debug info
// instead of oldbox for the same object.
BoxLockNode* newbox = oldbox->clone()->as_BoxLock();
// Note: BoxLock node is marked eliminated only here and it is used
// to indicate that all associated lock and unlock nodes are marked
// for elimination.
newbox->set_eliminated();
transform_later(newbox);
// Replace old box node with new box for all users of the same object.
for (uint i = 0; i < oldbox->outcnt();) {
bool next_edge = true;
Node* u = oldbox->raw_out(i);
if (u->is_AbstractLock()) {
AbstractLockNode* alock = u->as_AbstractLock();
if (alock->box_node() == oldbox && alock->obj_node()->eqv_uncast(obj)) {
// Replace Box and mark eliminated all related locks and unlocks.
#ifdef ASSERT
alock->log_lock_optimization(C, "eliminate_lock_set_non_esc5");
#endif
alock->set_non_esc_obj();
_igvn.rehash_node_delayed(alock);
alock->set_box_node(newbox);
next_edge = false;
}
}
if (u->is_FastLock() && u->as_FastLock()->obj_node()->eqv_uncast(obj)) {
FastLockNode* flock = u->as_FastLock();
assert(flock->box_node() == oldbox, "sanity");
_igvn.rehash_node_delayed(flock);
flock->set_box_node(newbox);
next_edge = false;
}
// Replace old box in monitor debug info.
if (u->is_SafePoint() && u->as_SafePoint()->jvms()) {
SafePointNode* sfn = u->as_SafePoint();
JVMState* youngest_jvms = sfn->jvms();
int max_depth = youngest_jvms->depth();
for (int depth = 1; depth <= max_depth; depth++) {
JVMState* jvms = youngest_jvms->of_depth(depth);
int num_mon = jvms->nof_monitors();
// Loop over monitors
for (int idx = 0; idx < num_mon; idx++) {
Node* obj_node = sfn->monitor_obj(jvms, idx);
Node* box_node = sfn->monitor_box(jvms, idx);
if (box_node == oldbox && obj_node->eqv_uncast(obj)) {
int j = jvms->monitor_box_offset(idx);
_igvn.replace_input_of(u, j, newbox);
next_edge = false;
}
}
}
}
if (next_edge) i++;
}
}
//-----------------------mark_eliminated_locking_nodes-----------------------
void PhaseMacroExpand::mark_eliminated_locking_nodes(AbstractLockNode *alock) {
if (EliminateNestedLocks) {
if (alock->is_nested()) {
assert(alock->box_node()->as_BoxLock()->is_eliminated(), "sanity");
return;
} else if (!alock->is_non_esc_obj()) { // Not eliminated or coarsened
// Only Lock node has JVMState needed here.
// Not that preceding claim is documented anywhere else.
if (alock->jvms() != NULL) {
if (alock->as_Lock()->is_nested_lock_region()) {
// Mark eliminated related nested locks and unlocks.
Node* obj = alock->obj_node();
BoxLockNode* box_node = alock->box_node()->as_BoxLock();
assert(!box_node->is_eliminated(), "should not be marked yet");
// Note: BoxLock node is marked eliminated only here
// and it is used to indicate that all associated lock
// and unlock nodes are marked for elimination.
box_node->set_eliminated(); // Box's hash is always NO_HASH here
for (uint i = 0; i < box_node->outcnt(); i++) {
Node* u = box_node->raw_out(i);
if (u->is_AbstractLock()) {
alock = u->as_AbstractLock();
if (alock->box_node() == box_node) {
// Verify that this Box is referenced only by related locks.
assert(alock->obj_node()->eqv_uncast(obj), "");
// Mark all related locks and unlocks.
#ifdef ASSERT
alock->log_lock_optimization(C, "eliminate_lock_set_nested");
#endif
alock->set_nested();
}
}
}
} else {
#ifdef ASSERT
alock->log_lock_optimization(C, "eliminate_lock_NOT_nested_lock_region");
if (C->log() != NULL)
alock->as_Lock()->is_nested_lock_region(C); // rerun for debugging output
#endif
}
}
return;
}
// Process locks for non escaping object
assert(alock->is_non_esc_obj(), "");
} // EliminateNestedLocks
if (alock->is_non_esc_obj()) { // Lock is used for non escaping object
// Look for all locks of this object and mark them and
// corresponding BoxLock nodes as eliminated.
Node* obj = alock->obj_node();
for (uint j = 0; j < obj->outcnt(); j++) {
Node* o = obj->raw_out(j);
if (o->is_AbstractLock() &&
o->as_AbstractLock()->obj_node()->eqv_uncast(obj)) {
alock = o->as_AbstractLock();
Node* box = alock->box_node();
// Replace old box node with new eliminated box for all users
// of the same object and mark related locks as eliminated.
mark_eliminated_box(box, obj);
}
}
}
}
// we have determined that this lock/unlock can be eliminated, we simply
// eliminate the node without expanding it.
//
// Note: The membar's associated with the lock/unlock are currently not
// eliminated. This should be investigated as a future enhancement.
//
bool PhaseMacroExpand::eliminate_locking_node(AbstractLockNode *alock) {
if (!alock->is_eliminated()) {
return false;
}
#ifdef ASSERT
if (!alock->is_coarsened()) {
// Check that new "eliminated" BoxLock node is created.
BoxLockNode* oldbox = alock->box_node()->as_BoxLock();
assert(oldbox->is_eliminated(), "should be done already");
}
#endif
alock->log_lock_optimization(C, "eliminate_lock");
#ifndef PRODUCT
if (PrintEliminateLocks) {
if (alock->is_Lock()) {
tty->print_cr("++++ Eliminated: %d Lock", alock->_idx);
} else {
tty->print_cr("++++ Eliminated: %d Unlock", alock->_idx);
}
}
#endif
Node* mem = alock->in(TypeFunc::Memory);
Node* ctrl = alock->in(TypeFunc::Control);
extract_call_projections(alock);
// There are 2 projections from the lock. The lock node will
// be deleted when its last use is subsumed below.
assert(alock->outcnt() == 2 &&
_fallthroughproj != NULL &&
_memproj_fallthrough != NULL,
"Unexpected projections from Lock/Unlock");
Node* fallthroughproj = _fallthroughproj;
Node* memproj_fallthrough = _memproj_fallthrough;
// The memory projection from a lock/unlock is RawMem
// The input to a Lock is merged memory, so extract its RawMem input
// (unless the MergeMem has been optimized away.)
if (alock->is_Lock()) {
// Seach for MemBarAcquireLock node and delete it also.
MemBarNode* membar = fallthroughproj->unique_ctrl_out()->as_MemBar();
assert(membar != NULL && membar->Opcode() == Op_MemBarAcquireLock, "");
Node* ctrlproj = membar->proj_out(TypeFunc::Control);
Node* memproj = membar->proj_out(TypeFunc::Memory);
_igvn.replace_node(ctrlproj, fallthroughproj);
_igvn.replace_node(memproj, memproj_fallthrough);
// Delete FastLock node also if this Lock node is unique user
// (a loop peeling may clone a Lock node).
Node* flock = alock->as_Lock()->fastlock_node();
if (flock->outcnt() == 1) {
assert(flock->unique_out() == alock, "sanity");
_igvn.replace_node(flock, top());
}
}
// Seach for MemBarReleaseLock node and delete it also.
if (alock->is_Unlock() && ctrl != NULL && ctrl->is_Proj() &&
ctrl->in(0)->is_MemBar()) {
MemBarNode* membar = ctrl->in(0)->as_MemBar();
assert(membar->Opcode() == Op_MemBarReleaseLock &&
mem->is_Proj() && membar == mem->in(0), "");
_igvn.replace_node(fallthroughproj, ctrl);
_igvn.replace_node(memproj_fallthrough, mem);
fallthroughproj = ctrl;
memproj_fallthrough = mem;
ctrl = membar->in(TypeFunc::Control);
mem = membar->in(TypeFunc::Memory);
}
_igvn.replace_node(fallthroughproj, ctrl);
_igvn.replace_node(memproj_fallthrough, mem);
return true;
}
//------------------------------expand_lock_node----------------------
void PhaseMacroExpand::expand_lock_node(LockNode *lock) {
Node* ctrl = lock->in(TypeFunc::Control);
Node* mem = lock->in(TypeFunc::Memory);
Node* obj = lock->obj_node();
Node* box = lock->box_node();
Node* flock = lock->fastlock_node();
assert(!box->as_BoxLock()->is_eliminated(), "sanity");
// Make the merge point
Node *region;
Node *mem_phi;
Node *slow_path;
if (UseOptoBiasInlining) {
/*
* See the full description in MacroAssembler::biased_locking_enter().
*
* if( (mark_word & biased_lock_mask) == biased_lock_pattern ) {
* // The object is biased.
* proto_node = klass->prototype_header;
* o_node = thread | proto_node;
* x_node = o_node ^ mark_word;
* if( (x_node & ~age_mask) == 0 ) { // Biased to the current thread ?
* // Done.
* } else {
* if( (x_node & biased_lock_mask) != 0 ) {
* // The klass's prototype header is no longer biased.
* cas(&mark_word, mark_word, proto_node)
* goto cas_lock;
* } else {
* // The klass's prototype header is still biased.
* if( (x_node & epoch_mask) != 0 ) { // Expired epoch?
* old = mark_word;
* new = o_node;
* } else {
* // Different thread or anonymous biased.
* old = mark_word & (epoch_mask | age_mask | biased_lock_mask);
* new = thread | old;
* }
* // Try to rebias.
* if( cas(&mark_word, old, new) == 0 ) {
* // Done.
* } else {
* goto slow_path; // Failed.
* }
* }
* }
* } else {
* // The object is not biased.
* cas_lock:
* if( FastLock(obj) == 0 ) {
* // Done.
* } else {
* slow_path:
* OptoRuntime::complete_monitor_locking_Java(obj);
* }
* }
*/
region = new (C) RegionNode(5);
// create a Phi for the memory state
mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
Node* fast_lock_region = new (C) RegionNode(3);
Node* fast_lock_mem_phi = new (C) PhiNode( fast_lock_region, Type::MEMORY, TypeRawPtr::BOTTOM);
// First, check mark word for the biased lock pattern.
Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());
// Get fast path - mark word has the biased lock pattern.
ctrl = opt_bits_test(ctrl, fast_lock_region, 1, mark_node,
markOopDesc::biased_lock_mask_in_place,
markOopDesc::biased_lock_pattern, true);
// fast_lock_region->in(1) is set to slow path.
fast_lock_mem_phi->init_req(1, mem);
// Now check that the lock is biased to the current thread and has
// the same epoch and bias as Klass::_prototype_header.
// Special-case a fresh allocation to avoid building nodes:
Node* klass_node = AllocateNode::Ideal_klass(obj, &_igvn);
if (klass_node == NULL) {
Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
klass_node = transform_later(LoadKlassNode::make(_igvn, NULL, mem, k_adr, _igvn.type(k_adr)->is_ptr()));
#ifdef _LP64
if (UseCompressedClassPointers && klass_node->is_DecodeNKlass()) {
assert(klass_node->in(1)->Opcode() == Op_LoadNKlass, "sanity");
klass_node->in(1)->init_req(0, ctrl);
} else
#endif
klass_node->init_req(0, ctrl);
}
Node *proto_node = make_load(ctrl, mem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeX_X, TypeX_X->basic_type());
Node* thread = transform_later(new (C) ThreadLocalNode());
Node* cast_thread = transform_later(new (C) CastP2XNode(ctrl, thread));
Node* o_node = transform_later(new (C) OrXNode(cast_thread, proto_node));
Node* x_node = transform_later(new (C) XorXNode(o_node, mark_node));
// Get slow path - mark word does NOT match the value.
Node* not_biased_ctrl = opt_bits_test(ctrl, region, 3, x_node,
(~markOopDesc::age_mask_in_place), 0);
// region->in(3) is set to fast path - the object is biased to the current thread.
mem_phi->init_req(3, mem);
// Mark word does NOT match the value (thread | Klass::_prototype_header).
// First, check biased pattern.
// Get fast path - _prototype_header has the same biased lock pattern.
ctrl = opt_bits_test(not_biased_ctrl, fast_lock_region, 2, x_node,
markOopDesc::biased_lock_mask_in_place, 0, true);
not_biased_ctrl = fast_lock_region->in(2); // Slow path
// fast_lock_region->in(2) - the prototype header is no longer biased
// and we have to revoke the bias on this object.
// We are going to try to reset the mark of this object to the prototype
// value and fall through to the CAS-based locking scheme.
Node* adr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
Node* cas = new (C) StoreXConditionalNode(not_biased_ctrl, mem, adr,
proto_node, mark_node);
transform_later(cas);
Node* proj = transform_later( new (C) SCMemProjNode(cas));
fast_lock_mem_phi->init_req(2, proj);
// Second, check epoch bits.
Node* rebiased_region = new (C) RegionNode(3);
Node* old_phi = new (C) PhiNode( rebiased_region, TypeX_X);
Node* new_phi = new (C) PhiNode( rebiased_region, TypeX_X);
// Get slow path - mark word does NOT match epoch bits.
Node* epoch_ctrl = opt_bits_test(ctrl, rebiased_region, 1, x_node,
markOopDesc::epoch_mask_in_place, 0);
// The epoch of the current bias is not valid, attempt to rebias the object
// toward the current thread.
rebiased_region->init_req(2, epoch_ctrl);
old_phi->init_req(2, mark_node);
new_phi->init_req(2, o_node);
// rebiased_region->in(1) is set to fast path.
// The epoch of the current bias is still valid but we know
// nothing about the owner; it might be set or it might be clear.
Node* cmask = MakeConX(markOopDesc::biased_lock_mask_in_place |
markOopDesc::age_mask_in_place |
markOopDesc::epoch_mask_in_place);
Node* old = transform_later(new (C) AndXNode(mark_node, cmask));
cast_thread = transform_later(new (C) CastP2XNode(ctrl, thread));
Node* new_mark = transform_later(new (C) OrXNode(cast_thread, old));
old_phi->init_req(1, old);
new_phi->init_req(1, new_mark);
transform_later(rebiased_region);
transform_later(old_phi);
transform_later(new_phi);
// Try to acquire the bias of the object using an atomic operation.
// If this fails we will go in to the runtime to revoke the object's bias.
cas = new (C) StoreXConditionalNode(rebiased_region, mem, adr,
new_phi, old_phi);
transform_later(cas);
proj = transform_later( new (C) SCMemProjNode(cas));
// Get slow path - Failed to CAS.
not_biased_ctrl = opt_bits_test(rebiased_region, region, 4, cas, 0, 0);
mem_phi->init_req(4, proj);
// region->in(4) is set to fast path - the object is rebiased to the current thread.
// Failed to CAS.
slow_path = new (C) RegionNode(3);
Node *slow_mem = new (C) PhiNode( slow_path, Type::MEMORY, TypeRawPtr::BOTTOM);
slow_path->init_req(1, not_biased_ctrl); // Capture slow-control
slow_mem->init_req(1, proj);
// Call CAS-based locking scheme (FastLock node).
transform_later(fast_lock_region);
transform_later(fast_lock_mem_phi);
// Get slow path - FastLock failed to lock the object.
ctrl = opt_bits_test(fast_lock_region, region, 2, flock, 0, 0);
mem_phi->init_req(2, fast_lock_mem_phi);
// region->in(2) is set to fast path - the object is locked to the current thread.
slow_path->init_req(2, ctrl); // Capture slow-control
slow_mem->init_req(2, fast_lock_mem_phi);
transform_later(slow_path);
transform_later(slow_mem);
// Reset lock's memory edge.
lock->set_req(TypeFunc::Memory, slow_mem);
} else {
region = new (C) RegionNode(3);
// create a Phi for the memory state
mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
// Optimize test; set region slot 2
slow_path = opt_bits_test(ctrl, region, 2, flock, 0, 0);
mem_phi->init_req(2, mem);
}
// Make slow path call
CallNode *call = make_slow_call( (CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(), OptoRuntime::complete_monitor_locking_Java(), NULL, slow_path, obj, box );
extract_call_projections(call);
// Slow path can only throw asynchronous exceptions, which are always
// de-opted. So the compiler thinks the slow-call can never throw an
// exception. If it DOES throw an exception we would need the debug
// info removed first (since if it throws there is no monitor).
assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL &&
_memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock");
// Capture slow path
// disconnect fall-through projection from call and create a new one
// hook up users of fall-through projection to region
Node *slow_ctrl = _fallthroughproj->clone();
transform_later(slow_ctrl);
_igvn.hash_delete(_fallthroughproj);
_fallthroughproj->disconnect_inputs(NULL, C);
region->init_req(1, slow_ctrl);
// region inputs are now complete
transform_later(region);
_igvn.replace_node(_fallthroughproj, region);
Node *memproj = transform_later( new(C) ProjNode(call, TypeFunc::Memory) );
mem_phi->init_req(1, memproj );
transform_later(mem_phi);
_igvn.replace_node(_memproj_fallthrough, mem_phi);
}
//------------------------------expand_unlock_node----------------------
void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) {
Node* ctrl = unlock->in(TypeFunc::Control);
Node* mem = unlock->in(TypeFunc::Memory);
Node* obj = unlock->obj_node();
Node* box = unlock->box_node();
assert(!box->as_BoxLock()->is_eliminated(), "sanity");
// No need for a null check on unlock
// Make the merge point
Node *region;
Node *mem_phi;
if (UseOptoBiasInlining) {
// Check for biased locking unlock case, which is a no-op.
// See the full description in MacroAssembler::biased_locking_exit().
region = new (C) RegionNode(4);
// create a Phi for the memory state
mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
mem_phi->init_req(3, mem);
Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());
ctrl = opt_bits_test(ctrl, region, 3, mark_node,
markOopDesc::biased_lock_mask_in_place,
markOopDesc::biased_lock_pattern);
} else {
region = new (C) RegionNode(3);
// create a Phi for the memory state
mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
}
FastUnlockNode *funlock = new (C) FastUnlockNode( ctrl, obj, box );
funlock = transform_later( funlock )->as_FastUnlock();
// Optimize test; set region slot 2
Node *slow_path = opt_bits_test(ctrl, region, 2, funlock, 0, 0);
CallNode *call = make_slow_call( (CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C), "complete_monitor_unlocking_C", slow_path, obj, box );
extract_call_projections(call);
assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL &&
_memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock");
// No exceptions for unlocking
// Capture slow path
// disconnect fall-through projection from call and create a new one
// hook up users of fall-through projection to region
Node *slow_ctrl = _fallthroughproj->clone();
transform_later(slow_ctrl);
_igvn.hash_delete(_fallthroughproj);
_fallthroughproj->disconnect_inputs(NULL, C);
region->init_req(1, slow_ctrl);
// region inputs are now complete
transform_later(region);
_igvn.replace_node(_fallthroughproj, region);
Node *memproj = transform_later( new(C) ProjNode(call, TypeFunc::Memory) );
mem_phi->init_req(1, memproj );
mem_phi->init_req(2, mem);
transform_later(mem_phi);
_igvn.replace_node(_memproj_fallthrough, mem_phi);
}
//---------------------------eliminate_macro_nodes----------------------
// Eliminate scalar replaced allocations and associated locks.
void PhaseMacroExpand::eliminate_macro_nodes() {
if (C->macro_count() == 0)
return;
// First, attempt to eliminate locks
int cnt = C->macro_count();
for (int i=0; i < cnt; i++) {
Node *n = C->macro_node(i);
if (n->is_AbstractLock()) { // Lock and Unlock nodes
// Before elimination mark all associated (same box and obj)
// lock and unlock nodes.
mark_eliminated_locking_nodes(n->as_AbstractLock());
}
}
bool progress = true;
while (progress) {
progress = false;
for (int i = C->macro_count(); i > 0; i--) {
Node * n = C->macro_node(i-1);
bool success = false;
debug_only(int old_macro_count = C->macro_count(););
if (n->is_AbstractLock()) {
success = eliminate_locking_node(n->as_AbstractLock());
}
assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
progress = progress || success;
}
}
// Next, attempt to eliminate allocations
_has_locks = false;
progress = true;
while (progress) {
progress = false;
for (int i = C->macro_count(); i > 0; i--) {
Node * n = C->macro_node(i-1);
bool success = false;
debug_only(int old_macro_count = C->macro_count(););
switch (n->class_id()) {
case Node::Class_Allocate:
case Node::Class_AllocateArray:
success = eliminate_allocate_node(n->as_Allocate());
break;
case Node::Class_CallStaticJava:
success = eliminate_boxing_node(n->as_CallStaticJava());
break;
case Node::Class_Lock:
case Node::Class_Unlock:
assert(!n->as_AbstractLock()->is_eliminated(), "sanity");
_has_locks = true;
break;
default:
assert(n->Opcode() == Op_LoopLimit ||
n->Opcode() == Op_Opaque1 ||
n->Opcode() == Op_Opaque2 ||
n->Opcode() == Op_Opaque3, "unknown node type in macro list");
}
assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
progress = progress || success;
}
}
}
//------------------------------expand_macro_nodes----------------------
// Returns true if a failure occurred.
bool PhaseMacroExpand::expand_macro_nodes() {
// Last attempt to eliminate macro nodes.
eliminate_macro_nodes();
// Make sure expansion will not cause node limit to be exceeded.
// Worst case is a macro node gets expanded into about 50 nodes.
// Allow 50% more for optimization.
if (C->check_node_count(C->macro_count() * 75, "out of nodes before macro expansion" ) )
return true;
// Eliminate Opaque and LoopLimit nodes. Do it after all loop optimizations.
bool progress = true;
while (progress) {
progress = false;
for (int i = C->macro_count(); i > 0; i--) {
Node * n = C->macro_node(i-1);
bool success = false;
debug_only(int old_macro_count = C->macro_count(););
if (n->Opcode() == Op_LoopLimit) {
// Remove it from macro list and put on IGVN worklist to optimize.
C->remove_macro_node(n);
_igvn._worklist.push(n);
success = true;
} else if (n->Opcode() == Op_CallStaticJava) {
// Remove it from macro list and put on IGVN worklist to optimize.
C->remove_macro_node(n);
_igvn._worklist.push(n);
success = true;
} else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) {
_igvn.replace_node(n, n->in(1));
success = true;
#if INCLUDE_RTM_OPT
} else if ((n->Opcode() == Op_Opaque3) && ((Opaque3Node*)n)->rtm_opt()) {
assert(C->profile_rtm(), "should be used only in rtm deoptimization code");
assert((n->outcnt() == 1) && n->unique_out()->is_Cmp(), "");
Node* cmp = n->unique_out();
#ifdef ASSERT
// Validate graph.
assert((cmp->outcnt() == 1) && cmp->unique_out()->is_Bool(), "");
BoolNode* bol = cmp->unique_out()->as_Bool();
assert((bol->outcnt() == 1) && bol->unique_out()->is_If() &&
(bol->_test._test == BoolTest::ne), "");
IfNode* ifn = bol->unique_out()->as_If();
assert((ifn->outcnt() == 2) &&
ifn->proj_out(1)->is_uncommon_trap_proj(Deoptimization::Reason_rtm_state_change), "");
#endif
Node* repl = n->in(1);
if (!_has_locks) {
// Remove RTM state check if there are no locks in the code.
// Replace input to compare the same value.
repl = (cmp->in(1) == n) ? cmp->in(2) : cmp->in(1);
}
_igvn.replace_node(n, repl);
success = true;
#endif
}
assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
progress = progress || success;
}
}
// expand "macro" nodes
// nodes are removed from the macro list as they are processed
while (C->macro_count() > 0) {
int macro_count = C->macro_count();
Node * n = C->macro_node(macro_count-1);
assert(n->is_macro(), "only macro nodes expected here");
if (_igvn.type(n) == Type::TOP || n->in(0)->is_top() ) {
// node is unreachable, so don't try to expand it
C->remove_macro_node(n);
continue;
}
switch (n->class_id()) {
case Node::Class_Allocate:
expand_allocate(n->as_Allocate());
break;
case Node::Class_AllocateArray:
expand_allocate_array(n->as_AllocateArray());
break;
case Node::Class_Lock:
expand_lock_node(n->as_Lock());
break;
case Node::Class_Unlock:
expand_unlock_node(n->as_Unlock());
break;
default:
assert(false, "unknown node type in macro list");
}
assert(C->macro_count() < macro_count, "must have deleted a node from macro list");
if (C->failing()) return true;
}
_igvn.set_delay_transform(false);
_igvn.optimize();
if (C->failing()) return true;
return false;
}
C:\hotspot-69087d08d473\src\share\vm/opto/macro.hpp
/*
* Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_VM_OPTO_MACRO_HPP
#define SHARE_VM_OPTO_MACRO_HPP
#include "opto/phase.hpp"
class AllocateNode;
class AllocateArrayNode;
class CallNode;
class Node;
class PhaseIterGVN;
class PhaseMacroExpand : public Phase {
private:
PhaseIterGVN &_igvn;
// Helper methods roughly modelled after GraphKit:
Node* top() const { return C->top(); }
Node* intcon(jint con) const { return _igvn.intcon(con); }
Node* longcon(jlong con) const { return _igvn.longcon(con); }
Node* makecon(const Type *t) const { return _igvn.makecon(t); }
Node* basic_plus_adr(Node* base, int offset) {
return (offset == 0)? base: basic_plus_adr(base, MakeConX(offset));
}
Node* basic_plus_adr(Node* base, Node* ptr, int offset) {
return (offset == 0)? ptr: basic_plus_adr(base, ptr, MakeConX(offset));
}
Node* basic_plus_adr(Node* base, Node* offset) {
return basic_plus_adr(base, base, offset);
}
Node* basic_plus_adr(Node* base, Node* ptr, Node* offset) {
Node* adr = new (C) AddPNode(base, ptr, offset);
return transform_later(adr);
}
Node* transform_later(Node* n) {
// equivalent to _gvn.transform in GraphKit, Ideal, etc.
_igvn.register_new_node_with_optimizer(n);
return n;
}
void set_eden_pointers(Node* &eden_top_adr, Node* &eden_end_adr);
Node* make_load( Node* ctl, Node* mem, Node* base, int offset,
const Type* value_type, BasicType bt);
Node* make_store(Node* ctl, Node* mem, Node* base, int offset,
Node* value, BasicType bt);
// projections extracted from a call node
ProjNode *_fallthroughproj;
ProjNode *_fallthroughcatchproj;
ProjNode *_ioproj_fallthrough;
ProjNode *_ioproj_catchall;
ProjNode *_catchallcatchproj;
ProjNode *_memproj_fallthrough;
ProjNode *_memproj_catchall;
ProjNode *_resproj;
// Additional data collected during macro expansion
bool _has_locks;
void expand_allocate(AllocateNode *alloc);
void expand_allocate_array(AllocateArrayNode *alloc);
void expand_allocate_common(AllocateNode* alloc,
Node* length,
const TypeFunc* slow_call_type,
address slow_call_address);
Node *value_from_mem(Node *mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, Node *alloc);
Node *value_from_mem_phi(Node *mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, Node *alloc, Node_Stack *value_phis, int level);
bool eliminate_boxing_node(CallStaticJavaNode *boxing);
bool eliminate_allocate_node(AllocateNode *alloc);
bool can_eliminate_allocation(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints);
bool scalar_replacement(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints_done);
void process_users_of_allocation(CallNode *alloc);
void eliminate_card_mark(Node *cm);
void mark_eliminated_box(Node* box, Node* obj);
void mark_eliminated_locking_nodes(AbstractLockNode *alock);
bool eliminate_locking_node(AbstractLockNode *alock);
void expand_lock_node(LockNode *lock);
void expand_unlock_node(UnlockNode *unlock);
int replace_input(Node *use, Node *oldref, Node *newref);
void copy_call_debug_info(CallNode *oldcall, CallNode * newcall);
Node* opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path = false);
void copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call);
CallNode* make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call,
const char* leaf_name, Node* slow_path, Node* parm0, Node* parm1);
void extract_call_projections(CallNode *call);
Node* initialize_object(AllocateNode* alloc,
Node* control, Node* rawmem, Node* object,
Node* klass_node, Node* length,
Node* size_in_bytes);
Node* prefetch_allocation(Node* i_o,
Node*& needgc_false, Node*& contended_phi_rawmem,
Node* old_eden_top, Node* new_eden_top,
Node* length);
public:
PhaseMacroExpand(PhaseIterGVN &igvn) : Phase(Macro_Expand), _igvn(igvn), _has_locks(false) {
_igvn.set_delay_transform(true);
}
void eliminate_macro_nodes();
bool expand_macro_nodes();
};
#endif // SHARE_VM_OPTO_MACRO_HPP
C:\hotspot-69087d08d473\src\share\vm/opto/matcher.cpp
/*
* Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "memory/allocation.inline.hpp"
#include "opto/addnode.hpp"
#include "opto/callnode.hpp"
#include "opto/connode.hpp"
#include "opto/idealGraphPrinter.hpp"
#include "opto/matcher.hpp"
#include "opto/memnode.hpp"
#include "opto/opcodes.hpp"
#include "opto/regmask.hpp"
#include "opto/rootnode.hpp"
#include "opto/runtime.hpp"
#include "opto/type.hpp"
#include "opto/vectornode.hpp"
#include "runtime/atomic.hpp"
#include "runtime/os.hpp"
#if defined AD_MD_HPP
# include AD_MD_HPP
#elif defined TARGET_ARCH_MODEL_x86_32
# include "adfiles/ad_x86_32.hpp"
#elif defined TARGET_ARCH_MODEL_x86_64
# include "adfiles/ad_x86_64.hpp"
#elif defined TARGET_ARCH_MODEL_aarch64
# include "adfiles/ad_aarch64.hpp"
#elif defined TARGET_ARCH_MODEL_sparc
# include "adfiles/ad_sparc.hpp"
#elif defined TARGET_ARCH_MODEL_zero
# include "adfiles/ad_zero.hpp"
#elif defined TARGET_ARCH_MODEL_ppc_64
# include "adfiles/ad_ppc_64.hpp"
#endif
OptoReg::Name OptoReg::c_frame_pointer;
const RegMask *Matcher::idealreg2regmask[_last_machine_leaf];
RegMask Matcher::mreg2regmask[_last_Mach_Reg];
RegMask Matcher::STACK_ONLY_mask;
RegMask Matcher::c_frame_ptr_mask;
const uint Matcher::_begin_rematerialize = _BEGIN_REMATERIALIZE;
const uint Matcher::_end_rematerialize = _END_REMATERIALIZE;
//---------------------------Matcher-------------------------------------------
Matcher::Matcher()
: PhaseTransform( Phase::Ins_Select ),
#ifdef ASSERT
_old2new_map(C->comp_arena()),
_new2old_map(C->comp_arena()),
#endif
_shared_nodes(C->comp_arena()),
_reduceOp(reduceOp), _leftOp(leftOp), _rightOp(rightOp),
_swallowed(swallowed),
_begin_inst_chain_rule(_BEGIN_INST_CHAIN_RULE),
_end_inst_chain_rule(_END_INST_CHAIN_RULE),
_must_clone(must_clone),
_register_save_policy(register_save_policy),
_c_reg_save_policy(c_reg_save_policy),
_register_save_type(register_save_type),
_ruleName(ruleName),
_allocation_started(false),
_states_arena(Chunk::medium_size, mtCompiler),
_visited(&_states_arena),
_shared(&_states_arena),
_dontcare(&_states_arena) {
C->set_matcher(this);
idealreg2spillmask [Op_RegI] = NULL;
idealreg2spillmask [Op_RegN] = NULL;
idealreg2spillmask [Op_RegL] = NULL;
idealreg2spillmask [Op_RegF] = NULL;
idealreg2spillmask [Op_RegD] = NULL;
idealreg2spillmask [Op_RegP] = NULL;
idealreg2spillmask [Op_VecS] = NULL;
idealreg2spillmask [Op_VecD] = NULL;
idealreg2spillmask [Op_VecX] = NULL;
idealreg2spillmask [Op_VecY] = NULL;
idealreg2spillmask [Op_RegFlags] = NULL;
idealreg2debugmask [Op_RegI] = NULL;
idealreg2debugmask [Op_RegN] = NULL;
idealreg2debugmask [Op_RegL] = NULL;
idealreg2debugmask [Op_RegF] = NULL;
idealreg2debugmask [Op_RegD] = NULL;
idealreg2debugmask [Op_RegP] = NULL;
idealreg2debugmask [Op_VecS] = NULL;
idealreg2debugmask [Op_VecD] = NULL;
idealreg2debugmask [Op_VecX] = NULL;
idealreg2debugmask [Op_VecY] = NULL;
idealreg2debugmask [Op_RegFlags] = NULL;
idealreg2mhdebugmask[Op_RegI] = NULL;
idealreg2mhdebugmask[Op_RegN] = NULL;
idealreg2mhdebugmask[Op_RegL] = NULL;
idealreg2mhdebugmask[Op_RegF] = NULL;
idealreg2mhdebugmask[Op_RegD] = NULL;
idealreg2mhdebugmask[Op_RegP] = NULL;
idealreg2mhdebugmask[Op_VecS] = NULL;
idealreg2mhdebugmask[Op_VecD] = NULL;
idealreg2mhdebugmask[Op_VecX] = NULL;
idealreg2mhdebugmask[Op_VecY] = NULL;
idealreg2mhdebugmask[Op_RegFlags] = NULL;
debug_only(_mem_node = NULL;) // Ideal memory node consumed by mach node
}
//------------------------------warp_incoming_stk_arg------------------------
// This warps a VMReg into an OptoReg::Name
OptoReg::Name Matcher::warp_incoming_stk_arg( VMReg reg ) {
OptoReg::Name warped;
if( reg->is_stack() ) { // Stack slot argument?
warped = OptoReg::add(_old_SP, reg->reg2stack() );
warped = OptoReg::add(warped, C->out_preserve_stack_slots());
if( warped >= _in_arg_limit )
_in_arg_limit = OptoReg::add(warped, 1); // Bump max stack slot seen
if (!RegMask::can_represent_arg(warped)) {
// the compiler cannot represent this method's calling sequence
C->record_method_not_compilable_all_tiers("unsupported incoming calling sequence");
return OptoReg::Bad;
}
return warped;
}
return OptoReg::as_OptoReg(reg);
}
//---------------------------compute_old_SP------------------------------------
OptoReg::Name Compile::compute_old_SP() {
int fixed = fixed_slots();
int preserve = in_preserve_stack_slots();
return OptoReg::stack2reg(round_to(fixed + preserve, Matcher::stack_alignment_in_slots()));
}
#ifdef ASSERT
void Matcher::verify_new_nodes_only(Node* xroot) {
// Make sure that the new graph only references new nodes
ResourceMark rm;
Unique_Node_List worklist;
VectorSet visited(Thread::current()->resource_area());
worklist.push(xroot);
while (worklist.size() > 0) {
Node* n = worklist.pop();
visited <<= n->_idx;
assert(C->node_arena()->contains(n), "dead node");
for (uint j = 0; j < n->req(); j++) {
Node* in = n->in(j);
if (in != NULL) {
assert(C->node_arena()->contains(in), "dead node");
if (!visited.test(in->_idx)) {
worklist.push(in);
}
}
}
}
}
#endif
10-28
1万+
1万+
07-25
05-10
192
192
05-07
624
624
“相关推荐”对你有帮助么？
-
非常没帮助 -
没帮助 -
一般 -
有帮助 -
非常有帮助
提交
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
