// Counted loops that are guarded should be able to find their guards
if( _head->is_CountedLoop() && _head->as_CountedLoop()->is_main_loop() ) {
CountedLoopNode *cl = _head->as_CountedLoop();
Node *init = cl->init_trip();
Node *ctrl = cl->in(LoopNode::EntryControl);
assert( ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "" );
Node *iff = ctrl->in(0);
assert( iff->Opcode() == Op_If, "" );
Node *bol = iff->in(1);
assert( bol->Opcode() == Op_Bool, "" );
Node *cmp = bol->in(1);
assert( cmp->Opcode() == Op_CmpI, "" );
Node *add = cmp->in(1);
Node *opaq;
if( add->Opcode() == Op_Opaque1 ) {
opaq = add;
} else {
assert( add->Opcode() == Op_AddI || add->Opcode() == Op_ConI , "" );
assert( add == init, "" );
opaq = cmp->in(2);
}
assert( opaq->Opcode() == Op_Opaque1, "" );
}
if (_child != NULL) _child->verify_tree(loop->_child, this);
if (_next != NULL) _next ->verify_tree(loop->_next, parent);
// Innermost loops need to verify loop bodies,
// but only if no 'major_progress'
int fail = 0;
if (!Compile::current()->major_progress() && _child == NULL) {
for( uint i = 0; i < _body.size(); i++ ) {
Node *n = _body.at(i);
if (n->outcnt() == 0) continue; // Ignore dead
uint j;
for( j = 0; j < loop->_body.size(); j++ )
if( loop->_body.at(j) == n )
break;
if( j == loop->_body.size() ) { // Not found in loop body
// Last ditch effort to avoid assertion: Its possible that we
// have some users (so outcnt not zero) but are still dead.
// Try to find from root.
if (Compile::current()->root()->find(n->_idx)) {
fail++;
tty->print("We have that verify does not: ");
n->dump();
}
}
}
for( uint i2 = 0; i2 < loop->_body.size(); i2++ ) {
Node *n = loop->_body.at(i2);
if (n->outcnt() == 0) continue; // Ignore dead
uint j;
for( j = 0; j < _body.size(); j++ )
if( _body.at(j) == n )
break;
if( j == _body.size() ) { // Not found in loop body
// Last ditch effort to avoid assertion: Its possible that we
// have some users (so outcnt not zero) but are still dead.
// Try to find from root.
if (Compile::current()->root()->find(n->_idx)) {
fail++;
tty->print("Verify has that we do not: ");
n->dump();
}
}
}
assert( !fail, "loop body mismatch" );
}
}
#endif
//------------------------------set_idom---------------------------------------
void PhaseIdealLoop::set_idom(Node* d, Node* n, uint dom_depth) {
uint idx = d->_idx;
if (idx >= _idom_size) {
uint newsize = _idom_size<<1;
while( idx >= newsize ) {
newsize <<= 1;
}
_idom = REALLOC_RESOURCE_ARRAY( Node*, _idom,_idom_size,newsize);
_dom_depth = REALLOC_RESOURCE_ARRAY( uint, _dom_depth,_idom_size,newsize);
memset( _dom_depth + _idom_size, 0, (newsize - _idom_size) * sizeof(uint) );
_idom_size = newsize;
}
_idom[idx] = n;
_dom_depth[idx] = dom_depth;
}
//------------------------------recompute_dom_depth---------------------------------------
// The dominator tree is constructed with only parent pointers.
// This recomputes the depth in the tree by first tagging all
// nodes as "no depth yet" marker. The next pass then runs up
// the dom tree from each node marked "no depth yet", and computes
// the depth on the way back down.
void PhaseIdealLoop::recompute_dom_depth() {
uint no_depth_marker = C->unique();
uint i;
// Initialize depth to "no depth yet"
for (i = 0; i < _idom_size; i++) {
if (_dom_depth[i] > 0 && _idom[i] != NULL) {
_dom_depth[i] = no_depth_marker;
}
}
if (_dom_stk == NULL) {
uint init_size = C->live_nodes() / 100; // Guess that 1/100 is a reasonable initial size.
if (init_size < 10) init_size = 10;
_dom_stk = new GrowableArray<uint>(init_size);
}
// Compute new depth for each node.
for (i = 0; i < _idom_size; i++) {
uint j = i;
// Run up the dom tree to find a node with a depth
while (_dom_depth[j] == no_depth_marker) {
_dom_stk->push(j);
j = _idom[j]->_idx;
}
// Compute the depth on the way back down this tree branch
uint dd = _dom_depth[j] + 1;
while (_dom_stk->length() > 0) {
uint j = _dom_stk->pop();
_dom_depth[j] = dd;
dd++;
}
}
}
//------------------------------sort-------------------------------------------
// Insert 'loop' into the existing loop tree. 'innermost' is a leaf of the
// loop tree, not the root.
IdealLoopTree *PhaseIdealLoop::sort( IdealLoopTree *loop, IdealLoopTree *innermost ) {
if( !innermost ) return loop; // New innermost loop
int loop_preorder = get_preorder(loop->_head); // Cache pre-order number
assert( loop_preorder, "not yet post-walked loop" );
IdealLoopTree **pp = &innermost; // Pointer to previous next-pointer
IdealLoopTree *l = *pp; // Do I go before or after 'l'?
// Insert at start of list
while( l ) { // Insertion sort based on pre-order
if( l == loop ) return innermost; // Already on list!
int l_preorder = get_preorder(l->_head); // Cache pre-order number
assert( l_preorder, "not yet post-walked l" );
// Check header pre-order number to figure proper nesting
if( loop_preorder > l_preorder )
break; // End of insertion
// If headers tie (e.g., shared headers) check tail pre-order numbers.
// Since I split shared headers, you'd think this could not happen.
// BUT: I must first do the preorder numbering before I can discover I
// have shared headers, so the split headers all get the same preorder
// number as the RegionNode they split from.
if( loop_preorder == l_preorder &&
get_preorder(loop->_tail) < get_preorder(l->_tail) )
break; // Also check for shared headers (same pre#)
pp = &l->_parent; // Chain up list
l = *pp;
}
// Link into list
// Point predecessor to me
*pp = loop;
// Point me to successor
IdealLoopTree *p = loop->_parent;
loop->_parent = l; // Point me to successor
if( p ) sort( p, innermost ); // Insert my parents into list as well
return innermost;
}
//------------------------------build_loop_tree--------------------------------
// I use a modified Vick/Tarjan algorithm. I need pre- and a post- visit
// bits. The _nodes[] array is mapped by Node index and holds a NULL for
// not-yet-pre-walked, pre-order # for pre-but-not-post-walked and holds the
// tightest enclosing IdealLoopTree for post-walked.
//
// During my forward walk I do a short 1-layer lookahead to see if I can find
// a loop backedge with that doesn't have any work on the backedge. This
// helps me construct nested loops with shared headers better.
//
// Once I've done the forward recursion, I do the post-work. For each child
// I check to see if there is a backedge. Backedges define a loop! I
// insert an IdealLoopTree at the target of the backedge.
//
// During the post-work I also check to see if I have several children
// belonging to different loops. If so, then this Node is a decision point
// where control flow can choose to change loop nests. It is at this
// decision point where I can figure out how loops are nested. At this
// time I can properly order the different loop nests from my children.
// Note that there may not be any backedges at the decision point!
//
// Since the decision point can be far removed from the backedges, I can't
// order my loops at the time I discover them. Thus at the decision point
// I need to inspect loop header pre-order numbers to properly nest my
// loops. This means I need to sort my childrens' loops by pre-order.
// The sort is of size number-of-control-children, which generally limits
// it to size 2 (i.e., I just choose between my 2 target loops).
void PhaseIdealLoop::build_loop_tree() {
// Allocate stack of size C->live_nodes()/2 to avoid frequent realloc
GrowableArray <Node *> bltstack(C->live_nodes() >> 1);
Node *n = C->root();
bltstack.push(n);
int pre_order = 1;
int stack_size;
while ( ( stack_size = bltstack.length() ) != 0 ) {
n = bltstack.top(); // Leave node on stack
if ( !is_visited(n) ) {
// ---- Pre-pass Work ----
// Pre-walked but not post-walked nodes need a pre_order number.
set_preorder_visited( n, pre_order ); // set as visited
// ---- Scan over children ----
// Scan first over control projections that lead to loop headers.
// This helps us find inner-to-outer loops with shared headers better.
// Scan children's children for loop headers.
for ( int i = n->outcnt() - 1; i >= 0; --i ) {
Node* m = n->raw_out(i); // Child
if( m->is_CFG() && !is_visited(m) ) { // Only for CFG children
// Scan over children's children to find loop
for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) {
Node* l = m->fast_out(j);
if( is_visited(l) && // Been visited?
!is_postvisited(l) && // But not post-visited
get_preorder(l) < pre_order ) { // And smaller pre-order
// Found! Scan the DFS down this path before doing other paths
bltstack.push(m);
break;
}
}
}
}
pre_order++;
}
else if ( !is_postvisited(n) ) {
// Note: build_loop_tree_impl() adds out edges on rare occasions,
// such as com.sun.rsasign.am::a.
// For non-recursive version, first, process current children.
// On next iteration, check if additional children were added.
for ( int k = n->outcnt() - 1; k >= 0; --k ) {
Node* u = n->raw_out(k);
if ( u->is_CFG() && !is_visited(u) ) {
bltstack.push(u);
}
}
if ( bltstack.length() == stack_size ) {
// There were no additional children, post visit node now
(void)bltstack.pop(); // Remove node from stack
pre_order = build_loop_tree_impl( n, pre_order );
// Check for bailout
if (C->failing()) {
return;
}
// Check to grow _preorders[] array for the case when
// build_loop_tree_impl() adds new nodes.
check_grow_preorders();
}
}
else {
(void)bltstack.pop(); // Remove post-visited node from stack
}
}
}
//------------------------------build_loop_tree_impl---------------------------
int PhaseIdealLoop::build_loop_tree_impl( Node *n, int pre_order ) {
// ---- Post-pass Work ----
// Pre-walked but not post-walked nodes need a pre_order number.
// Tightest enclosing loop for this Node
IdealLoopTree *innermost = NULL;
// For all children, see if any edge is a backedge. If so, make a loop
// for it. Then find the tightest enclosing loop for the self Node.
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
Node* m = n->fast_out(i); // Child
if( n == m ) continue; // Ignore control self-cycles
if( !m->is_CFG() ) continue;// Ignore non-CFG edges
IdealLoopTree *l; // Child's loop
if( !is_postvisited(m) ) { // Child visited but not post-visited?
// Found a backedge
assert( get_preorder(m) < pre_order, "should be backedge" );
// Check for the RootNode, which is already a LoopNode and is allowed
// to have multiple "backedges".
if( m == C->root()) { // Found the root?
l = _ltree_root; // Root is the outermost LoopNode
} else { // Else found a nested loop
// Insert a LoopNode to mark this loop.
l = new IdealLoopTree(this, m, n);
} // End of Else found a nested loop
if( !has_loop(m) ) // If 'm' does not already have a loop set
set_loop(m, l); // Set loop header to loop now
} else { // Else not a nested loop
if( !_nodes[m->_idx] ) continue; // Dead code has no loop
l = get_loop(m); // Get previously determined loop
// If successor is header of a loop (nest), move up-loop till it
// is a member of some outer enclosing loop. Since there are no
// shared headers (I've split them already) I only need to go up
// at most 1 level.
while( l && l->_head == m ) // Successor heads loop?
l = l->_parent; // Move up 1 for me
// If this loop is not properly parented, then this loop
// has no exit path out, i.e. its an infinite loop.
if( !l ) {
// Make loop "reachable" from root so the CFG is reachable. Basically
// insert a bogus loop exit that is never taken. 'm', the loop head,
// points to 'n', one (of possibly many) fall-in paths. There may be
// many backedges as well.
// Here I set the loop to be the root loop. I could have, after
// inserting a bogus loop exit, restarted the recursion and found my
// new loop exit. This would make the infinite loop a first-class
// loop and it would then get properly optimized. What's the use of
// optimizing an infinite loop?
l = _ltree_root; // Oops, found infinite loop
if (!_verify_only) {
// Insert the NeverBranch between 'm' and it's control user.
NeverBranchNode *iff = new (C) NeverBranchNode( m );
_igvn.register_new_node_with_optimizer(iff);
set_loop(iff, l);
Node *if_t = new (C) CProjNode( iff, 0 );
_igvn.register_new_node_with_optimizer(if_t);
set_loop(if_t, l);
Node* cfg = NULL; // Find the One True Control User of m
for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) {
Node* x = m->fast_out(j);
if (x->is_CFG() && x != m && x != iff)
{ cfg = x; break; }
}
assert(cfg != NULL, "must find the control user of m");
uint k = 0; // Probably cfg->in(0)
while( cfg->in(k) != m ) k++; // But check incase cfg is a Region
cfg->set_req( k, if_t ); // Now point to NeverBranch
// Now create the never-taken loop exit
Node *if_f = new (C) CProjNode( iff, 1 );
_igvn.register_new_node_with_optimizer(if_f);
set_loop(if_f, l);
// Find frame ptr for Halt. Relies on the optimizer
// V-N'ing. Easier and quicker than searching through
// the program structure.
Node *frame = new (C) ParmNode( C->start(), TypeFunc::FramePtr );
_igvn.register_new_node_with_optimizer(frame);
// Halt & Catch Fire
Node *halt = new (C) HaltNode( if_f, frame );
_igvn.register_new_node_with_optimizer(halt);
set_loop(halt, l);
C->root()->add_req(halt);
}
set_loop(C->root(), _ltree_root);
}
}
// Weeny check for irreducible. This child was already visited (this
// IS the post-work phase). Is this child's loop header post-visited
// as well? If so, then I found another entry into the loop.
if (!_verify_only) {
while( is_postvisited(l->_head) ) {
// found irreducible
l->_irreducible = 1; // = true
l = l->_parent;
_has_irreducible_loops = true;
// Check for bad CFG here to prevent crash, and bailout of compile
if (l == NULL) {
C->record_method_not_compilable("unhandled CFG detected during loop optimization");
return pre_order;
}
}
C->set_has_irreducible_loop(_has_irreducible_loops);
}
// This Node might be a decision point for loops. It is only if
// it's children belong to several different loops. The sort call
// does a trivial amount of work if there is only 1 child or all
// children belong to the same loop. If however, the children
// belong to different loops, the sort call will properly set the
// _parent pointers to show how the loops nest.
//
// In any case, it returns the tightest enclosing loop.
innermost = sort( l, innermost );
}
// Def-use info will have some dead stuff; dead stuff will have no
// loop decided on.
// Am I a loop header? If so fix up my parent's child and next ptrs.
if( innermost && innermost->_head == n ) {
assert( get_loop(n) == innermost, "" );
IdealLoopTree *p = innermost->_parent;
IdealLoopTree *l = innermost;
while( p && l->_head == n ) {
l->_next = p->_child; // Put self on parents 'next child'
p->_child = l; // Make self as first child of parent
l = p; // Now walk up the parent chain
p = l->_parent;
}
} else {
// Note that it is possible for a LoopNode to reach here, if the
// backedge has been made unreachable (hence the LoopNode no longer
// denotes a Loop, and will eventually be removed).
// Record tightest enclosing loop for self. Mark as post-visited.
set_loop(n, innermost);
// Also record has_call flag early on
if( innermost ) {
if( n->is_Call() && !n->is_CallLeaf() && !n->is_macro() ) {
// Do not count uncommon calls
if( !n->is_CallStaticJava() || !n->as_CallStaticJava()->_name ) {
Node *iff = n->in(0)->in(0);
// No any calls for vectorized loops.
if( UseSuperWord || !iff->is_If() ||
(n->in(0)->Opcode() == Op_IfFalse &&
(1.0 - iff->as_If()->_prob) >= 0.01) ||
(iff->as_If()->_prob >= 0.01) )
innermost->_has_call = 1;
}
} else if( n->is_Allocate() && n->as_Allocate()->_is_scalar_replaceable ) {
// Disable loop optimizations if the loop has a scalar replaceable
// allocation. This disabling may cause a potential performance lost
// if the allocation is not eliminated for some reason.
innermost->_allow_optimizations = false;
innermost->_has_call = 1; // = true
} else if (n->Opcode() == Op_SafePoint) {
// Record all safepoints in this loop.
if (innermost->_safepts == NULL) innermost->_safepts = new Node_List();
innermost->_safepts->push(n);
}
}
}
// Flag as post-visited now
set_postvisited(n);
return pre_order;
}
//------------------------------build_loop_early-------------------------------
// Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping.
// First pass computes the earliest controlling node possible. This is the
// controlling input with the deepest dominating depth.
void PhaseIdealLoop::build_loop_early( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ) {
while (worklist.size() != 0) {
// Use local variables nstack_top_n & nstack_top_i to cache values
// on nstack's top.
Node *nstack_top_n = worklist.pop();
uint nstack_top_i = 0;
//while_nstack_nonempty:
while (true) {
// Get parent node and next input's index from stack's top.
Node *n = nstack_top_n;
uint i = nstack_top_i;
uint cnt = n->req(); // Count of inputs
if (i == 0) { // Pre-process the node.
if( has_node(n) && // Have either loop or control already?
!has_ctrl(n) ) { // Have loop picked out already?
// During "merge_many_backedges" we fold up several nested loops
// into a single loop. This makes the members of the original
// loop bodies pointing to dead loops; they need to move up
// to the new UNION'd larger loop. I set the _head field of these
// dead loops to NULL and the _parent field points to the owning
// loop. Shades of UNION-FIND algorithm.
IdealLoopTree *ilt;
while( !(ilt = get_loop(n))->_head ) {
// Normally I would use a set_loop here. But in this one special
// case, it is legal (and expected) to change what loop a Node
// belongs to.
_nodes.map(n->_idx, (Node*)(ilt->_parent) );
}
// Remove safepoints ONLY if I've already seen I don't need one.
// (the old code here would yank a 2nd safepoint after seeing a
// first one, even though the 1st did not dominate in the loop body
// and thus could be avoided indefinitely)
if( !_verify_only && !_verify_me && ilt->_has_sfpt && n->Opcode() == Op_SafePoint &&
is_deleteable_safept(n)) {
Node *in = n->in(TypeFunc::Control);
lazy_replace(n,in); // Pull safepoint now
if (ilt->_safepts != NULL) {
ilt->_safepts->yank(n);
}
// Carry on with the recursion "as if" we are walking
// only the control input
if( !visited.test_set( in->_idx ) ) {
worklist.push(in); // Visit this guy later, using worklist
}
// Get next node from nstack:
// - skip n's inputs processing by setting i > cnt;
// - we also will not call set_early_ctrl(n) since
// has_node(n) == true (see the condition above).
i = cnt + 1;
}
}
} // if (i == 0)
// Visit all inputs
bool done = true; // Assume all n's inputs will be processed
while (i < cnt) {
Node *in = n->in(i);
++i;
if (in == NULL) continue;
if (in->pinned() && !in->is_CFG())
set_ctrl(in, in->in(0));
int is_visited = visited.test_set( in->_idx );
if (!has_node(in)) { // No controlling input yet?
assert( !in->is_CFG(), "CFG Node with no controlling input?" );
assert( !is_visited, "visit only once" );
nstack.push(n, i); // Save parent node and next input's index.
nstack_top_n = in; // Process current input now.
nstack_top_i = 0;
done = false; // Not all n's inputs processed.
break; // continue while_nstack_nonempty;
} else if (!is_visited) {
// This guy has a location picked out for him, but has not yet
// been visited. Happens to all CFG nodes, for instance.
// Visit him using the worklist instead of recursion, to break
// cycles. Since he has a location already we do not need to
// find his location before proceeding with the current Node.
worklist.push(in); // Visit this guy later, using worklist
}
}
if (done) {
// All of n's inputs have been processed, complete post-processing.
// Compute earliest point this Node can go.
// CFG, Phi, pinned nodes already know their controlling input.
if (!has_node(n)) {
// Record earliest legal location
set_early_ctrl( n );
}
if (nstack.is_empty()) {
// Finished all nodes on stack.
// Process next node on the worklist.
break;
}
// Get saved parent node and next input's index.
nstack_top_n = nstack.node();
nstack_top_i = nstack.index();
nstack.pop();
}
} // while (true)
}
}
//------------------------------dom_lca_internal--------------------------------
// Pair-wise LCA
Node *PhaseIdealLoop::dom_lca_internal( Node *n1, Node *n2 ) const {
if( !n1 ) return n2; // Handle NULL original LCA
assert( n1->is_CFG(), "" );
assert( n2->is_CFG(), "" );
// find LCA of all uses
uint d1 = dom_depth(n1);
uint d2 = dom_depth(n2);
while (n1 != n2) {
if (d1 > d2) {
n1 = idom(n1);
d1 = dom_depth(n1);
} else if (d1 < d2) {
n2 = idom(n2);
d2 = dom_depth(n2);
} else {
// Here d1 == d2. Due to edits of the dominator-tree, sections
// of the tree might have the same depth. These sections have
// to be searched more carefully.
// Scan up all the n1's with equal depth, looking for n2.
Node *t1 = idom(n1);
while (dom_depth(t1) == d1) {
if (t1 == n2) return n2;
t1 = idom(t1);
}
// Scan up all the n2's with equal depth, looking for n1.
Node *t2 = idom(n2);
while (dom_depth(t2) == d2) {
if (t2 == n1) return n1;
t2 = idom(t2);
}
// Move up to a new dominator-depth value as well as up the dom-tree.
n1 = t1;
n2 = t2;
d1 = dom_depth(n1);
d2 = dom_depth(n2);
}
}
return n1;
}
//------------------------------compute_idom-----------------------------------
// Locally compute IDOM using dom_lca call. Correct only if the incoming
// IDOMs are correct.
Node *PhaseIdealLoop::compute_idom( Node *region ) const {
assert( region->is_Region(), "" );
Node *LCA = NULL;
for( uint i = 1; i < region->req(); i++ ) {
if( region->in(i) != C->top() )
LCA = dom_lca( LCA, region->in(i) );
}
return LCA;
}
bool PhaseIdealLoop::verify_dominance(Node* n, Node* use, Node* LCA, Node* early) {
bool had_error = false;
#ifdef ASSERT
if (early != C->root()) {
// Make sure that there's a dominance path from LCA to early
Node* d = LCA;
while (d != early) {
if (d == C->root()) {
dump_bad_graph("Bad graph detected in compute_lca_of_uses", n, early, LCA);
tty->print_cr("*** Use %d isn't dominated by def %d ***", use->_idx, n->_idx);
had_error = true;
break;
}
d = idom(d);
}
}
#endif
return had_error;
}
Node* PhaseIdealLoop::compute_lca_of_uses(Node* n, Node* early, bool verify) {
// Compute LCA over list of uses
bool had_error = false;
Node *LCA = NULL;
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && LCA != early; i++) {
Node* c = n->fast_out(i);
if (_nodes[c->_idx] == NULL)
continue; // Skip the occasional dead node
if( c->is_Phi() ) { // For Phis, we must land above on the path
for( uint j=1; j<c->req(); j++ ) {// For all inputs
if( c->in(j) == n ) { // Found matching input?
Node *use = c->in(0)->in(j);
if (_verify_only && use->is_top()) continue;
LCA = dom_lca_for_get_late_ctrl( LCA, use, n );
if (verify) had_error = verify_dominance(n, use, LCA, early) || had_error;
}
}
} else {
// For CFG data-users, use is in the block just prior
Node *use = has_ctrl(c) ? get_ctrl(c) : c->in(0);
LCA = dom_lca_for_get_late_ctrl( LCA, use, n );
if (verify) had_error = verify_dominance(n, use, LCA, early) || had_error;
}
}
assert(!had_error, "bad dominance");
return LCA;
}
// Check the shape of the graph at the loop entry. In some cases,
// the shape of the graph does not match the shape outlined below.
// That is caused by the Opaque1 node "protecting" the shape of
// the graph being removed by, for example, the IGVN performed
// in PhaseIdealLoop::build_and_optimize().
//
// After the Opaque1 node has been removed, optimizations (e.g., split-if,
// loop unswitching, and IGVN, or a combination of them) can freely change
// the graph's shape. As a result, the graph shape outlined below cannot
// be guaranteed anymore.
bool PhaseIdealLoop::is_canonical_main_loop_entry(CountedLoopNode* cl) {
assert(cl->is_main_loop(), "check should be applied to main loops");
Node* ctrl = cl->in(LoopNode::EntryControl);
if (ctrl == NULL || (!ctrl->is_IfTrue() && !ctrl->is_IfFalse())) {
return false;
}
Node* iffm = ctrl->in(0);
if (iffm == NULL || !iffm->is_If()) {
return false;
}
Node* bolzm = iffm->in(1);
if (bolzm == NULL || !bolzm->is_Bool()) {
return false;
}
Node* cmpzm = bolzm->in(1);
if (cmpzm == NULL || !cmpzm->is_Cmp()) {
return false;
}
Node* opqzm = cmpzm->in(2);
if (opqzm == NULL || opqzm->Opcode() != Op_Opaque1) {
return false;
}
return true;
}
//------------------------------get_late_ctrl----------------------------------
// Compute latest legal control.
Node *PhaseIdealLoop::get_late_ctrl( Node *n, Node *early ) {
assert(early != NULL, "early control should not be NULL");
Node* LCA = compute_lca_of_uses(n, early);
#ifdef ASSERT
if (LCA == C->root() && LCA != early) {
// def doesn't dominate uses so print some useful debugging output
compute_lca_of_uses(n, early, true);
}
#endif
// if this is a load, check for anti-dependent stores
// We use a conservative algorithm to identify potential interfering
// instructions and for rescheduling the load. The users of the memory
// input of this load are examined. Any use which is not a load and is
// dominated by early is considered a potentially interfering store.
// This can produce false positives.
if (n->is_Load() && LCA != early) {
Node_List worklist;
Node *mem = n->in(MemNode::Memory);
for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
Node* s = mem->fast_out(i);
worklist.push(s);
}
while(worklist.size() != 0 && LCA != early) {
Node* s = worklist.pop();
if (s->is_Load()) {
continue;
} else if (s->is_MergeMem()) {
for (DUIterator_Fast imax, i = s->fast_outs(imax); i < imax; i++) {
Node* s1 = s->fast_out(i);
worklist.push(s1);
}
} else {
Node *sctrl = has_ctrl(s) ? get_ctrl(s) : s->in(0);
assert(sctrl != NULL || s->outcnt() == 0, "must have control");
if (sctrl != NULL && !sctrl->is_top() && is_dominator(early, sctrl)) {
LCA = dom_lca_for_get_late_ctrl(LCA, sctrl, n);
}
}
}
}
assert(LCA == find_non_split_ctrl(LCA), "unexpected late control");
return LCA;
}
// true if CFG node d dominates CFG node n
bool PhaseIdealLoop::is_dominator(Node *d, Node *n) {
if (d == n)
return true;
assert(d->is_CFG() && n->is_CFG(), "must have CFG nodes");
uint dd = dom_depth(d);
while (dom_depth(n) >= dd) {
if (n == d)
return true;
n = idom(n);
}
return false;
}
//------------------------------dom_lca_for_get_late_ctrl_internal-------------
// Pair-wise LCA with tags.
// Tag each index with the node 'tag' currently being processed
// before advancing up the dominator chain using idom().
// Later calls that find a match to 'tag' know that this path has already
// been considered in the current LCA (which is input 'n1' by convention).
// Since get_late_ctrl() is only called once for each node, the tag array
// does not need to be cleared between calls to get_late_ctrl().
// Algorithm trades a larger constant factor for better asymptotic behavior
//
Node *PhaseIdealLoop::dom_lca_for_get_late_ctrl_internal( Node *n1, Node *n2, Node *tag ) {
uint d1 = dom_depth(n1);
uint d2 = dom_depth(n2);
do {
if (d1 > d2) {
// current lca is deeper than n2
_dom_lca_tags.map(n1->_idx, tag);
n1 = idom(n1);
d1 = dom_depth(n1);
} else if (d1 < d2) {
// n2 is deeper than current lca
Node *memo = _dom_lca_tags[n2->_idx];
if( memo == tag ) {
return n1; // Return the current LCA
}
_dom_lca_tags.map(n2->_idx, tag);
n2 = idom(n2);
d2 = dom_depth(n2);
} else {
// Here d1 == d2. Due to edits of the dominator-tree, sections
// of the tree might have the same depth. These sections have
// to be searched more carefully.
// Scan up all the n1's with equal depth, looking for n2.
_dom_lca_tags.map(n1->_idx, tag);
Node *t1 = idom(n1);
while (dom_depth(t1) == d1) {
if (t1 == n2) return n2;
_dom_lca_tags.map(t1->_idx, tag);
t1 = idom(t1);
}
// Scan up all the n2's with equal depth, looking for n1.
_dom_lca_tags.map(n2->_idx, tag);
Node *t2 = idom(n2);
while (dom_depth(t2) == d2) {
if (t2 == n1) return n1;
_dom_lca_tags.map(t2->_idx, tag);
t2 = idom(t2);
}
// Move up to a new dominator-depth value as well as up the dom-tree.
n1 = t1;
n2 = t2;
d1 = dom_depth(n1);
d2 = dom_depth(n2);
}
} while (n1 != n2);
return n1;
}
//------------------------------init_dom_lca_tags------------------------------
// Tag could be a node's integer index, 32bits instead of 64bits in some cases
// Intended use does not involve any growth for the array, so it could
// be of fixed size.
void PhaseIdealLoop::init_dom_lca_tags() {
uint limit = C->unique() + 1;
_dom_lca_tags.map( limit, NULL );
#ifdef ASSERT
for( uint i = 0; i < limit; ++i ) {
assert(_dom_lca_tags[i] == NULL, "Must be distinct from each node pointer");
}
#endif // ASSERT
}
//------------------------------clear_dom_lca_tags------------------------------
// Tag could be a node's integer index, 32bits instead of 64bits in some cases
// Intended use does not involve any growth for the array, so it could
// be of fixed size.
void PhaseIdealLoop::clear_dom_lca_tags() {
uint limit = C->unique() + 1;
_dom_lca_tags.map( limit, NULL );
_dom_lca_tags.clear();
#ifdef ASSERT
for( uint i = 0; i < limit; ++i ) {
assert(_dom_lca_tags[i] == NULL, "Must be distinct from each node pointer");
}
#endif // ASSERT
}
//------------------------------build_loop_late--------------------------------
// Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping.
// Second pass finds latest legal placement, and ideal loop placement.
void PhaseIdealLoop::build_loop_late( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ) {
while (worklist.size() != 0) {
Node *n = worklist.pop();
// Only visit once
if (visited.test_set(n->_idx)) continue;
uint cnt = n->outcnt();
uint i = 0;
while (true) {
assert( _nodes[n->_idx], "no dead nodes" );
// Visit all children
if (i < cnt) {
Node* use = n->raw_out(i);
++i;
// Check for dead uses. Aggressively prune such junk. It might be
// dead in the global sense, but still have local uses so I cannot
// easily call 'remove_dead_node'.
if( _nodes[use->_idx] != NULL || use->is_top() ) { // Not dead?
// Due to cycles, we might not hit the same fixed point in the verify
// pass as we do in the regular pass. Instead, visit such phis as
// simple uses of the loop head.
if( use->in(0) && (use->is_CFG() || use->is_Phi()) ) {
if( !visited.test(use->_idx) )
worklist.push(use);
} else if( !visited.test_set(use->_idx) ) {
nstack.push(n, i); // Save parent and next use's index.
n = use; // Process all children of current use.
cnt = use->outcnt();
i = 0;
}
} else {
// Do not visit around the backedge of loops via data edges.
// push dead code onto a worklist
_deadlist.push(use);
}
} else {
// All of n's children have been processed, complete post-processing.
build_loop_late_post(n);
if (nstack.is_empty()) {
// Finished all nodes on stack.
// Process next node on the worklist.
break;
}
// Get saved parent node and next use's index. Visit the rest of uses.
n = nstack.node();
cnt = n->outcnt();
i = nstack.index();
nstack.pop();
}
}
}
}
//------------------------------build_loop_late_post---------------------------
// Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping.
// Second pass finds latest legal placement, and ideal loop placement.
void PhaseIdealLoop::build_loop_late_post( Node *n ) {
if (n->req() == 2 && n->Opcode() == Op_ConvI2L && !C->major_progress() && !_verify_only) {
_igvn._worklist.push(n); // Maybe we'll normalize it, if no more loops.
}
#ifdef ASSERT
if (_verify_only && !n->is_CFG()) {
// Check def-use domination.
compute_lca_of_uses(n, get_ctrl(n), true /* verify */);
}
#endif
// CFG and pinned nodes already handled
if( n->in(0) ) {
if( n->in(0)->is_top() ) return; // Dead?
// We'd like +VerifyLoopOptimizations to not believe that Mod's/Loads
// _must_ be pinned (they have to observe their control edge of course).
// Unlike Stores (which modify an unallocable resource, the memory
// state), Mods/Loads can float around. So free them up.
bool pinned = true;
switch( n->Opcode() ) {
case Op_DivI:
case Op_DivF:
case Op_DivD:
case Op_ModI:
case Op_ModF:
case Op_ModD:
case Op_LoadB: // Same with Loads; they can sink
case Op_LoadUB: // during loop optimizations.
case Op_LoadUS:
case Op_LoadD:
case Op_LoadF:
case Op_LoadI:
case Op_LoadKlass:
case Op_LoadNKlass:
case Op_LoadL:
case Op_LoadS:
case Op_LoadP:
case Op_LoadN:
case Op_LoadRange:
case Op_LoadD_unaligned:
case Op_LoadL_unaligned:
case Op_StrComp: // Does a bunch of load-like effects
case Op_StrEquals:
case Op_StrIndexOf:
case Op_AryEq:
pinned = false;
}
if( pinned ) {
IdealLoopTree *chosen_loop = get_loop(n->is_CFG() ? n : get_ctrl(n));
if( !chosen_loop->_child ) // Inner loop?
chosen_loop->_body.push(n); // Collect inner loops
return;
}
} else { // No slot zero
if( n->is_CFG() ) { // CFG with no slot 0 is dead
_nodes.map(n->_idx,0); // No block setting, it's globally dead
return;
}
assert(!n->is_CFG() || n->outcnt() == 0, "");
}
// Do I have a "safe range" I can select over?
Node *early = get_ctrl(n);// Early location already computed
// Compute latest point this Node can go
Node *LCA = get_late_ctrl( n, early );
// LCA is NULL due to uses being dead
if( LCA == NULL ) {
#ifdef ASSERT
for (DUIterator i1 = n->outs(); n->has_out(i1); i1++) {
assert( _nodes[n->out(i1)->_idx] == NULL, "all uses must also be dead");
}
#endif
_nodes.map(n->_idx, 0); // This node is useless
_deadlist.push(n);
return;
}
assert(LCA != NULL && !LCA->is_top(), "no dead nodes");
Node *legal = LCA; // Walk 'legal' up the IDOM chain
Node *least = legal; // Best legal position so far
while( early != legal ) { // While not at earliest legal
#ifdef ASSERT
if (legal->is_Start() && !early->is_Root()) {
// Bad graph. Print idom path and fail.
dump_bad_graph("Bad graph detected in build_loop_late", n, early, LCA);
assert(false, "Bad graph detected in build_loop_late");
}
#endif
// Find least loop nesting depth
legal = idom(legal); // Bump up the IDOM tree
// Check for lower nesting depth
if( get_loop(legal)->_nest < get_loop(least)->_nest )
least = legal;
}
assert(early == legal || legal != C->root(), "bad dominance of inputs");
// Try not to place code on a loop entry projection
// which can inhibit range check elimination.
if (least != early) {
Node* ctrl_out = least->unique_ctrl_out();
if (ctrl_out && ctrl_out->is_CountedLoop() &&
least == ctrl_out->in(LoopNode::EntryControl)) {
Node* least_dom = idom(least);
if (get_loop(least_dom)->is_member(get_loop(least))) {
least = least_dom;
}
}
}
#ifdef ASSERT
// If verifying, verify that 'verify_me' has a legal location
// and choose it as our location.
if( _verify_me ) {
Node *v_ctrl = _verify_me->get_ctrl_no_update(n);
Node *legal = LCA;
while( early != legal ) { // While not at earliest legal
if( legal == v_ctrl ) break; // Check for prior good location
legal = idom(legal) ;// Bump up the IDOM tree
}
// Check for prior good location
if( legal == v_ctrl ) least = legal; // Keep prior if found
}
#endif
// Assign discovered "here or above" point
least = find_non_split_ctrl(least);
set_ctrl(n, least);
// Collect inner loop bodies
IdealLoopTree *chosen_loop = get_loop(least);
if( !chosen_loop->_child ) // Inner loop?
chosen_loop->_body.push(n);// Collect inner loops
}
#ifdef ASSERT
void PhaseIdealLoop::dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA) {
tty->print_cr("%s", msg);
tty->print("n: "); n->dump();
tty->print("early(n): "); early->dump();
if (n->in(0) != NULL && !n->in(0)->is_top() &&
n->in(0) != early && !n->in(0)->is_Root()) {
tty->print("n->in(0): "); n->in(0)->dump();
}
for (uint i = 1; i < n->req(); i++) {
Node* in1 = n->in(i);
if (in1 != NULL && in1 != n && !in1->is_top()) {
tty->print("n->in(%d): ", i); in1->dump();
Node* in1_early = get_ctrl(in1);
tty->print("early(n->in(%d)): ", i); in1_early->dump();
if (in1->in(0) != NULL && !in1->in(0)->is_top() &&
in1->in(0) != in1_early && !in1->in(0)->is_Root()) {
tty->print("n->in(%d)->in(0): ", i); in1->in(0)->dump();
}
for (uint j = 1; j < in1->req(); j++) {
Node* in2 = in1->in(j);
if (in2 != NULL && in2 != n && in2 != in1 && !in2->is_top()) {
tty->print("n->in(%d)->in(%d): ", i, j); in2->dump();
Node* in2_early = get_ctrl(in2);
tty->print("early(n->in(%d)->in(%d)): ", i, j); in2_early->dump();
if (in2->in(0) != NULL && !in2->in(0)->is_top() &&
in2->in(0) != in2_early && !in2->in(0)->is_Root()) {
tty->print("n->in(%d)->in(%d)->in(0): ", i, j); in2->in(0)->dump();
}
}
}
}
}
tty->cr();
tty->print("LCA(n): "); LCA->dump();
for (uint i = 0; i < n->outcnt(); i++) {
Node* u1 = n->raw_out(i);
if (u1 == n)
continue;
tty->print("n->out(%d): ", i); u1->dump();
if (u1->is_CFG()) {
for (uint j = 0; j < u1->outcnt(); j++) {
Node* u2 = u1->raw_out(j);
if (u2 != u1 && u2 != n && u2->is_CFG()) {
tty->print("n->out(%d)->out(%d): ", i, j); u2->dump();
}
}
} else {
Node* u1_later = get_ctrl(u1);
tty->print("later(n->out(%d)): ", i); u1_later->dump();
if (u1->in(0) != NULL && !u1->in(0)->is_top() &&
u1->in(0) != u1_later && !u1->in(0)->is_Root()) {
tty->print("n->out(%d)->in(0): ", i); u1->in(0)->dump();
}
for (uint j = 0; j < u1->outcnt(); j++) {
Node* u2 = u1->raw_out(j);
if (u2 == n || u2 == u1)
continue;
tty->print("n->out(%d)->out(%d): ", i, j); u2->dump();
if (!u2->is_CFG()) {
Node* u2_later = get_ctrl(u2);
tty->print("later(n->out(%d)->out(%d)): ", i, j); u2_later->dump();
if (u2->in(0) != NULL && !u2->in(0)->is_top() &&
u2->in(0) != u2_later && !u2->in(0)->is_Root()) {
tty->print("n->out(%d)->in(0): ", i); u2->in(0)->dump();
}
}
}
}
}
tty->cr();
int ct = 0;
Node *dbg_legal = LCA;
while(!dbg_legal->is_Start() && ct < 100) {
tty->print("idom[%d] ",ct); dbg_legal->dump();
ct++;
dbg_legal = idom(dbg_legal);
}
tty->cr();
}
#endif
#ifndef PRODUCT
//------------------------------dump-------------------------------------------
void PhaseIdealLoop::dump( ) const {
ResourceMark rm;
Arena* arena = Thread::current()->resource_area();
Node_Stack stack(arena, C->live_nodes() >> 2);
Node_List rpo_list;
VectorSet visited(arena);
visited.set(C->top()->_idx);
rpo( C->root(), stack, visited, rpo_list );
// Dump root loop indexed by last element in PO order
dump( _ltree_root, rpo_list.size(), rpo_list );
}
void PhaseIdealLoop::dump( IdealLoopTree *loop, uint idx, Node_List &rpo_list ) const {
loop->dump_head();
// Now scan for CFG nodes in the same loop
for( uint j=idx; j > 0; j-- ) {
Node *n = rpo_list[j-1];
if( !_nodes[n->_idx] ) // Skip dead nodes
continue;
if( get_loop(n) != loop ) { // Wrong loop nest
if( get_loop(n)->_head == n && // Found nested loop?
get_loop(n)->_parent == loop )
dump(get_loop(n),rpo_list.size(),rpo_list); // Print it nested-ly
continue;
}
// Dump controlling node
for( uint x = 0; x < loop->_nest; x++ )
tty->print(" ");
tty->print("C");
if( n == C->root() ) {
n->dump();
} else {
Node* cached_idom = idom_no_update(n);
Node *computed_idom = n->in(0);
if( n->is_Region() ) {
computed_idom = compute_idom(n);
// computed_idom() will return n->in(0) when idom(n) is an IfNode (or
// any MultiBranch ctrl node), so apply a similar transform to
// the cached idom returned from idom_no_update.
cached_idom = find_non_split_ctrl(cached_idom);
}
tty->print(" ID:%d",computed_idom->_idx);
n->dump();
if( cached_idom != computed_idom ) {
tty->print_cr("*** BROKEN IDOM! Computed as: %d, cached as: %d",
computed_idom->_idx, cached_idom->_idx);
}
}
// Dump nodes it controls
for( uint k = 0; k < _nodes.Size(); k++ ) {
// (k < C->unique() && get_ctrl(find(k)) == n)
if (k < C->unique() && _nodes[k] == (Node*)((intptr_t)n + 1)) {
Node *m = C->root()->find(k);
if( m && m->outcnt() > 0 ) {
if (!(has_ctrl(m) && get_ctrl_no_update(m) == n)) {
tty->print_cr("*** BROKEN CTRL ACCESSOR! _nodes[k] is %p, ctrl is %p",
_nodes[k], has_ctrl(m) ? get_ctrl_no_update(m) : NULL);
}
for( uint j = 0; j < loop->_nest; j++ )
tty->print(" ");
tty->print(" ");
m->dump();
}
}
}
}
}
// Collect a R-P-O for the whole CFG.
// Result list is in post-order (scan backwards for RPO)
void PhaseIdealLoop::rpo( Node *start, Node_Stack &stk, VectorSet &visited, Node_List &rpo_list ) const {
stk.push(start, 0);
visited.set(start->_idx);
while (stk.is_nonempty()) {
Node* m = stk.node();
uint idx = stk.index();
if (idx < m->outcnt()) {
stk.set_index(idx + 1);
Node* n = m->raw_out(idx);
if (n->is_CFG() && !visited.test_set(n->_idx)) {
stk.push(n, 0);
}
} else {
rpo_list.push(m);
stk.pop();
}
}
}
#endif
//=============================================================================
//------------------------------LoopTreeIterator-----------------------------------
// Advance to next loop tree using a preorder, left-to-right traversal.
void LoopTreeIterator::next() {
assert(!done(), "must not be done.");
if (_curnt->_child != NULL) {
_curnt = _curnt->_child;
} else if (_curnt->_next != NULL) {
_curnt = _curnt->_next;
} else {
while (_curnt != _root && _curnt->_next == NULL) {
_curnt = _curnt->_parent;
}
if (_curnt == _root) {
_curnt = NULL;
assert(done(), "must be done.");
} else {
assert(_curnt->_next != NULL, "must be more to do");
_curnt = _curnt->_next;
}
}
}
C:\hotspot-69087d08d473\src\share\vm/opto/loopnode.hpp
/*
* Copyright (c) 1998, 2017, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_VM_OPTO_LOOPNODE_HPP
#define SHARE_VM_OPTO_LOOPNODE_HPP
#include "opto/cfgnode.hpp"
#include "opto/multnode.hpp"
#include "opto/phaseX.hpp"
#include "opto/subnode.hpp"
#include "opto/type.hpp"
class CmpNode;
class CountedLoopEndNode;
class CountedLoopNode;
class IdealLoopTree;
class LoopNode;
class Node;
class PhaseIdealLoop;
class VectorSet;
class Invariance;
struct small_cache;
//
// I D E A L I Z E D L O O P S
//
// Idealized loops are the set of loops I perform more interesting
// transformations on, beyond simple hoisting.
//------------------------------LoopNode---------------------------------------
// Simple loop header. Fall in path on left, loop-back path on right.
class LoopNode : public RegionNode {
// Size is bigger to hold the flags. However, the flags do not change
// the semantics so it does not appear in the hash & cmp functions.
virtual uint size_of() const { return sizeof(*this); }
protected:
short _loop_flags;
// Names for flag bitfields
enum { Normal=0, Pre=1, Main=2, Post=3, PreMainPostFlagsMask=3,
MainHasNoPreLoop=4,
HasExactTripCount=8,
InnerLoop=16,
PartialPeelLoop=32,
PartialPeelFailed=64 };
char _unswitch_count;
enum { _unswitch_max=3 };
public:
// Names for edge indices
enum { Self=0, EntryControl, LoopBackControl };
int is_inner_loop() const { return _loop_flags & InnerLoop; }
void set_inner_loop() { _loop_flags |= InnerLoop; }
int is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; }
void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; }
int partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; }
void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; }
int unswitch_max() { return _unswitch_max; }
int unswitch_count() { return _unswitch_count; }
void set_unswitch_count(int val) {
assert (val <= unswitch_max(), "too many unswitches");
_unswitch_count = val;
}
LoopNode( Node *entry, Node *backedge ) : RegionNode(3), _loop_flags(0), _unswitch_count(0) {
init_class_id(Class_Loop);
init_req(EntryControl, entry);
init_req(LoopBackControl, backedge);
}
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
virtual int Opcode() const;
bool can_be_counted_loop(PhaseTransform* phase) const {
return req() == 3 && in(0) != NULL &&
in(1) != NULL && phase->type(in(1)) != Type::TOP &&
in(2) != NULL && phase->type(in(2)) != Type::TOP;
}
bool is_valid_counted_loop() const;
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
//------------------------------Counted Loops----------------------------------
// Counted loops are all trip-counted loops, with exactly 1 trip-counter exit
// path (and maybe some other exit paths). The trip-counter exit is always
// last in the loop. The trip-counter have to stride by a constant;
// the exit value is also loop invariant.
// CountedLoopNodes and CountedLoopEndNodes come in matched pairs. The
// CountedLoopNode has the incoming loop control and the loop-back-control
// which is always the IfTrue before the matching CountedLoopEndNode. The
// CountedLoopEndNode has an incoming control (possibly not the
// CountedLoopNode if there is control flow in the loop), the post-increment
// trip-counter value, and the limit. The trip-counter value is always of
// the form (Op old-trip-counter stride). The old-trip-counter is produced
// by a Phi connected to the CountedLoopNode. The stride is constant.
// The Op is any commutable opcode, including Add, Mul, Xor. The
// CountedLoopEndNode also takes in the loop-invariant limit value.
// From a CountedLoopNode I can reach the matching CountedLoopEndNode via the
// loop-back control. From CountedLoopEndNodes I can reach CountedLoopNodes
// via the old-trip-counter from the Op node.
//------------------------------CountedLoopNode--------------------------------
// CountedLoopNodes head simple counted loops. CountedLoopNodes have as
// inputs the incoming loop-start control and the loop-back control, so they
// act like RegionNodes. They also take in the initial trip counter, the
// loop-invariant stride and the loop-invariant limit value. CountedLoopNodes
// produce a loop-body control and the trip counter value. Since
// CountedLoopNodes behave like RegionNodes I still have a standard CFG model.
class CountedLoopNode : public LoopNode {
// Size is bigger to hold _main_idx. However, _main_idx does not change
// the semantics so it does not appear in the hash & cmp functions.
virtual uint size_of() const { return sizeof(*this); }
// For Pre- and Post-loops during debugging ONLY, this holds the index of
// the Main CountedLoop. Used to assert that we understand the graph shape.
node_idx_t _main_idx;
// Known trip count calculated by compute_exact_trip_count()
uint _trip_count;
// Expected trip count from profile data
float _profile_trip_cnt;
// Log2 of original loop bodies in unrolled loop
int _unrolled_count_log2;
// Node count prior to last unrolling - used to decide if
// unroll,optimize,unroll,optimize,... is making progress
int _node_count_before_unroll;
public:
CountedLoopNode( Node *entry, Node *backedge )
: LoopNode(entry, backedge), _main_idx(0), _trip_count(max_juint),
_profile_trip_cnt(COUNT_UNKNOWN), _unrolled_count_log2(0),
_node_count_before_unroll(0) {
init_class_id(Class_CountedLoop);
// Initialize _trip_count to the largest possible value.
// Will be reset (lower) if the loop's trip count is known.
}
virtual int Opcode() const;
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
Node *init_control() const { return in(EntryControl); }
Node *back_control() const { return in(LoopBackControl); }
CountedLoopEndNode *loopexit() const;
Node *init_trip() const;
Node *stride() const;
int stride_con() const;
bool stride_is_con() const;
Node *limit() const;
Node *incr() const;
Node *phi() const;
// Match increment with optional truncation
static Node* match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, const TypeInt** trunc_type);
// A 'main' loop has a pre-loop and a post-loop. The 'main' loop
// can run short a few iterations and may start a few iterations in.
// It will be RCE'd and unrolled and aligned.
// A following 'post' loop will run any remaining iterations. Used
// during Range Check Elimination, the 'post' loop will do any final
// iterations with full checks. Also used by Loop Unrolling, where
// the 'post' loop will do any epilog iterations needed. Basically,
// a 'post' loop can not profitably be further unrolled or RCE'd.
// A preceding 'pre' loop will run at least 1 iteration (to do peeling),
// it may do under-flow checks for RCE and may do alignment iterations
// so the following main loop 'knows' that it is striding down cache
// lines.
// A 'main' loop that is ONLY unrolled or peeled, never RCE'd or
// Aligned, may be missing it's pre-loop.
int is_normal_loop() const { return (_loop_flags&PreMainPostFlagsMask) == Normal; }
int is_pre_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Pre; }
int is_main_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Main; }
int is_post_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Post; }
int is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; }
void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; }
int main_idx() const { return _main_idx; }
void set_pre_loop (CountedLoopNode *main) { assert(is_normal_loop(),""); _loop_flags |= Pre ; _main_idx = main->_idx; }
void set_main_loop ( ) { assert(is_normal_loop(),""); _loop_flags |= Main; }
void set_post_loop (CountedLoopNode *main) { assert(is_normal_loop(),""); _loop_flags |= Post; _main_idx = main->_idx; }
void set_normal_loop( ) { _loop_flags &= ~PreMainPostFlagsMask; }
void set_trip_count(uint tc) { _trip_count = tc; }
uint trip_count() { return _trip_count; }
bool has_exact_trip_count() const { return (_loop_flags & HasExactTripCount) != 0; }
void set_exact_trip_count(uint tc) {
_trip_count = tc;
_loop_flags |= HasExactTripCount;
}
void set_nonexact_trip_count() {
_loop_flags &= ~HasExactTripCount;
}
void set_profile_trip_cnt(float ptc) { _profile_trip_cnt = ptc; }
float profile_trip_cnt() { return _profile_trip_cnt; }
void double_unrolled_count() { _unrolled_count_log2++; }
int unrolled_count() { return 1 << MIN2(_unrolled_count_log2, BitsPerInt-3); }
void set_node_count_before_unroll(int ct) { _node_count_before_unroll = ct; }
int node_count_before_unroll() { return _node_count_before_unroll; }
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
//------------------------------CountedLoopEndNode-----------------------------
// CountedLoopEndNodes end simple trip counted loops. They act much like
// IfNodes.
class CountedLoopEndNode : public IfNode {
public:
enum { TestControl, TestValue };
CountedLoopEndNode( Node *control, Node *test, float prob, float cnt )
: IfNode( control, test, prob, cnt) {
init_class_id(Class_CountedLoopEnd);
}
virtual int Opcode() const;
Node *cmp_node() const { return (in(TestValue)->req() >=2) ? in(TestValue)->in(1) : NULL; }
Node *incr() const { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }
Node *limit() const { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }
Node *stride() const { Node *tmp = incr (); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }
Node *init_trip() const { Node *tmp = phi (); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }
int stride_con() const;
bool stride_is_con() const { Node *tmp = stride (); return (tmp != NULL && tmp->is_Con()); }
BoolTest::mask test_trip() const { return in(TestValue)->as_Bool()->_test._test; }
PhiNode *phi() const {
Node *tmp = incr();
if (tmp && tmp->req() == 3) {
Node* phi = tmp->in(1);
if (phi->is_Phi()) {
return phi->as_Phi();
}
}
return NULL;
}
CountedLoopNode *loopnode() const {
// The CountedLoopNode that goes with this CountedLoopEndNode may
// have been optimized out by the IGVN so be cautious with the
// pattern matching on the graph
PhiNode* iv_phi = phi();
if (iv_phi == NULL) {
return NULL;
}
assert(iv_phi->is_Phi(), "should be PhiNode");
Node *ln = iv_phi->in(0);
if (ln->is_CountedLoop() && ln->as_CountedLoop()->loopexit() == this) {
return (CountedLoopNode*)ln;
}
return NULL;
}
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
inline CountedLoopEndNode *CountedLoopNode::loopexit() const {
Node *bc = back_control();
if( bc == NULL ) return NULL;
Node *le = bc->in(0);
if( le->Opcode() != Op_CountedLoopEnd )
return NULL;
return (CountedLoopEndNode*)le;
}
inline Node *CountedLoopNode::init_trip() const { return loopexit() ? loopexit()->init_trip() : NULL; }
inline Node *CountedLoopNode::stride() const { return loopexit() ? loopexit()->stride() : NULL; }
inline int CountedLoopNode::stride_con() const { return loopexit() ? loopexit()->stride_con() : 0; }
inline bool CountedLoopNode::stride_is_con() const { return loopexit() && loopexit()->stride_is_con(); }
inline Node *CountedLoopNode::limit() const { return loopexit() ? loopexit()->limit() : NULL; }
inline Node *CountedLoopNode::incr() const { return loopexit() ? loopexit()->incr() : NULL; }
inline Node *CountedLoopNode::phi() const { return loopexit() ? loopexit()->phi() : NULL; }
//------------------------------LoopLimitNode-----------------------------
// Counted Loop limit node which represents exact final iterator value:
// trip_count = (limit - init_trip + stride - 1)/stride
// final_value= trip_count * stride + init_trip.
// Use HW instructions to calculate it when it can overflow in integer.
// Note, final_value should fit into integer since counted loop has
// limit check: limit <= max_int-stride.
class LoopLimitNode : public Node {
enum { Init=1, Limit=2, Stride=3 };
public:
LoopLimitNode( Compile* C, Node *init, Node *limit, Node *stride ) : Node(0,init,limit,stride) {
// Put it on the Macro nodes list to optimize during macro nodes expansion.
init_flags(Flag_is_macro);
C->add_macro_node(this);
}
virtual int Opcode() const;
virtual const Type *bottom_type() const { return TypeInt::INT; }
virtual uint ideal_reg() const { return Op_RegI; }
virtual const Type *Value( PhaseTransform *phase ) const;
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
virtual Node *Identity( PhaseTransform *phase );
};
// -----------------------------IdealLoopTree----------------------------------
class IdealLoopTree : public ResourceObj {
public:
IdealLoopTree *_parent; // Parent in loop tree
IdealLoopTree *_next; // Next sibling in loop tree
IdealLoopTree *_child; // First child in loop tree
// The head-tail backedge defines the loop.
// If tail is NULL then this loop has multiple backedges as part of the
// same loop. During cleanup I'll peel off the multiple backedges; merge
// them at the loop bottom and flow 1 real backedge into the loop.
Node *_head; // Head of loop
Node *_tail; // Tail of loop
inline Node *tail(); // Handle lazy update of _tail field
PhaseIdealLoop* _phase;
Node_List _body; // Loop body for inner loops
uint8 _nest; // Nesting depth
uint8 _irreducible:1, // True if irreducible
_has_call:1, // True if has call safepoint
_has_sfpt:1, // True if has non-call safepoint
_rce_candidate:1; // True if candidate for range check elimination
Node_List* _safepts; // List of safepoints in this loop
Node_List* _required_safept; // A inner loop cannot delete these safepts;
bool _allow_optimizations; // Allow loop optimizations
IdealLoopTree( PhaseIdealLoop* phase, Node *head, Node *tail )
: _parent(0), _next(0), _child(0),
_head(head), _tail(tail),
_phase(phase),
_safepts(NULL),
_required_safept(NULL),
_allow_optimizations(true),
_nest(0), _irreducible(0), _has_call(0), _has_sfpt(0), _rce_candidate(0)
{ }
// Is 'l' a member of 'this'?
int is_member( const IdealLoopTree *l ) const; // Test for nested membership
// Set loop nesting depth. Accumulate has_call bits.
int set_nest( uint depth );
// Split out multiple fall-in edges from the loop header. Move them to a
// private RegionNode before the loop. This becomes the loop landing pad.
void split_fall_in( PhaseIdealLoop *phase, int fall_in_cnt );
// Split out the outermost loop from this shared header.
void split_outer_loop( PhaseIdealLoop *phase );
// Merge all the backedges from the shared header into a private Region.
// Feed that region as the one backedge to this loop.
void merge_many_backedges( PhaseIdealLoop *phase );
// Split shared headers and insert loop landing pads.
// Insert a LoopNode to replace the RegionNode.
// Returns TRUE if loop tree is structurally changed.
bool beautify_loops( PhaseIdealLoop *phase );
// Perform optimization to use the loop predicates for null checks and range checks.
// Applies to any loop level (not just the innermost one)
bool loop_predication( PhaseIdealLoop *phase);
// Perform iteration-splitting on inner loops. Split iterations to
// avoid range checks or one-shot null checks. Returns false if the
// current round of loop opts should stop.
bool iteration_split( PhaseIdealLoop *phase, Node_List &old_new );
// Driver for various flavors of iteration splitting. Returns false
// if the current round of loop opts should stop.
bool iteration_split_impl( PhaseIdealLoop *phase, Node_List &old_new );
// Given dominators, try to find loops with calls that must always be
// executed (call dominates loop tail). These loops do not need non-call
// safepoints (ncsfpt).
void check_safepts(VectorSet &visited, Node_List &stack);
// Allpaths backwards scan from loop tail, terminating each path at first safepoint
// encountered.
void allpaths_check_safepts(VectorSet &visited, Node_List &stack);
// Remove safepoints from loop. Optionally keeping one.
void remove_safepoints(PhaseIdealLoop* phase, bool keep_one);
// Convert to counted loops where possible
void counted_loop( PhaseIdealLoop *phase );
// Check for Node being a loop-breaking test
Node *is_loop_exit(Node *iff) const;
// Returns true if ctrl is executed on every complete iteration
bool dominates_backedge(Node* ctrl);
// Remove simplistic dead code from loop body
void DCE_loop_body();
// Look for loop-exit tests with my 50/50 guesses from the Parsing stage.
// Replace with a 1-in-10 exit guess.
void adjust_loop_exit_prob( PhaseIdealLoop *phase );
// Return TRUE or FALSE if the loop should never be RCE'd or aligned.
// Useful for unrolling loops with NO array accesses.
bool policy_peel_only( PhaseIdealLoop *phase ) const;
// Return TRUE or FALSE if the loop should be unswitched -- clone
// loop with an invariant test
bool policy_unswitching( PhaseIdealLoop *phase ) const;
// Micro-benchmark spamming. Remove empty loops.
bool policy_do_remove_empty_loop( PhaseIdealLoop *phase );
// Convert one iteration loop into normal code.
bool policy_do_one_iteration_loop( PhaseIdealLoop *phase );
// Return TRUE or FALSE if the loop should be peeled or not. Peel if we can
// make some loop-invariant test (usually a null-check) happen before the
// loop.
bool policy_peeling( PhaseIdealLoop *phase ) const;
// Return TRUE or FALSE if the loop should be maximally unrolled. Stash any
// known trip count in the counted loop node.
bool policy_maximally_unroll( PhaseIdealLoop *phase ) const;
// Return TRUE or FALSE if the loop should be unrolled or not. Unroll if
// the loop is a CountedLoop and the body is small enough.
bool policy_unroll( PhaseIdealLoop *phase ) const;
// Return TRUE or FALSE if the loop should be range-check-eliminated.
// Gather a list of IF tests that are dominated by iteration splitting;
// also gather the end of the first split and the start of the 2nd split.
bool policy_range_check( PhaseIdealLoop *phase ) const;
// Return TRUE or FALSE if the loop should be cache-line aligned.
// Gather the expression that does the alignment. Note that only
// one array base can be aligned in a loop (unless the VM guarantees
// mutual alignment). Note that if we vectorize short memory ops
// into longer memory ops, we may want to increase alignment.
bool policy_align( PhaseIdealLoop *phase ) const;
// Return TRUE if "iff" is a range check.
bool is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar) const;
// Compute loop exact trip count if possible
void compute_exact_trip_count( PhaseIdealLoop *phase );
// Compute loop trip count from profile data
void compute_profile_trip_cnt( PhaseIdealLoop *phase );
// Reassociate invariant expressions.
void reassociate_invariants(PhaseIdealLoop *phase);
// Reassociate invariant add and subtract expressions.
Node* reassociate_add_sub(Node* n1, PhaseIdealLoop *phase);
// Return nonzero index of invariant operand if invariant and variant
// are combined with an Add or Sub. Helper for reassociate_invariants.
int is_invariant_addition(Node* n, PhaseIdealLoop *phase);
// Return true if n is invariant
bool is_invariant(Node* n) const;
// Put loop body on igvn work list
void record_for_igvn();
bool is_loop() { return !_irreducible && _tail && !_tail->is_top(); }
bool is_inner() { return is_loop() && _child == NULL; }
bool is_counted() { return is_loop() && _head != NULL && _head->is_CountedLoop(); }
#ifndef PRODUCT
void dump_head( ) const; // Dump loop head only
void dump() const; // Dump this loop recursively
void verify_tree(IdealLoopTree *loop, const IdealLoopTree *parent) const;
#endif
};
// -----------------------------PhaseIdealLoop---------------------------------
// Computes the mapping from Nodes to IdealLoopTrees. Organizes IdealLoopTrees into a
// loop tree. Drives the loop-based transformations on the ideal graph.
class PhaseIdealLoop : public PhaseTransform {
friend class IdealLoopTree;
friend class SuperWord;
// Pre-computed def-use info
PhaseIterGVN &_igvn;
// Head of loop tree
IdealLoopTree *_ltree_root;
// Array of pre-order numbers, plus post-visited bit.
// ZERO for not pre-visited. EVEN for pre-visited but not post-visited.
// ODD for post-visited. Other bits are the pre-order number.
uint *_preorders;
uint _max_preorder;
const PhaseIdealLoop* _verify_me;
bool _verify_only;
// Allocate _preorders[] array
void allocate_preorders() {
_max_preorder = C->unique()+8;
_preorders = NEW_RESOURCE_ARRAY(uint, _max_preorder);
memset(_preorders, 0, sizeof(uint) * _max_preorder);
}
// Allocate _preorders[] array
void reallocate_preorders() {
if ( _max_preorder < C->unique() ) {
_preorders = REALLOC_RESOURCE_ARRAY(uint, _preorders, _max_preorder, C->unique());
_max_preorder = C->unique();
}
memset(_preorders, 0, sizeof(uint) * _max_preorder);
}
// Check to grow _preorders[] array for the case when build_loop_tree_impl()
// adds new nodes.
void check_grow_preorders( ) {
if ( _max_preorder < C->unique() ) {
uint newsize = _max_preorder<<1; // double size of array
_preorders = REALLOC_RESOURCE_ARRAY(uint, _preorders, _max_preorder, newsize);
memset(&_preorders[_max_preorder],0,sizeof(uint)*(newsize-_max_preorder));
_max_preorder = newsize;
}
}
// Check for pre-visited. Zero for NOT visited; non-zero for visited.
int is_visited( Node *n ) const { return _preorders[n->_idx]; }
// Pre-order numbers are written to the Nodes array as low-bit-set values.
void set_preorder_visited( Node *n, int pre_order ) {
assert( !is_visited( n ), "already set" );
_preorders[n->_idx] = (pre_order<<1);
};
// Return pre-order number.
int get_preorder( Node *n ) const { assert( is_visited(n), "" ); return _preorders[n->_idx]>>1; }
// Check for being post-visited.
// Should be previsited already (checked with assert(is_visited(n))).
int is_postvisited( Node *n ) const { assert( is_visited(n), "" ); return _preorders[n->_idx]&1; }
// Mark as post visited
void set_postvisited( Node *n ) { assert( !is_postvisited( n ), "" ); _preorders[n->_idx] |= 1; }
// Set/get control node out. Set lower bit to distinguish from IdealLoopTree
// Returns true if "n" is a data node, false if it's a control node.
bool has_ctrl( Node *n ) const { return ((intptr_t)_nodes[n->_idx]) & 1; }
// clear out dead code after build_loop_late
Node_List _deadlist;
// Support for faster execution of get_late_ctrl()/dom_lca()
// when a node has many uses and dominator depth is deep.
Node_Array _dom_lca_tags;
void init_dom_lca_tags();
void clear_dom_lca_tags();
// Helper for debugging bad dominance relationships
bool verify_dominance(Node* n, Node* use, Node* LCA, Node* early);
Node* compute_lca_of_uses(Node* n, Node* early, bool verify = false);
// Inline wrapper for frequent cases:
// 1) only one use
// 2) a use is the same as the current LCA passed as 'n1'
Node *dom_lca_for_get_late_ctrl( Node *lca, Node *n, Node *tag ) {
assert( n->is_CFG(), "" );
// Fast-path NULL lca
if( lca != NULL && lca != n ) {
assert( lca->is_CFG(), "" );
// find LCA of all uses
n = dom_lca_for_get_late_ctrl_internal( lca, n, tag );
}
return find_non_split_ctrl(n);
}
Node *dom_lca_for_get_late_ctrl_internal( Node *lca, Node *n, Node *tag );
// Helper function for directing control inputs away from CFG split
// points.
Node *find_non_split_ctrl( Node *ctrl ) const {
if (ctrl != NULL) {
if (ctrl->is_MultiBranch()) {
ctrl = ctrl->in(0);
}
assert(ctrl->is_CFG(), "CFG");
}
return ctrl;
}
bool cast_incr_before_loop(Node* incr, Node* ctrl, Node* loop);
public:
static bool is_canonical_main_loop_entry(CountedLoopNode* cl);
bool has_node( Node* n ) const {
guarantee(n != NULL, "No Node.");
return _nodes[n->_idx] != NULL;
}
// check if transform created new nodes that need _ctrl recorded
Node *get_late_ctrl( Node *n, Node *early );
Node *get_early_ctrl( Node *n );
Node *get_early_ctrl_for_expensive(Node *n, Node* earliest);
void set_early_ctrl( Node *n );
void set_subtree_ctrl( Node *root );
void set_ctrl( Node *n, Node *ctrl ) {
assert( !has_node(n) || has_ctrl(n), "" );
assert( ctrl->in(0), "cannot set dead control node" );
assert( ctrl == find_non_split_ctrl(ctrl), "must set legal crtl" );
_nodes.map( n->_idx, (Node*)((intptr_t)ctrl + 1) );
}
// Set control and update loop membership
void set_ctrl_and_loop(Node* n, Node* ctrl) {
IdealLoopTree* old_loop = get_loop(get_ctrl(n));
IdealLoopTree* new_loop = get_loop(ctrl);
if (old_loop != new_loop) {
if (old_loop->_child == NULL) old_loop->_body.yank(n);
if (new_loop->_child == NULL) new_loop->_body.push(n);
}
set_ctrl(n, ctrl);
}
// Control nodes can be replaced or subsumed. During this pass they
// get their replacement Node in slot 1. Instead of updating the block
// location of all Nodes in the subsumed block, we lazily do it. As we
// pull such a subsumed block out of the array, we write back the final
// correct block.
Node *get_ctrl( Node *i ) {
assert(has_node(i), "");
Node *n = get_ctrl_no_update(i);
_nodes.map( i->_idx, (Node*)((intptr_t)n + 1) );
assert(has_node(i) && has_ctrl(i), "");
assert(n == find_non_split_ctrl(n), "must return legal ctrl" );
return n;
}
// true if CFG node d dominates CFG node n
bool is_dominator(Node *d, Node *n);
// return get_ctrl for a data node and self(n) for a CFG node
Node* ctrl_or_self(Node* n) {
if (has_ctrl(n))
return get_ctrl(n);
else {
assert (n->is_CFG(), "must be a CFG node");
return n;
}
}
private:
Node *get_ctrl_no_update_helper(Node *i) const {
assert(has_ctrl(i), "should be control, not loop");
return (Node*)(((intptr_t)_nodes[i->_idx]) & ~1);
}
Node *get_ctrl_no_update(Node *i) const {
assert( has_ctrl(i), "" );
Node *n = get_ctrl_no_update_helper(i);
if (!n->in(0)) {
// Skip dead CFG nodes
do {
n = get_ctrl_no_update_helper(n);
} while (!n->in(0));
n = find_non_split_ctrl(n);
}
return n;
}
// Check for loop being set
// "n" must be a control node. Returns true if "n" is known to be in a loop.
bool has_loop( Node *n ) const {
assert(!has_node(n) || !has_ctrl(n), "");
return has_node(n);
}
// Set loop
void set_loop( Node *n, IdealLoopTree *loop ) {
_nodes.map(n->_idx, (Node*)loop);
}
// Lazy-dazy update of 'get_ctrl' and 'idom_at' mechanisms. Replace
// the 'old_node' with 'new_node'. Kill old-node. Add a reference
// from old_node to new_node to support the lazy update. Reference
// replaces loop reference, since that is not needed for dead node.
public:
void lazy_update(Node *old_node, Node *new_node) {
assert(old_node != new_node, "no cycles please");
// Re-use the side array slot for this node to provide the
// forwarding pointer.
_nodes.map(old_node->_idx, (Node*)((intptr_t)new_node + 1));
}
void lazy_replace(Node *old_node, Node *new_node) {
_igvn.replace_node(old_node, new_node);
lazy_update(old_node, new_node);
}
private:
// Place 'n' in some loop nest, where 'n' is a CFG node
void build_loop_tree();
int build_loop_tree_impl( Node *n, int pre_order );
// Insert loop into the existing loop tree. 'innermost' is a leaf of the
// loop tree, not the root.
IdealLoopTree *sort( IdealLoopTree *loop, IdealLoopTree *innermost );
// Place Data nodes in some loop nest
void build_loop_early( VectorSet &visited, Node_List &worklist, Node_Stack &nstack );
void build_loop_late ( VectorSet &visited, Node_List &worklist, Node_Stack &nstack );
void build_loop_late_post ( Node* n );
// Array of immediate dominance info for each CFG node indexed by node idx
private:
uint _idom_size;
Node **_idom; // Array of immediate dominators
uint *_dom_depth; // Used for fast LCA test
GrowableArray<uint>* _dom_stk; // For recomputation of dom depth
Node* idom_no_update(Node* d) const {
assert(d->_idx < _idom_size, "oob");
Node* n = _idom[d->_idx];
assert(n != NULL,"Bad immediate dominator info.");
while (n->in(0) == NULL) { // Skip dead CFG nodes
//n = n->in(1);
n = (Node*)(((intptr_t)_nodes[n->_idx]) & ~1);
assert(n != NULL,"Bad immediate dominator info.");
}
return n;
}
Node *idom(Node* d) const {
uint didx = d->_idx;
Node *n = idom_no_update(d);
_idom[didx] = n; // Lazily remove dead CFG nodes from table.
return n;
}
uint dom_depth(Node* d) const {
guarantee(d != NULL, "Null dominator info.");
guarantee(d->_idx < _idom_size, "");
return _dom_depth[d->_idx];
}
void set_idom(Node* d, Node* n, uint dom_depth);
// Locally compute IDOM using dom_lca call
Node *compute_idom( Node *region ) const;
// Recompute dom_depth
void recompute_dom_depth();
// Is safept not required by an outer loop?
bool is_deleteable_safept(Node* sfpt);
// Replace parallel induction variable (parallel to trip counter)
void replace_parallel_iv(IdealLoopTree *loop);
// Perform verification that the graph is valid.
PhaseIdealLoop( PhaseIterGVN &igvn) :
PhaseTransform(Ideal_Loop),
_igvn(igvn),
_dom_lca_tags(arena()), // Thread::resource_area
_verify_me(NULL),
_verify_only(true) {
build_and_optimize(false, false);
}
// build the loop tree and perform any requested optimizations
void build_and_optimize(bool do_split_if, bool skip_loop_opts);
public:
// Dominators for the sea of nodes
void Dominators();
Node *dom_lca( Node *n1, Node *n2 ) const {
return find_non_split_ctrl(dom_lca_internal(n1, n2));
}
Node *dom_lca_internal( Node *n1, Node *n2 ) const;
// Compute the Ideal Node to Loop mapping
PhaseIdealLoop( PhaseIterGVN &igvn, bool do_split_ifs, bool skip_loop_opts = false) :
PhaseTransform(Ideal_Loop),
_igvn(igvn),
_dom_lca_tags(arena()), // Thread::resource_area
_verify_me(NULL),
_verify_only(false) {
build_and_optimize(do_split_ifs, skip_loop_opts);
}
// Verify that verify_me made the same decisions as a fresh run.
PhaseIdealLoop( PhaseIterGVN &igvn, const PhaseIdealLoop *verify_me) :
PhaseTransform(Ideal_Loop),
_igvn(igvn),
_dom_lca_tags(arena()), // Thread::resource_area
_verify_me(verify_me),
_verify_only(false) {
build_and_optimize(false, false);
}
// Build and verify the loop tree without modifying the graph. This
// is useful to verify that all inputs properly dominate their uses.
static void verify(PhaseIterGVN& igvn) {
#ifdef ASSERT
PhaseIdealLoop v(igvn);
#endif
}
// True if the method has at least 1 irreducible loop
bool _has_irreducible_loops;
// Per-Node transform
virtual Node *transform( Node *a_node ) { return 0; }
bool is_counted_loop( Node *x, IdealLoopTree *loop );
Node* exact_limit( IdealLoopTree *loop );
// Return a post-walked LoopNode
IdealLoopTree *get_loop( Node *n ) const {
// Dead nodes have no loop, so return the top level loop instead
if (!has_node(n)) return _ltree_root;
assert(!has_ctrl(n), "");
return (IdealLoopTree*)_nodes[n->_idx];
}
// Is 'n' a (nested) member of 'loop'?
int is_member( const IdealLoopTree *loop, Node *n ) const {
return loop->is_member(get_loop(n)); }
// This is the basic building block of the loop optimizations. It clones an
// entire loop body. It makes an old_new loop body mapping; with this
// mapping you can find the new-loop equivalent to an old-loop node. All
// new-loop nodes are exactly equal to their old-loop counterparts, all
// edges are the same. All exits from the old-loop now have a RegionNode
// that merges the equivalent new-loop path. This is true even for the
// normal "loop-exit" condition. All uses of loop-invariant old-loop values
// now come from (one or more) Phis that merge their new-loop equivalents.
// Parameter side_by_side_idom:
// When side_by_size_idom is NULL, the dominator tree is constructed for
// the clone loop to dominate the original. Used in construction of
// pre-main-post loop sequence.
// When nonnull, the clone and original are side-by-side, both are
// dominated by the passed in side_by_side_idom node. Used in
// construction of unswitched loops.
void clone_loop( IdealLoopTree *loop, Node_List &old_new, int dom_depth,
Node* side_by_side_idom = NULL);
// If we got the effect of peeling, either by actually peeling or by
// making a pre-loop which must execute at least once, we can remove
// all loop-invariant dominated tests in the main body.
void peeled_dom_test_elim( IdealLoopTree *loop, Node_List &old_new );
// Generate code to do a loop peel for the given loop (and body).
// old_new is a temp array.
void do_peeling( IdealLoopTree *loop, Node_List &old_new );
// Add pre and post loops around the given loop. These loops are used
// during RCE, unrolling and aligning loops.
void insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_new, bool peel_only );
// If Node n lives in the back_ctrl block, we clone a private version of n
// in preheader_ctrl block and return that, otherwise return n.
Node *clone_up_backedge_goo( Node *back_ctrl, Node *preheader_ctrl, Node *n, VectorSet &visited, Node_Stack &clones );
// Take steps to maximally unroll the loop. Peel any odd iterations, then
// unroll to do double iterations. The next round of major loop transforms
// will repeat till the doubled loop body does all remaining iterations in 1
// pass.
void do_maximally_unroll( IdealLoopTree *loop, Node_List &old_new );
// Unroll the loop body one step - make each trip do 2 iterations.
void do_unroll( IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip );
// Return true if exp is a constant times an induction var
bool is_scaled_iv(Node* exp, Node* iv, int* p_scale);
// Return true if exp is a scaled induction var plus (or minus) constant
bool is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset, int depth = 0);
// Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted
ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
Deoptimization::DeoptReason reason);
void register_control(Node* n, IdealLoopTree *loop, Node* pred);
// Clone loop predicates to cloned loops (peeled, unswitched)
static ProjNode* clone_predicate(ProjNode* predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason,
PhaseIdealLoop* loop_phase,
PhaseIterGVN* igvn);
static Node* clone_loop_predicates(Node* old_entry, Node* new_entry,
bool clone_limit_check,
PhaseIdealLoop* loop_phase,
PhaseIterGVN* igvn);
Node* clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check);
static Node* skip_loop_predicates(Node* entry);
// Find a good location to insert a predicate
static ProjNode* find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason);
// Find a predicate
static Node* find_predicate(Node* entry);
// Construct a range check for a predicate if
BoolNode* rc_predicate(IdealLoopTree *loop, Node* ctrl,
int scale, Node* offset,
Node* init, Node* limit, jint stride,
Node* range, bool upper, bool &overflow);
// Implementation of the loop predication to promote checks outside the loop
bool loop_predication_impl(IdealLoopTree *loop);
// Helper function to collect predicate for eliminating the useless ones
void collect_potentially_useful_predicates(IdealLoopTree *loop, Unique_Node_List &predicate_opaque1);
void eliminate_useless_predicates();
// Change the control input of expensive nodes to allow commoning by
// IGVN when it is guaranteed to not result in a more frequent
// execution of the expensive node. Return true if progress.
bool process_expensive_nodes();
// Check whether node has become unreachable
bool is_node_unreachable(Node *n) const {
return !has_node(n) || n->is_unreachable(_igvn);
}
// Eliminate range-checks and other trip-counter vs loop-invariant tests.
void do_range_check( IdealLoopTree *loop, Node_List &old_new );
// Create a slow version of the loop by cloning the loop
// and inserting an if to select fast-slow versions.
ProjNode* create_slow_version_of_loop(IdealLoopTree *loop,
Node_List &old_new);
// Clone loop with an invariant test (that does not exit) and
// insert a clone of the test that selects which version to
// execute.
void do_unswitching (IdealLoopTree *loop, Node_List &old_new);
// Find candidate "if" for unswitching
IfNode* find_unswitching_candidate(const IdealLoopTree *loop) const;
// Range Check Elimination uses this function!
// Constrain the main loop iterations so the affine function:
// low_limit <= scale_con * I + offset < upper_limit
// always holds true. That is, either increase the number of iterations in
// the pre-loop or the post-loop until the condition holds true in the main
// loop. Scale_con, offset and limit are all loop invariant.
void add_constraint(jlong stride_con, jlong scale_con, Node* offset, Node* low_limit, Node* upper_limit, Node* pre_ctrl, Node** pre_limit, Node** main_limit);
// Helper function for add_constraint().
Node* adjust_limit(bool reduce, Node* scale, Node* offset, Node* rc_limit, Node* old_limit, Node* pre_ctrl, bool round);
// Partially peel loop up through last_peel node.
bool partial_peel( IdealLoopTree *loop, Node_List &old_new );
// Create a scheduled list of nodes control dependent on ctrl set.
void scheduled_nodelist( IdealLoopTree *loop, VectorSet& ctrl, Node_List &sched );
// Has a use in the vector set
bool has_use_in_set( Node* n, VectorSet& vset );
// Has use internal to the vector set (ie. not in a phi at the loop head)
bool has_use_internal_to_set( Node* n, VectorSet& vset, IdealLoopTree *loop );
// clone "n" for uses that are outside of loop
int clone_for_use_outside_loop( IdealLoopTree *loop, Node* n, Node_List& worklist );
// clone "n" for special uses that are in the not_peeled region
void clone_for_special_use_inside_loop( IdealLoopTree *loop, Node* n,
VectorSet& not_peel, Node_List& sink_list, Node_List& worklist );
// Insert phi(lp_entry_val, back_edge_val) at use->in(idx) for loop lp if phi does not already exist
void insert_phi_for_loop( Node* use, uint idx, Node* lp_entry_val, Node* back_edge_val, LoopNode* lp );
#ifdef ASSERT
// Validate the loop partition sets: peel and not_peel
bool is_valid_loop_partition( IdealLoopTree *loop, VectorSet& peel, Node_List& peel_list, VectorSet& not_peel );
// Ensure that uses outside of loop are of the right form
bool is_valid_clone_loop_form( IdealLoopTree *loop, Node_List& peel_list,
uint orig_exit_idx, uint clone_exit_idx);
bool is_valid_clone_loop_exit_use( IdealLoopTree *loop, Node* use, uint exit_idx);
#endif
// Returns nonzero constant stride if-node is a possible iv test (otherwise returns zero.)
int stride_of_possible_iv( Node* iff );
bool is_possible_iv_test( Node* iff ) { return stride_of_possible_iv(iff) != 0; }
// Return the (unique) control output node that's in the loop (if it exists.)
Node* stay_in_loop( Node* n, IdealLoopTree *loop);
// Insert a signed compare loop exit cloned from an unsigned compare.
IfNode* insert_cmpi_loop_exit(IfNode* if_cmpu, IdealLoopTree *loop);
void remove_cmpi_loop_exit(IfNode* if_cmp, IdealLoopTree *loop);
// Utility to register node "n" with PhaseIdealLoop
void register_node(Node* n, IdealLoopTree *loop, Node* pred, int ddepth);
// Utility to create an if-projection
ProjNode* proj_clone(ProjNode* p, IfNode* iff);
// Force the iff control output to be the live_proj
Node* short_circuit_if(IfNode* iff, ProjNode* live_proj);
// Insert a region before an if projection
RegionNode* insert_region_before_proj(ProjNode* proj);
// Insert a new if before an if projection
ProjNode* insert_if_before_proj(Node* left, bool Signed, BoolTest::mask relop, Node* right, ProjNode* proj);
// Passed in a Phi merging (recursively) some nearly equivalent Bool/Cmps.
// "Nearly" because all Nodes have been cloned from the original in the loop,
// but the fall-in edges to the Cmp are different. Clone bool/Cmp pairs
// through the Phi recursively, and return a Bool.
BoolNode *clone_iff( PhiNode *phi, IdealLoopTree *loop );
CmpNode *clone_bool( PhiNode *phi, IdealLoopTree *loop );
// Rework addressing expressions to get the most loop-invariant stuff
// moved out. We'd like to do all associative operators, but it's especially
// important (common) to do address expressions.
Node *remix_address_expressions( Node *n );
// Attempt to use a conditional move instead of a phi/branch
Node *conditional_move( Node *n );
// Reorganize offset computations to lower register pressure.
// Mostly prevent loop-fallout uses of the pre-incremented trip counter
// (which are then alive with the post-incremented trip counter
// forcing an extra register move)
void reorg_offsets( IdealLoopTree *loop );
// Check for aggressive application of 'split-if' optimization,
// using basic block level info.
void split_if_with_blocks ( VectorSet &visited, Node_Stack &nstack );
Node *split_if_with_blocks_pre ( Node *n );
void split_if_with_blocks_post( Node *n );
Node *has_local_phi_input( Node *n );
// Mark an IfNode as being dominated by a prior test,
// without actually altering the CFG (and hence IDOM info).
void dominated_by( Node *prevdom, Node *iff, bool flip = false, bool exclude_loop_predicate = false );
// Split Node 'n' through merge point
Node *split_thru_region( Node *n, Node *region );
// Split Node 'n' through merge point if there is enough win.
Node *split_thru_phi( Node *n, Node *region, int policy );
// Found an If getting its condition-code input from a Phi in the
// same block. Split thru the Region.
void do_split_if( Node *iff );
// Conversion of fill/copy patterns into intrisic versions
bool do_intrinsify_fill();
bool intrinsify_fill(IdealLoopTree* lpt);
bool match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& store_value,
Node*& shift, Node*& offset);
private:
// Return a type based on condition control flow
const TypeInt* filtered_type( Node *n, Node* n_ctrl);
const TypeInt* filtered_type( Node *n ) { return filtered_type(n, NULL); }
// Helpers for filtered type
const TypeInt* filtered_type_from_dominators( Node* val, Node *val_ctrl);
// Helper functions
Node *spinup( Node *iff, Node *new_false, Node *new_true, Node *region, Node *phi, small_cache *cache );
Node *find_use_block( Node *use, Node *def, Node *old_false, Node *new_false, Node *old_true, Node *new_true );
void handle_use( Node *use, Node *def, small_cache *cache, Node *region_dom, Node *new_false, Node *new_true, Node *old_false, Node *old_true );
bool split_up( Node *n, Node *blk1, Node *blk2 );
void sink_use( Node *use, Node *post_loop );
Node *place_near_use( Node *useblock ) const;
bool _created_loop_node;
public:
void set_created_loop_node() { _created_loop_node = true; }
bool created_loop_node() { return _created_loop_node; }
void register_new_node( Node *n, Node *blk );
#ifdef ASSERT
void dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA);
#endif
#ifndef PRODUCT
void dump( ) const;
void dump( IdealLoopTree *loop, uint rpo_idx, Node_List &rpo_list ) const;
void rpo( Node *start, Node_Stack &stk, VectorSet &visited, Node_List &rpo_list ) const;
void verify() const; // Major slow :-)
void verify_compare( Node *n, const PhaseIdealLoop *loop_verify, VectorSet &visited ) const;
IdealLoopTree *get_loop_idx(Node* n) const {
// Dead nodes have no loop, so return the top level loop instead
return _nodes[n->_idx] ? (IdealLoopTree*)_nodes[n->_idx] : _ltree_root;
}
// Print some stats
static void print_statistics();
static int _loop_invokes; // Count of PhaseIdealLoop invokes
static int _loop_work; // Sum of PhaseIdealLoop x _unique
#endif
};
inline Node* IdealLoopTree::tail() {
// Handle lazy update of _tail field
Node *n = _tail;
//while( !n->in(0) ) // Skip dead CFG nodes
//n = n->in(1);
if (n->in(0) == NULL)
n = _phase->get_ctrl(n);
_tail = n;
return n;
}
// Iterate over the loop tree using a preorder, left-to-right traversal.
//
// Example that visits all counted loops from within PhaseIdealLoop
//
// for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
// IdealLoopTree* lpt = iter.current();
// if (!lpt->is_counted()) continue;
// ...
class LoopTreeIterator : public StackObj {
private:
IdealLoopTree* _root;
IdealLoopTree* _curnt;
public:
LoopTreeIterator(IdealLoopTree* root) : _root(root), _curnt(root) {}
bool done() { return _curnt == NULL; } // Finished iterating?
void next(); // Advance to next loop tree
IdealLoopTree* current() { return _curnt; } // Return current value of iterator.
};
#endif // SHARE_VM_OPTO_LOOPNODE_HPP
C:\hotspot-69087d08d473\src\share\vm/opto/loopopts.cpp
/*
* Copyright (c) 1999, 2017, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "memory/allocation.inline.hpp"
#include "opto/addnode.hpp"
#include "opto/connode.hpp"
#include "opto/divnode.hpp"
#include "opto/loopnode.hpp"
#include "opto/matcher.hpp"
#include "opto/mulnode.hpp"
#include "opto/rootnode.hpp"
#include "opto/subnode.hpp"
//=============================================================================
//------------------------------split_thru_phi---------------------------------
// Split Node 'n' through merge point if there is enough win.
Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {
if (n->Opcode() == Op_ConvI2L && n->bottom_type() != TypeLong::LONG) {
// ConvI2L may have type information on it which is unsafe to push up
// so disable this for now
return NULL;
}
// Splitting range check CastIIs through a loop induction Phi can
// cause new Phis to be created that are left unrelated to the loop
// induction Phi and prevent optimizations (vectorization)
if (n->Opcode() == Op_CastII && n->as_CastII()->has_range_check() &&
region->is_CountedLoop() && n->in(1) == region->as_CountedLoop()->phi()) {
return NULL;
}
int wins = 0;
assert(!n->is_CFG(), "");
assert(region->is_Region(), "");
const Type* type = n->bottom_type();
const TypeOopPtr *t_oop = _igvn.type(n)->isa_oopptr();
Node *phi;
if (t_oop != NULL && t_oop->is_known_instance_field()) {
int iid = t_oop->instance_id();
int index = C->get_alias_index(t_oop);
int offset = t_oop->offset();
phi = new (C) PhiNode(region, type, NULL, iid, index, offset);
} else {
phi = PhiNode::make_blank(region, n);
}
uint old_unique = C->unique();
for (uint i = 1; i < region->req(); i++) {
Node *x;
Node* the_clone = NULL;
if (region->in(i) == C->top()) {
x = C->top(); // Dead path? Use a dead data op
} else {
x = n->clone(); // Else clone up the data op
the_clone = x; // Remember for possible deletion.
// Alter data node to use pre-phi inputs
if (n->in(0) == region)
x->set_req( 0, region->in(i) );
for (uint j = 1; j < n->req(); j++) {
Node *in = n->in(j);
if (in->is_Phi() && in->in(0) == region)
x->set_req( j, in->in(i) ); // Use pre-Phi input for the clone
}
}
// Check for a 'win' on some paths
const Type *t = x->Value(&_igvn);
bool singleton = t->singleton();
// A TOP singleton indicates that there are no possible values incoming
// along a particular edge. In most cases, this is OK, and the Phi will
// be eliminated later in an Ideal call. However, we can't allow this to
// happen if the singleton occurs on loop entry, as the elimination of
// the PhiNode may cause the resulting node to migrate back to a previous
// loop iteration.
if (singleton && t == Type::TOP) {
// Is_Loop() == false does not confirm the absence of a loop (e.g., an
// irreducible loop may not be indicated by an affirmative is_Loop());
// therefore, the only top we can split thru a phi is on a backedge of
// a loop.
singleton &= region->is_Loop() && (i != LoopNode::EntryControl);
}
if (singleton) {
wins++;
x = ((PhaseGVN&)_igvn).makecon(t);
} else {
// We now call Identity to try to simplify the cloned node.
// Note that some Identity methods call phase->type(this).
// Make sure that the type array is big enough for
// our new node, even though we may throw the node away.
// (Note: This tweaking with igvn only works because x is a new node.)
_igvn.set_type(x, t);
// If x is a TypeNode, capture any more-precise type permanently into Node
// otherwise it will be not updated during igvn->transform since
// igvn->type(x) is set to x->Value() already.
x->raise_bottom_type(t);
Node *y = x->Identity(&_igvn);
if (y != x) {
wins++;
x = y;
} else {
y = _igvn.hash_find(x);
if (y) {
wins++;
x = y;
} else {
// Else x is a new node we are keeping
// We do not need register_new_node_with_optimizer
// because set_type has already been called.
_igvn._worklist.push(x);
}
}
}
if (x != the_clone && the_clone != NULL)
_igvn.remove_dead_node(the_clone);
phi->set_req( i, x );
}
// Too few wins?
if (wins <= policy) {
_igvn.remove_dead_node(phi);
return NULL;
}
// Record Phi
register_new_node( phi, region );
for (uint i2 = 1; i2 < phi->req(); i2++) {
Node *x = phi->in(i2);
// If we commoned up the cloned 'x' with another existing Node,
// the existing Node picks up a new use. We need to make the
// existing Node occur higher up so it dominates its uses.
Node *old_ctrl;
IdealLoopTree *old_loop;
if (x->is_Con()) {
// Constant's control is always root.
set_ctrl(x, C->root());
continue;
}
// The occasional new node
if (x->_idx >= old_unique) { // Found a new, unplaced node?
old_ctrl = NULL;
old_loop = NULL; // Not in any prior loop
} else {
old_ctrl = get_ctrl(x);
old_loop = get_loop(old_ctrl); // Get prior loop
}
// New late point must dominate new use
Node *new_ctrl = dom_lca(old_ctrl, region->in(i2));
if (new_ctrl == old_ctrl) // Nothing is changed
continue;
IdealLoopTree *new_loop = get_loop(new_ctrl);
// Don't move x into a loop if its uses are
// outside of loop. Otherwise x will be cloned
// for each use outside of this loop.
IdealLoopTree *use_loop = get_loop(region);
if (!new_loop->is_member(use_loop) &&
(old_loop == NULL || !new_loop->is_member(old_loop))) {
// Take early control, later control will be recalculated
// during next iteration of loop optimizations.
new_ctrl = get_early_ctrl(x);
new_loop = get_loop(new_ctrl);
}
// Set new location
set_ctrl(x, new_ctrl);
// If changing loop bodies, see if we need to collect into new body
if (old_loop != new_loop) {
if (old_loop && !old_loop->_child)
old_loop->_body.yank(x);
if (!new_loop->_child)
new_loop->_body.push(x); // Collect body info
}
}
return phi;
}
//------------------------------dominated_by------------------------------------
// Replace the dominated test with an obvious true or false. Place it on the
// IGVN worklist for later cleanup. Move control-dependent data Nodes on the
// live path up to the dominating control.
void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff, bool flip, bool exclude_loop_predicate ) {
#ifndef PRODUCT
if (VerifyLoopOptimizations && PrintOpto) tty->print_cr("dominating test");
#endif
// prevdom is the dominating projection of the dominating test.
assert( iff->is_If(), "" );
assert( iff->Opcode() == Op_If || iff->Opcode() == Op_CountedLoopEnd, "Check this code when new subtype is added");
int pop = prevdom->Opcode();
assert( pop == Op_IfFalse || pop == Op_IfTrue, "" );
if (flip) {
if (pop == Op_IfTrue)
pop = Op_IfFalse;
else
pop = Op_IfTrue;
}
// 'con' is set to true or false to kill the dominated test.
Node *con = _igvn.makecon(pop == Op_IfTrue ? TypeInt::ONE : TypeInt::ZERO);
set_ctrl(con, C->root()); // Constant gets a new use
// Hack the dominated test
_igvn.replace_input_of(iff, 1, con);
// If I dont have a reachable TRUE and FALSE path following the IfNode then
// I can assume this path reaches an infinite loop. In this case it's not
// important to optimize the data Nodes - either the whole compilation will
// be tossed or this path (and all data Nodes) will go dead.
if (iff->outcnt() != 2) return;
// Make control-dependent data Nodes on the live path (path that will remain
// once the dominated IF is removed) become control-dependent on the
// dominating projection.
Node* dp = iff->as_If()->proj_out(pop == Op_IfTrue);
// Loop predicates may have depending checks which should not
// be skipped. For example, range check predicate has two checks
// for lower and upper bounds.
if (dp == NULL)
return;
ProjNode* dp_proj = dp->as_Proj();
ProjNode* unc_proj = iff->as_If()->proj_out(1 - dp_proj->_con)->as_Proj();
if (exclude_loop_predicate &&
(unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_predicate) ||
unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_range_check))) {
// If this is a range check (IfNode::is_range_check), do not
// reorder because Compile::allow_range_check_smearing might have
// changed the check.
return; // Let IGVN transformation change control dependence.
}
IdealLoopTree *old_loop = get_loop(dp);
for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) {
Node* cd = dp->fast_out(i); // Control-dependent node
if (cd->depends_only_on_test()) {
assert(cd->in(0) == dp, "");
_igvn.replace_input_of(cd, 0, prevdom);
set_early_ctrl(cd);
IdealLoopTree *new_loop = get_loop(get_ctrl(cd));
if (old_loop != new_loop) {
if (!old_loop->_child) old_loop->_body.yank(cd);
if (!new_loop->_child) new_loop->_body.push(cd);
}
--i;
--imax;
}
}
}
//------------------------------has_local_phi_input----------------------------
// Return TRUE if 'n' has Phi inputs from its local block and no other
// block-local inputs (all non-local-phi inputs come from earlier blocks)
Node *PhaseIdealLoop::has_local_phi_input( Node *n ) {
Node *n_ctrl = get_ctrl(n);
// See if some inputs come from a Phi in this block, or from before
// this block.
uint i;
for( i = 1; i < n->req(); i++ ) {
Node *phi = n->in(i);
if( phi->is_Phi() && phi->in(0) == n_ctrl )
break;
}
if( i >= n->req() )
return NULL; // No Phi inputs; nowhere to clone thru
// Check for inputs created between 'n' and the Phi input. These
// must split as well; they have already been given the chance
// (courtesy of a post-order visit) and since they did not we must
// recover the 'cost' of splitting them by being very profitable
// when splitting 'n'. Since this is unlikely we simply give up.
for( i = 1; i < n->req(); i++ ) {
Node *m = n->in(i);
if( get_ctrl(m) == n_ctrl && !m->is_Phi() ) {
// We allow the special case of AddP's with no local inputs.
// This allows us to split-up address expressions.
if (m->is_AddP() &&
get_ctrl(m->in(2)) != n_ctrl &&
get_ctrl(m->in(3)) != n_ctrl) {
// Move the AddP up to dominating point
set_ctrl_and_loop(m, find_non_split_ctrl(idom(n_ctrl)));
continue;
}
return NULL;
}
assert(n->is_Phi() || m->is_Phi() || is_dominator(get_ctrl(m), n_ctrl), "m has strange control");
}
return n_ctrl;
}
//------------------------------remix_address_expressions----------------------
// Rework addressing expressions to get the most loop-invariant stuff
// moved out. We'd like to do all associative operators, but it's especially
// important (common) to do address expressions.
Node *PhaseIdealLoop::remix_address_expressions( Node *n ) {
if (!has_ctrl(n)) return NULL;
Node *n_ctrl = get_ctrl(n);
IdealLoopTree *n_loop = get_loop(n_ctrl);
// See if 'n' mixes loop-varying and loop-invariant inputs and
// itself is loop-varying.
// Only interested in binary ops (and AddP)
if( n->req() < 3 || n->req() > 4 ) return NULL;
Node *n1_ctrl = get_ctrl(n->in( 1));
Node *n2_ctrl = get_ctrl(n->in( 2));
Node *n3_ctrl = get_ctrl(n->in(n->req() == 3 ? 2 : 3));
IdealLoopTree *n1_loop = get_loop( n1_ctrl );
IdealLoopTree *n2_loop = get_loop( n2_ctrl );
IdealLoopTree *n3_loop = get_loop( n3_ctrl );
// Does one of my inputs spin in a tighter loop than self?
if( (n_loop->is_member( n1_loop ) && n_loop != n1_loop) ||
(n_loop->is_member( n2_loop ) && n_loop != n2_loop) ||
(n_loop->is_member( n3_loop ) && n_loop != n3_loop) )
return NULL; // Leave well enough alone
// Is at least one of my inputs loop-invariant?
if( n1_loop == n_loop &&
n2_loop == n_loop &&
n3_loop == n_loop )
return NULL; // No loop-invariant inputs
int n_op = n->Opcode();
// Replace expressions like ((V+I) << 2) with (V<<2 + I<<2).
if( n_op == Op_LShiftI ) {
// Scale is loop invariant
Node *scale = n->in(2);
Node *scale_ctrl = get_ctrl(scale);
IdealLoopTree *scale_loop = get_loop(scale_ctrl );
if( n_loop == scale_loop || !scale_loop->is_member( n_loop ) )
return NULL;
const TypeInt *scale_t = scale->bottom_type()->isa_int();
if( scale_t && scale_t->is_con() && scale_t->get_con() >= 16 )
return NULL; // Dont bother with byte/short masking
// Add must vary with loop (else shift would be loop-invariant)
Node *add = n->in(1);
Node *add_ctrl = get_ctrl(add);
IdealLoopTree *add_loop = get_loop(add_ctrl);
//assert( n_loop == add_loop, "" );
if( n_loop != add_loop ) return NULL; // happens w/ evil ZKM loops
// Convert I-V into I+ (0-V); same for V-I
if( add->Opcode() == Op_SubI &&
_igvn.type( add->in(1) ) != TypeInt::ZERO ) {
Node *zero = _igvn.intcon(0);
set_ctrl(zero, C->root());
Node *neg = new (C) SubINode( _igvn.intcon(0), add->in(2) );
register_new_node( neg, get_ctrl(add->in(2) ) );
add = new (C) AddINode( add->in(1), neg );
register_new_node( add, add_ctrl );
}
if( add->Opcode() != Op_AddI ) return NULL;
// See if one add input is loop invariant
Node *add_var = add->in(1);
Node *add_var_ctrl = get_ctrl(add_var);
IdealLoopTree *add_var_loop = get_loop(add_var_ctrl );
Node *add_invar = add->in(2);
Node *add_invar_ctrl = get_ctrl(add_invar);
IdealLoopTree *add_invar_loop = get_loop(add_invar_ctrl );
if( add_var_loop == n_loop ) {
} else if( add_invar_loop == n_loop ) {
// Swap to find the invariant part
add_invar = add_var;
add_invar_ctrl = add_var_ctrl;
add_invar_loop = add_var_loop;
add_var = add->in(2);
Node *add_var_ctrl = get_ctrl(add_var);
IdealLoopTree *add_var_loop = get_loop(add_var_ctrl );
} else // Else neither input is loop invariant
return NULL;
if( n_loop == add_invar_loop || !add_invar_loop->is_member( n_loop ) )
return NULL; // No invariant part of the add?
// Yes! Reshape address expression!
Node *inv_scale = new (C) LShiftINode( add_invar, scale );
Node *inv_scale_ctrl =
dom_depth(add_invar_ctrl) > dom_depth(scale_ctrl) ?
add_invar_ctrl : scale_ctrl;
register_new_node( inv_scale, inv_scale_ctrl );
Node *var_scale = new (C) LShiftINode( add_var, scale );
register_new_node( var_scale, n_ctrl );
Node *var_add = new (C) AddINode( var_scale, inv_scale );
register_new_node( var_add, n_ctrl );
_igvn.replace_node( n, var_add );
return var_add;
}
// Replace (I+V) with (V+I)
if( n_op == Op_AddI ||
n_op == Op_AddL ||
n_op == Op_AddF ||
n_op == Op_AddD ||
n_op == Op_MulI ||
n_op == Op_MulL ||
n_op == Op_MulF ||
n_op == Op_MulD ) {
if( n2_loop == n_loop ) {
assert( n1_loop != n_loop, "" );
n->swap_edges(1, 2);
}
}
// Replace ((I1 +p V) +p I2) with ((I1 +p I2) +p V),
// but not if I2 is a constant.
if( n_op == Op_AddP ) {
if( n2_loop == n_loop && n3_loop != n_loop ) {
if( n->in(2)->Opcode() == Op_AddP && !n->in(3)->is_Con() ) {
Node *n22_ctrl = get_ctrl(n->in(2)->in(2));
Node *n23_ctrl = get_ctrl(n->in(2)->in(3));
IdealLoopTree *n22loop = get_loop( n22_ctrl );
IdealLoopTree *n23_loop = get_loop( n23_ctrl );
if( n22loop != n_loop && n22loop->is_member(n_loop) &&
n23_loop == n_loop ) {
Node *add1 = new (C) AddPNode( n->in(1), n->in(2)->in(2), n->in(3) );
// Stuff new AddP in the loop preheader
register_new_node( add1, n_loop->_head->in(LoopNode::EntryControl) );
Node *add2 = new (C) AddPNode( n->in(1), add1, n->in(2)->in(3) );
register_new_node( add2, n_ctrl );
_igvn.replace_node( n, add2 );
return add2;
}
}
}
// Replace (I1 +p (I2 + V)) with ((I1 +p I2) +p V)
if( n2_loop != n_loop && n3_loop == n_loop ) {
if( n->in(3)->Opcode() == Op_AddI ) {
Node *V = n->in(3)->in(1);
Node *I = n->in(3)->in(2);
if( is_member(n_loop,get_ctrl(V)) ) {
} else {
Node *tmp = V; V = I; I = tmp;
}
if( !is_member(n_loop,get_ctrl(I)) ) {
Node *add1 = new (C) AddPNode( n->in(1), n->in(2), I );
// Stuff new AddP in the loop preheader
register_new_node( add1, n_loop->_head->in(LoopNode::EntryControl) );
Node *add2 = new (C) AddPNode( n->in(1), add1, V );
register_new_node( add2, n_ctrl );
_igvn.replace_node( n, add2 );
return add2;
}
}
}
}
return NULL;
}
//------------------------------conditional_move-------------------------------
// Attempt to replace a Phi with a conditional move. We have some pretty
// strict profitability requirements. All Phis at the merge point must
// be converted, so we can remove the control flow. We need to limit the
// number of c-moves to a small handful. All code that was in the side-arms
// of the CFG diamond is now speculatively executed. This code has to be
// "cheap enough". We are pretty much limited to CFG diamonds that merge
// 1 or 2 items with a total of 1 or 2 ops executed speculatively.
Node *PhaseIdealLoop::conditional_move( Node *region ) {
assert(region->is_Region(), "sanity check");
if (region->req() != 3) return NULL;
// Check for CFG diamond
Node *lp = region->in(1);
Node *rp = region->in(2);
if (!lp || !rp) return NULL;
Node *lp_c = lp->in(0);
if (lp_c == NULL || lp_c != rp->in(0) || !lp_c->is_If()) return NULL;
IfNode *iff = lp_c->as_If();
// Check for ops pinned in an arm of the diamond.
// Can't remove the control flow in this case
if (lp->outcnt() > 1) return NULL;
if (rp->outcnt() > 1) return NULL;
IdealLoopTree* r_loop = get_loop(region);
assert(r_loop == get_loop(iff), "sanity");
// Always convert to CMOVE if all results are used only outside this loop.
bool used_inside_loop = (r_loop == _ltree_root);
// Check profitability
int cost = 0;
int phis = 0;
for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
Node *out = region->fast_out(i);
if (!out->is_Phi()) continue; // Ignore other control edges, etc
phis++;
PhiNode* phi = out->as_Phi();
BasicType bt = phi->type()->basic_type();
switch (bt) {
case T_FLOAT:
case T_DOUBLE: {
cost += Matcher::float_cmove_cost(); // Could be very expensive
break;
}
case T_LONG: {
cost += Matcher::long_cmove_cost(); // May encodes as 2 CMOV's
}
case T_INT: // These all CMOV fine
case T_ADDRESS: { // (RawPtr)
cost++;
break;
}
case T_NARROWOOP: // Fall through
case T_OBJECT: { // Base oops are OK, but not derived oops
const TypeOopPtr *tp = phi->type()->make_ptr()->isa_oopptr();
// Derived pointers are Bad (tm): what's the Base (for GC purposes) of a
// CMOVE'd derived pointer? It's a CMOVE'd derived base. Thus
// CMOVE'ing a derived pointer requires we also CMOVE the base. If we
// have a Phi for the base here that we convert to a CMOVE all is well
// and good. But if the base is dead, we'll not make a CMOVE. Later
// the allocator will have to produce a base by creating a CMOVE of the
// relevant bases. This puts the allocator in the business of
// manufacturing expensive instructions, generally a bad plan.
// Just Say No to Conditionally-Moved Derived Pointers.
if (tp && tp->offset() != 0)
return NULL;
cost++;
break;
}
default:
return NULL; // In particular, can't do memory or I/O
}
// Add in cost any speculative ops
for (uint j = 1; j < region->req(); j++) {
Node *proj = region->in(j);
Node *inp = phi->in(j);
if (get_ctrl(inp) == proj) { // Found local op
cost++;
// Check for a chain of dependent ops; these will all become
// speculative in a CMOV.
for (uint k = 1; k < inp->req(); k++)
if (get_ctrl(inp->in(k)) == proj)
cost += ConditionalMoveLimit; // Too much speculative goo
}
}
// See if the Phi is used by a Cmp or Narrow oop Decode/Encode.
// This will likely Split-If, a higher-payoff operation.
for (DUIterator_Fast kmax, k = phi->fast_outs(kmax); k < kmax; k++) {
Node* use = phi->fast_out(k);
if (use->is_Cmp() || use->is_DecodeNarrowPtr() || use->is_EncodeNarrowPtr())
cost += ConditionalMoveLimit;
// Is there a use inside the loop?
// Note: check only basic types since CMoveP is pinned.
if (!used_inside_loop && is_java_primitive(bt)) {
IdealLoopTree* u_loop = get_loop(has_ctrl(use) ? get_ctrl(use) : use);
if (r_loop == u_loop || r_loop->is_member(u_loop)) {
used_inside_loop = true;
}
}
}
}
Node* bol = iff->in(1);
assert(bol->Opcode() == Op_Bool, "");
int cmp_op = bol->in(1)->Opcode();
// It is expensive to generate flags from a float compare.
// Avoid duplicated float compare.
if (phis > 1 && (cmp_op == Op_CmpF || cmp_op == Op_CmpD)) return NULL;
float infrequent_prob = PROB_UNLIKELY_MAG(3);
// Ignore cost and blocks frequency if CMOVE can be moved outside the loop.
if (used_inside_loop) {
if (cost >= ConditionalMoveLimit) return NULL; // Too much goo
// BlockLayoutByFrequency optimization moves infrequent branch
// from hot path. No point in CMOV'ing in such case (110 is used
// instead of 100 to take into account not exactness of float value).
if (BlockLayoutByFrequency) {
infrequent_prob = MAX2(infrequent_prob, (float)BlockLayoutMinDiamondPercentage/110.0f);
}
}
// Check for highly predictable branch. No point in CMOV'ing if
// we are going to predict accurately all the time.
if (iff->_prob < infrequent_prob ||
iff->_prob > (1.0f - infrequent_prob))
return NULL;
// --------------
// Now replace all Phis with CMOV's
Node *cmov_ctrl = iff->in(0);
uint flip = (lp->Opcode() == Op_IfTrue);
Node_List wq;
while (1) {
PhiNode* phi = NULL;
for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
Node *out = region->fast_out(i);
if (out->is_Phi()) {
phi = out->as_Phi();
break;
}
}
if (phi == NULL) break;
#ifndef PRODUCT
if (PrintOpto && VerifyLoopOptimizations) tty->print_cr("CMOV");
#endif
// Move speculative ops
wq.push(phi);
while (wq.size() > 0) {
Node *n = wq.pop();
for (uint j = 1; j < n->req(); j++) {
Node* m = n->in(j);
if (m != NULL && !is_dominator(get_ctrl(m), cmov_ctrl)) {
#ifndef PRODUCT
if (PrintOpto && VerifyLoopOptimizations) {
tty->print(" speculate: ");
m->dump();
}
#endif
set_ctrl(m, cmov_ctrl);
wq.push(m);
}
}
}
Node *cmov = CMoveNode::make( C, cmov_ctrl, iff->in(1), phi->in(1+flip), phi->in(2-flip), _igvn.type(phi) );
register_new_node( cmov, cmov_ctrl );
_igvn.replace_node( phi, cmov );
#ifndef PRODUCT
if (TraceLoopOpts) {
tty->print("CMOV ");
r_loop->dump_head();
if (Verbose) {
bol->in(1)->dump(1);
cmov->dump(1);
}
}
if (VerifyLoopOptimizations) verify();
#endif
}
// The useless CFG diamond will fold up later; see the optimization in
// RegionNode::Ideal.
_igvn._worklist.push(region);
return iff->in(1);
}
//------------------------------split_if_with_blocks_pre-----------------------
// Do the real work in a non-recursive function. Data nodes want to be
// cloned in the pre-order so they can feed each other nicely.
Node *PhaseIdealLoop::split_if_with_blocks_pre( Node *n ) {
// Cloning these guys is unlikely to win
int n_op = n->Opcode();
if( n_op == Op_MergeMem ) return n;
if( n->is_Proj() ) return n;
// Do not clone-up CmpFXXX variations, as these are always
// followed by a CmpI
if( n->is_Cmp() ) return n;
// Attempt to use a conditional move instead of a phi/branch
if( ConditionalMoveLimit > 0 && n_op == Op_Region ) {
Node *cmov = conditional_move( n );
if( cmov ) return cmov;
}
if( n->is_CFG() || n->is_LoadStore() )
return n;
if( n_op == Op_Opaque1 || // Opaque nodes cannot be mod'd
n_op == Op_Opaque2 ) {
if( !C->major_progress() ) // If chance of no more loop opts...
_igvn._worklist.push(n); // maybe we'll remove them
return n;
}
if( n->is_Con() ) return n; // No cloning for Con nodes
Node *n_ctrl = get_ctrl(n);
if( !n_ctrl ) return n; // Dead node
// Attempt to remix address expressions for loop invariants
Node *m = remix_address_expressions( n );
if( m ) return m;
// Determine if the Node has inputs from some local Phi.
// Returns the block to clone thru.
Node *n_blk = has_local_phi_input( n );
if( !n_blk ) return n;
// Do not clone the trip counter through on a CountedLoop
// (messes up the canonical shape).
if( n_blk->is_CountedLoop() && n->Opcode() == Op_AddI ) return n;
// Check for having no control input; not pinned. Allow
// dominating control.
if( n->in(0) ) {
Node *dom = idom(n_blk);
if( dom_lca( n->in(0), dom ) != n->in(0) )
return n;
}
// Policy: when is it profitable. You must get more wins than
// policy before it is considered profitable. Policy is usually 0,
// so 1 win is considered profitable. Big merges will require big
// cloning, so get a larger policy.
int policy = n_blk->req() >> 2;
// If the loop is a candidate for range check elimination,
// delay splitting through it's phi until a later loop optimization
if (n_blk->is_CountedLoop()) {
IdealLoopTree *lp = get_loop(n_blk);
if (lp && lp->_rce_candidate) {
return n;
}
}
// Use same limit as split_if_with_blocks_post
if( C->unique() > 35000 ) return n; // Method too big
// Split 'n' through the merge point if it is profitable
Node *phi = split_thru_phi( n, n_blk, policy );
if (!phi) return n;
// Found a Phi to split thru!
// Replace 'n' with the new phi
_igvn.replace_node( n, phi );
// Moved a load around the loop, 'en-registering' something.
if (n_blk->is_Loop() && n->is_Load() &&
!phi->in(LoopNode::LoopBackControl)->is_Load())
C->set_major_progress();
return phi;
}
static bool merge_point_too_heavy(Compile* C, Node* region) {
// Bail out if the region and its phis have too many users.
int weight = 0;
for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
weight += region->fast_out(i)->outcnt();
}
int nodes_left = C->max_node_limit() - C->live_nodes();
if (weight * 8 > nodes_left) {
#ifndef PRODUCT
if (PrintOpto)
tty->print_cr("*** Split-if bails out: %d nodes, region weight %d", C->unique(), weight);
#endif
return true;
} else {
return false;
}
}
static bool merge_point_safe(Node* region) {
// 4799512: Stop split_if_with_blocks from splitting a block with a ConvI2LNode
// having a PhiNode input. This sidesteps the dangerous case where the split
// ConvI2LNode may become TOP if the input Value() does not
// overlap the ConvI2L range, leaving a node which may not dominate its
// uses.
// A better fix for this problem can be found in the BugTraq entry, but
// expediency for Mantis demands this hack.
// 6855164: If the merge point has a FastLockNode with a PhiNode input, we stop
// split_if_with_blocks from splitting a block because we could not move around
// the FastLockNode.
for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
Node* n = region->fast_out(i);
if (n->is_Phi()) {
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* m = n->fast_out(j);
if (m->is_FastLock())
return false;
#ifdef _LP64
if (m->Opcode() == Op_ConvI2L)
return false;
if (m->is_CastII() && m->isa_CastII()->has_range_check()) {
return false;
}
#endif
}
}
}
return true;
}
//------------------------------place_near_use---------------------------------
// Place some computation next to use but not inside inner loops.
// For inner loop uses move it to the preheader area.
Node *PhaseIdealLoop::place_near_use( Node *useblock ) const {
IdealLoopTree *u_loop = get_loop( useblock );
return (u_loop->_irreducible || u_loop->_child)
? useblock
: u_loop->_head->in(LoopNode::EntryControl);
}
//------------------------------split_if_with_blocks_post----------------------
// Do the real work in a non-recursive function. CFG hackery wants to be
// in the post-order, so it can dirty the I-DOM info and not use the dirtied
// info.
void PhaseIdealLoop::split_if_with_blocks_post( Node *n ) {
// Cloning Cmp through Phi's involves the split-if transform.
// FastLock is not used by an If
if( n->is_Cmp() && !n->is_FastLock() ) {
if( C->unique() > 35000 ) return; // Method too big
// Do not do 'split-if' if irreducible loops are present.
if( _has_irreducible_loops )
return;
Node *n_ctrl = get_ctrl(n);
// Determine if the Node has inputs from some local Phi.
// Returns the block to clone thru.
Node *n_blk = has_local_phi_input( n );
if( n_blk != n_ctrl ) return;
if( merge_point_too_heavy(C, n_ctrl) )
return;
if( n->outcnt() != 1 ) return; // Multiple bool's from 1 compare?
Node *bol = n->unique_out();
assert( bol->is_Bool(), "expect a bool here" );
if( bol->outcnt() != 1 ) return;// Multiple branches from 1 compare?
Node *iff = bol->unique_out();
// Check some safety conditions
if( iff->is_If() ) { // Classic split-if?
if( iff->in(0) != n_ctrl ) return; // Compare must be in same blk as if
} else if (iff->is_CMove()) { // Trying to split-up a CMOVE
// Can't split CMove with different control edge.
if (iff->in(0) != NULL && iff->in(0) != n_ctrl ) return;
if( get_ctrl(iff->in(2)) == n_ctrl ||
get_ctrl(iff->in(3)) == n_ctrl )
return; // Inputs not yet split-up
if ( get_loop(n_ctrl) != get_loop(get_ctrl(iff)) ) {
return; // Loop-invar test gates loop-varying CMOVE
}
} else {
return; // some other kind of node, such as an Allocate
}
// Do not do 'split-if' if some paths are dead. First do dead code
// elimination and then see if its still profitable.
for( uint i = 1; i < n_ctrl->req(); i++ )
if( n_ctrl->in(i) == C->top() )
return;
// When is split-if profitable? Every 'win' on means some control flow
// goes dead, so it's almost always a win.
int policy = 0;
// If trying to do a 'Split-If' at the loop head, it is only
// profitable if the cmp folds up on BOTH paths. Otherwise we
// risk peeling a loop forever.
// CNC - Disabled for now. Requires careful handling of loop
// body selection for the cloned code. Also, make sure we check
// for any input path not being in the same loop as n_ctrl. For
// irreducible loops we cannot check for 'n_ctrl->is_Loop()'
// because the alternative loop entry points won't be converted
// into LoopNodes.
IdealLoopTree *n_loop = get_loop(n_ctrl);
for( uint j = 1; j < n_ctrl->req(); j++ )
if( get_loop(n_ctrl->in(j)) != n_loop )
return;
// Check for safety of the merge point.
if( !merge_point_safe(n_ctrl) ) {
return;
}
// Split compare 'n' through the merge point if it is profitable
Node *phi = split_thru_phi( n, n_ctrl, policy );
if( !phi ) return;
// Found a Phi to split thru!
// Replace 'n' with the new phi
_igvn.replace_node( n, phi );
// Now split the bool up thru the phi
Node *bolphi = split_thru_phi( bol, n_ctrl, -1 );
guarantee(bolphi != NULL, "null boolean phi node");
_igvn.replace_node( bol, bolphi );
assert( iff->in(1) == bolphi, "" );
if( bolphi->Value(&_igvn)->singleton() )
return;
// Conditional-move? Must split up now
if( !iff->is_If() ) {
Node *cmovphi = split_thru_phi( iff, n_ctrl, -1 );
_igvn.replace_node( iff, cmovphi );
return;
}
// Now split the IF
do_split_if( iff );
return;
}
// Check for an IF ready to split; one that has its
// condition codes input coming from a Phi at the block start.
int n_op = n->Opcode();
// Check for an IF being dominated by another IF same test
if (n_op == Op_If) {
Node *bol = n->in(1);
uint max = bol->outcnt();
// Check for same test used more than once?
if (max > 1 && bol->is_Bool()) {
// Search up IDOMs to see if this IF is dominated.
Node *cutoff = get_ctrl(bol);
// Now search up IDOMs till cutoff, looking for a dominating test
Node *prevdom = n;
Node *dom = idom(prevdom);
while (dom != cutoff) {
if (dom->req() > 1 && dom->in(1) == bol && prevdom->in(0) == dom) {
// Replace the dominated test with an obvious true or false.
// Place it on the IGVN worklist for later cleanup.
C->set_major_progress();
dominated_by(prevdom, n, false, true);
#ifndef PRODUCT
if( VerifyLoopOptimizations ) verify();
#endif
return;
}
prevdom = dom;
dom = idom(prevdom);
}
}
}
// See if a shared loop-varying computation has no loop-varying uses.
// Happens if something is only used for JVM state in uncommon trap exits,
// like various versions of induction variable+offset. Clone the
// computation per usage to allow it to sink out of the loop.
if (has_ctrl(n) && !n->in(0)) {// n not dead and has no control edge (can float about)
Node *n_ctrl = get_ctrl(n);
IdealLoopTree *n_loop = get_loop(n_ctrl);
if( n_loop != _ltree_root ) {
DUIterator_Fast imax, i = n->fast_outs(imax);
for (; i < imax; i++) {
Node* u = n->fast_out(i);
if( !has_ctrl(u) ) break; // Found control user
IdealLoopTree *u_loop = get_loop(get_ctrl(u));
if( u_loop == n_loop ) break; // Found loop-varying use
if( n_loop->is_member( u_loop ) ) break; // Found use in inner loop
if( u->Opcode() == Op_Opaque1 ) break; // Found loop limit, bugfix for 4677003
}
bool did_break = (i < imax); // Did we break out of the previous loop?
if (!did_break && n->outcnt() > 1) { // All uses in outer loops!
Node *late_load_ctrl = NULL;
if (n->is_Load()) {
// If n is a load, get and save the result from get_late_ctrl(),
// to be later used in calculating the control for n's clones.
clear_dom_lca_tags();
late_load_ctrl = get_late_ctrl(n, n_ctrl);
}
// If n is a load, and the late control is the same as the current
// control, then the cloning of n is a pointless exercise, because
// GVN will ensure that we end up where we started.
if (!n->is_Load() || late_load_ctrl != n_ctrl) {
for (DUIterator_Last jmin, j = n->last_outs(jmin); j >= jmin; ) {
Node *u = n->last_out(j); // Clone private computation per use
_igvn.rehash_node_delayed(u);
Node *x = n->clone(); // Clone computation
Node *x_ctrl = NULL;
if( u->is_Phi() ) {
// Replace all uses of normal nodes. Replace Phi uses
// individually, so the separate Nodes can sink down
// different paths.
uint k = 1;
while( u->in(k) != n ) k++;
u->set_req( k, x );
// x goes next to Phi input path
x_ctrl = u->in(0)->in(k);
--j;
} else { // Normal use
// Replace all uses
for( uint k = 0; k < u->req(); k++ ) {
if( u->in(k) == n ) {
u->set_req( k, x );
--j;
}
}
x_ctrl = get_ctrl(u);
}
// Find control for 'x' next to use but not inside inner loops.
// For inner loop uses get the preheader area.
x_ctrl = place_near_use(x_ctrl);
if (n->is_Load()) {
// For loads, add a control edge to a CFG node outside of the loop
// to force them to not combine and return back inside the loop
// during GVN optimization (4641526).
//
// Because we are setting the actual control input, factor in
// the result from get_late_ctrl() so we respect any
// anti-dependences. (6233005).
x_ctrl = dom_lca(late_load_ctrl, x_ctrl);
// Don't allow the control input to be a CFG splitting node.
// Such nodes should only have ProjNodes as outs, e.g. IfNode
// should only have IfTrueNode and IfFalseNode (4985384).
x_ctrl = find_non_split_ctrl(x_ctrl);
assert(dom_depth(n_ctrl) <= dom_depth(x_ctrl), "n is later than its clone");
x->set_req(0, x_ctrl);
}
register_new_node(x, x_ctrl);
// Some institutional knowledge is needed here: 'x' is
// yanked because if the optimizer runs GVN on it all the
// cloned x's will common up and undo this optimization and
// be forced back in the loop. This is annoying because it
// makes +VerifyOpto report false-positives on progress. I
// tried setting control edges on the x's to force them to
// not combine, but the matching gets worried when it tries
// to fold a StoreP and an AddP together (as part of an
// address expression) and the AddP and StoreP have
// different controls.
if (!x->is_Load() && !x->is_DecodeNarrowPtr()) _igvn._worklist.yank(x);
}
_igvn.remove_dead_node(n);
}
}
}
}
// Check for Opaque2's who's loop has disappeared - who's input is in the
// same loop nest as their output. Remove 'em, they are no longer useful.
if( n_op == Op_Opaque2 &&
n->in(1) != NULL &&
get_loop(get_ctrl(n)) == get_loop(get_ctrl(n->in(1))) ) {
_igvn.replace_node( n, n->in(1) );
}
}
//------------------------------split_if_with_blocks---------------------------
// Check for aggressive application of 'split-if' optimization,
// using basic block level info.
void PhaseIdealLoop::split_if_with_blocks( VectorSet &visited, Node_Stack &nstack ) {
Node *n = C->root();
visited.set(n->_idx); // first, mark node as visited
// Do pre-visit work for root
n = split_if_with_blocks_pre( n );
uint cnt = n->outcnt();
uint i = 0;
while (true) {
// Visit all children
if (i < cnt) {
Node* use = n->raw_out(i);
++i;
if (use->outcnt() != 0 && !visited.test_set(use->_idx)) {
// Now do pre-visit work for this use
use = split_if_with_blocks_pre( use );
nstack.push(n, i); // Save parent and next use's index.
n = use; // Process all children of current use.
cnt = use->outcnt();
i = 0;
}
}
else {
// All of n's children have been processed, complete post-processing.
if (cnt != 0 && !n->is_Con()) {
assert(has_node(n), "no dead nodes");
split_if_with_blocks_post( n );
}
if (nstack.is_empty()) {
// Finished all nodes on stack.
break;
}
// Get saved parent node and next use's index. Visit the rest of uses.
n = nstack.node();
cnt = n->outcnt();
i = nstack.index();
nstack.pop();
}
}
}
//=============================================================================
//
// C L O N E A L O O P B O D Y
//
//------------------------------clone_iff--------------------------------------
// Passed in a Phi merging (recursively) some nearly equivalent Bool/Cmps.
// "Nearly" because all Nodes have been cloned from the original in the loop,
// but the fall-in edges to the Cmp are different. Clone bool/Cmp pairs
// through the Phi recursively, and return a Bool.
BoolNode *PhaseIdealLoop::clone_iff( PhiNode *phi, IdealLoopTree *loop ) {
// Convert this Phi into a Phi merging Bools
uint i;
for( i = 1; i < phi->req(); i++ ) {
Node *b = phi->in(i);
if( b->is_Phi() ) {
_igvn.replace_input_of(phi, i, clone_iff( b->as_Phi(), loop ));
} else {
assert( b->is_Bool(), "" );
}
}
Node *sample_bool = phi->in(1);
Node *sample_cmp = sample_bool->in(1);
// Make Phis to merge the Cmp's inputs.
PhiNode *phi1 = new (C) PhiNode( phi->in(0), Type::TOP );
PhiNode *phi2 = new (C) PhiNode( phi->in(0), Type::TOP );
for( i = 1; i < phi->req(); i++ ) {
Node *n1 = phi->in(i)->in(1)->in(1);
Node *n2 = phi->in(i)->in(1)->in(2);
phi1->set_req( i, n1 );
phi2->set_req( i, n2 );
phi1->set_type( phi1->type()->meet_speculative(n1->bottom_type()));
phi2->set_type( phi2->type()->meet_speculative(n2->bottom_type()));
}
// See if these Phis have been made before.
// Register with optimizer
Node *hit1 = _igvn.hash_find_insert(phi1);
if( hit1 ) { // Hit, toss just made Phi
_igvn.remove_dead_node(phi1); // Remove new phi
assert( hit1->is_Phi(), "" );
phi1 = (PhiNode*)hit1; // Use existing phi
} else { // Miss
_igvn.register_new_node_with_optimizer(phi1);
}
Node *hit2 = _igvn.hash_find_insert(phi2);
if( hit2 ) { // Hit, toss just made Phi
_igvn.remove_dead_node(phi2); // Remove new phi
assert( hit2->is_Phi(), "" );
phi2 = (PhiNode*)hit2; // Use existing phi
} else { // Miss
_igvn.register_new_node_with_optimizer(phi2);
}
// Register Phis with loop/block info
set_ctrl(phi1, phi->in(0));
set_ctrl(phi2, phi->in(0));
// Make a new Cmp
Node *cmp = sample_cmp->clone();
cmp->set_req( 1, phi1 );
cmp->set_req( 2, phi2 );
_igvn.register_new_node_with_optimizer(cmp);
set_ctrl(cmp, phi->in(0));
// Make a new Bool
Node *b = sample_bool->clone();
b->set_req(1,cmp);
_igvn.register_new_node_with_optimizer(b);
set_ctrl(b, phi->in(0));
assert( b->is_Bool(), "" );
return (BoolNode*)b;
}
//------------------------------clone_bool-------------------------------------
// Passed in a Phi merging (recursively) some nearly equivalent Bool/Cmps.
// "Nearly" because all Nodes have been cloned from the original in the loop,
// but the fall-in edges to the Cmp are different. Clone bool/Cmp pairs
// through the Phi recursively, and return a Bool.
CmpNode *PhaseIdealLoop::clone_bool( PhiNode *phi, IdealLoopTree *loop ) {
uint i;
// Convert this Phi into a Phi merging Bools
for( i = 1; i < phi->req(); i++ ) {
Node *b = phi->in(i);
if( b->is_Phi() ) {
_igvn.replace_input_of(phi, i, clone_bool( b->as_Phi(), loop ));
} else {
assert( b->is_Cmp() || b->is_top(), "inputs are all Cmp or TOP" );
}
}
Node *sample_cmp = phi->in(1);
// Make Phis to merge the Cmp's inputs.
PhiNode *phi1 = new (C) PhiNode( phi->in(0), Type::TOP );
PhiNode *phi2 = new (C) PhiNode( phi->in(0), Type::TOP );
for( uint j = 1; j < phi->req(); j++ ) {
Node *cmp_top = phi->in(j); // Inputs are all Cmp or TOP
Node *n1, *n2;
if( cmp_top->is_Cmp() ) {
n1 = cmp_top->in(1);
n2 = cmp_top->in(2);
} else {
n1 = n2 = cmp_top;
}
phi1->set_req( j, n1 );
phi2->set_req( j, n2 );
phi1->set_type(phi1->type()->meet_speculative(n1->bottom_type()));
phi2->set_type(phi2->type()->meet_speculative(n2->bottom_type()));
}
// See if these Phis have been made before.
// Register with optimizer
Node *hit1 = _igvn.hash_find_insert(phi1);
if( hit1 ) { // Hit, toss just made Phi
_igvn.remove_dead_node(phi1); // Remove new phi
assert( hit1->is_Phi(), "" );
phi1 = (PhiNode*)hit1; // Use existing phi
} else { // Miss
_igvn.register_new_node_with_optimizer(phi1);
}
Node *hit2 = _igvn.hash_find_insert(phi2);
if( hit2 ) { // Hit, toss just made Phi
_igvn.remove_dead_node(phi2); // Remove new phi
assert( hit2->is_Phi(), "" );
phi2 = (PhiNode*)hit2; // Use existing phi
} else { // Miss
_igvn.register_new_node_with_optimizer(phi2);
}
// Register Phis with loop/block info
set_ctrl(phi1, phi->in(0));
set_ctrl(phi2, phi->in(0));
// Make a new Cmp
Node *cmp = sample_cmp->clone();
cmp->set_req( 1, phi1 );
cmp->set_req( 2, phi2 );
_igvn.register_new_node_with_optimizer(cmp);
set_ctrl(cmp, phi->in(0));
assert( cmp->is_Cmp(), "" );
return (CmpNode*)cmp;
}
//------------------------------sink_use---------------------------------------
// If 'use' was in the loop-exit block, it now needs to be sunk
// below the post-loop merge point.
void PhaseIdealLoop::sink_use( Node *use, Node *post_loop ) {
if (!use->is_CFG() && get_ctrl(use) == post_loop->in(2)) {
set_ctrl(use, post_loop);
for (DUIterator j = use->outs(); use->has_out(j); j++)
sink_use(use->out(j), post_loop);
}
}
//------------------------------clone_loop-------------------------------------
//
// C L O N E A L O O P B O D Y
//
// This is the basic building block of the loop optimizations. It clones an
// entire loop body. It makes an old_new loop body mapping; with this mapping
// you can find the new-loop equivalent to an old-loop node. All new-loop
// nodes are exactly equal to their old-loop counterparts, all edges are the
// same. All exits from the old-loop now have a RegionNode that merges the
// equivalent new-loop path. This is true even for the normal "loop-exit"
// condition. All uses of loop-invariant old-loop values now come from (one
// or more) Phis that merge their new-loop equivalents.
//
// This operation leaves the graph in an illegal state: there are two valid
// control edges coming from the loop pre-header to both loop bodies. I'll
// definitely have to hack the graph after running this transform.
//
// From this building block I will further edit edges to perform loop peeling
// or loop unrolling or iteration splitting (Range-Check-Elimination), etc.
//
// Parameter side_by_size_idom:
// When side_by_size_idom is NULL, the dominator tree is constructed for
// the clone loop to dominate the original. Used in construction of
// pre-main-post loop sequence.
// When nonnull, the clone and original are side-by-side, both are
// dominated by the side_by_side_idom node. Used in construction of
// unswitched loops.
void PhaseIdealLoop::clone_loop( IdealLoopTree *loop, Node_List &old_new, int dd,
Node* side_by_side_idom) {
// Step 1: Clone the loop body. Make the old->new mapping.
uint i;
for( i = 0; i < loop->_body.size(); i++ ) {
Node *old = loop->_body.at(i);
Node *nnn = old->clone();
old_new.map( old->_idx, nnn );
_igvn.register_new_node_with_optimizer(nnn);
}
// Step 2: Fix the edges in the new body. If the old input is outside the
// loop use it. If the old input is INside the loop, use the corresponding
// new node instead.
for( i = 0; i < loop->_body.size(); i++ ) {
Node *old = loop->_body.at(i);
Node *nnn = old_new[old->_idx];
// Fix CFG/Loop controlling the new node
if (has_ctrl(old)) {
set_ctrl(nnn, old_new[get_ctrl(old)->_idx]);
} else {
set_loop(nnn, loop->_parent);
if (old->outcnt() > 0) {
set_idom( nnn, old_new[idom(old)->_idx], dd );
}
}
// Correct edges to the new node
for( uint j = 0; j < nnn->req(); j++ ) {
Node *n = nnn->in(j);
if( n ) {
IdealLoopTree *old_in_loop = get_loop( has_ctrl(n) ? get_ctrl(n) : n );
if( loop->is_member( old_in_loop ) )
nnn->set_req(j, old_new[n->_idx]);
}
}
_igvn.hash_find_insert(nnn);
}
Node *newhead = old_new[loop->_head->_idx];
set_idom(newhead, newhead->in(LoopNode::EntryControl), dd);
// Step 3: Now fix control uses. Loop varying control uses have already
// been fixed up (as part of all input edges in Step 2). Loop invariant
// control uses must be either an IfFalse or an IfTrue. Make a merge
// point to merge the old and new IfFalse/IfTrue nodes; make the use
// refer to this.
ResourceArea *area = Thread::current()->resource_area();
Node_List worklist(area);
uint new_counter = C->unique();
for( i = 0; i < loop->_body.size(); i++ ) {
Node* old = loop->_body.at(i);
if( !old->is_CFG() ) continue;
Node* nnn = old_new[old->_idx];
// Copy uses to a worklist, so I can munge the def-use info
// with impunity.
for (DUIterator_Fast jmax, j = old->fast_outs(jmax); j < jmax; j++)
worklist.push(old->fast_out(j));
while( worklist.size() ) { // Visit all uses
Node *use = worklist.pop();
if (!has_node(use)) continue; // Ignore dead nodes
IdealLoopTree *use_loop = get_loop( has_ctrl(use) ? get_ctrl(use) : use );
if( !loop->is_member( use_loop ) && use->is_CFG() ) {
// Both OLD and USE are CFG nodes here.
assert( use->is_Proj(), "" );
// Clone the loop exit control projection
Node *newuse = use->clone();
newuse->set_req(0,nnn);
_igvn.register_new_node_with_optimizer(newuse);
set_loop(newuse, use_loop);
set_idom(newuse, nnn, dom_depth(nnn) + 1 );
// We need a Region to merge the exit from the peeled body and the
// exit from the old loop body.
RegionNode *r = new (C) RegionNode(3);
// Map the old use to the new merge point
old_new.map( use->_idx, r );
uint dd_r = MIN2(dom_depth(newuse),dom_depth(use));
assert( dd_r >= dom_depth(dom_lca(newuse,use)), "" );
// The original user of 'use' uses 'r' instead.
for (DUIterator_Last lmin, l = use->last_outs(lmin); l >= lmin;) {
Node* useuse = use->last_out(l);
_igvn.rehash_node_delayed(useuse);
uint uses_found = 0;
if( useuse->in(0) == use ) {
useuse->set_req(0, r);
uses_found++;
if( useuse->is_CFG() ) {
assert( dom_depth(useuse) > dd_r, "" );
set_idom(useuse, r, dom_depth(useuse));
}
}
for( uint k = 1; k < useuse->req(); k++ ) {
if( useuse->in(k) == use ) {
useuse->set_req(k, r);
uses_found++;
}
}
l -= uses_found; // we deleted 1 or more copies of this edge
}
// Now finish up 'r'
r->set_req( 1, newuse );
r->set_req( 2, use );
_igvn.register_new_node_with_optimizer(r);
set_loop(r, use_loop);
set_idom(r, !side_by_side_idom ? newuse->in(0) : side_by_side_idom, dd_r);
} // End of if a loop-exit test
}
}
// Step 4: If loop-invariant use is not control, it must be dominated by a
// loop exit IfFalse/IfTrue. Find "proper" loop exit. Make a Region
// there if needed. Make a Phi there merging old and new used values.
Node_List *split_if_set = NULL;
Node_List *split_bool_set = NULL;
Node_List *split_cex_set = NULL;
for( i = 0; i < loop->_body.size(); i++ ) {
Node* old = loop->_body.at(i);
Node* nnn = old_new[old->_idx];
// Copy uses to a worklist, so I can munge the def-use info
// with impunity.
for (DUIterator_Fast jmax, j = old->fast_outs(jmax); j < jmax; j++)
worklist.push(old->fast_out(j));
while( worklist.size() ) {
Node *use = worklist.pop();
if (!has_node(use)) continue; // Ignore dead nodes
if (use->in(0) == C->top()) continue;
IdealLoopTree *use_loop = get_loop( has_ctrl(use) ? get_ctrl(use) : use );
// Check for data-use outside of loop - at least one of OLD or USE
// must not be a CFG node.
if( !loop->is_member( use_loop ) && (!old->is_CFG() || !use->is_CFG())) {
// If the Data use is an IF, that means we have an IF outside of the
// loop that is switching on a condition that is set inside of the
// loop. Happens if people set a loop-exit flag; then test the flag
// in the loop to break the loop, then test is again outside of the
// loop to determine which way the loop exited.
// Loop predicate If node connects to Bool node through Opaque1 node.
if (use->is_If() || use->is_CMove() || C->is_predicate_opaq(use)) {
// Since this code is highly unlikely, we lazily build the worklist
// of such Nodes to go split.
if( !split_if_set )
split_if_set = new Node_List(area);
split_if_set->push(use);
}
if( use->is_Bool() ) {
if( !split_bool_set )
split_bool_set = new Node_List(area);
split_bool_set->push(use);
}
if( use->Opcode() == Op_CreateEx ) {
if( !split_cex_set )
split_cex_set = new Node_List(area);
split_cex_set->push(use);
}
// Get "block" use is in
uint idx = 0;
while( use->in(idx) != old ) idx++;
Node *prev = use->is_CFG() ? use : get_ctrl(use);
assert( !loop->is_member( get_loop( prev ) ), "" );
Node *cfg = prev->_idx >= new_counter
? prev->in(2)
: idom(prev);
if( use->is_Phi() ) // Phi use is in prior block
cfg = prev->in(idx); // NOT in block of Phi itself
if (cfg->is_top()) { // Use is dead?
_igvn.replace_input_of(use, idx, C->top());
continue;
}
while( !loop->is_member( get_loop( cfg ) ) ) {
prev = cfg;
cfg = cfg->_idx >= new_counter ? cfg->in(2) : idom(cfg);
}
// If the use occurs after merging several exits from the loop, then
// old value must have dominated all those exits. Since the same old
// value was used on all those exits we did not need a Phi at this
// merge point. NOW we do need a Phi here. Each loop exit value
// is now merged with the peeled body exit; each exit gets its own
// private Phi and those Phis need to be merged here.
Node *phi;
if( prev->is_Region() ) {
if( idx == 0 ) { // Updating control edge?
phi = prev; // Just use existing control
} else { // Else need a new Phi
phi = PhiNode::make( prev, old );
// Now recursively fix up the new uses of old!
for( uint i = 1; i < prev->req(); i++ ) {
worklist.push(phi); // Onto worklist once for each 'old' input
}
}
} else {
// Get new RegionNode merging old and new loop exits
prev = old_new[prev->_idx];
assert( prev, "just made this in step 7" );
if( idx == 0 ) { // Updating control edge?
phi = prev; // Just use existing control
} else { // Else need a new Phi
// Make a new Phi merging data values properly
phi = PhiNode::make( prev, old );
phi->set_req( 1, nnn );
}
}
// If inserting a new Phi, check for prior hits
if( idx != 0 ) {
Node *hit = _igvn.hash_find_insert(phi);
if( hit == NULL ) {
_igvn.register_new_node_with_optimizer(phi); // Register new phi
} else { // or
// Remove the new phi from the graph and use the hit
_igvn.remove_dead_node(phi);
phi = hit; // Use existing phi
}
set_ctrl(phi, prev);
}
// Make 'use' use the Phi instead of the old loop body exit value
_igvn.replace_input_of(use, idx, phi);
if( use->_idx >= new_counter ) { // If updating new phis
// Not needed for correctness, but prevents a weak assert
// in AddPNode from tripping (when we end up with different
// base & derived Phis that will become the same after
// IGVN does CSE).
Node *hit = _igvn.hash_find_insert(use);
if( hit ) // Go ahead and re-hash for hits.
_igvn.replace_node( use, hit );
}
// If 'use' was in the loop-exit block, it now needs to be sunk
// below the post-loop merge point.
sink_use( use, prev );
}
}
}
// Check for IFs that need splitting/cloning. Happens if an IF outside of
// the loop uses a condition set in the loop. The original IF probably
// takes control from one or more OLD Regions (which in turn get from NEW
// Regions). In any case, there will be a set of Phis for each merge point
// from the IF up to where the original BOOL def exists the loop.
if( split_if_set ) {
while( split_if_set->size() ) {
Node *iff = split_if_set->pop();
if( iff->in(1)->is_Phi() ) {
BoolNode *b = clone_iff( iff->in(1)->as_Phi(), loop );
_igvn.replace_input_of(iff, 1, b);
}
}
}
if( split_bool_set ) {
while( split_bool_set->size() ) {
Node *b = split_bool_set->pop();
Node *phi = b->in(1);
assert( phi->is_Phi(), "" );
CmpNode *cmp = clone_bool( (PhiNode*)phi, loop );
_igvn.replace_input_of(b, 1, cmp);
}
}
if( split_cex_set ) {
while( split_cex_set->size() ) {
Node *b = split_cex_set->pop();
assert( b->in(0)->is_Region(), "" );
assert( b->in(1)->is_Phi(), "" );
assert( b->in(0)->in(0) == b->in(1)->in(0), "" );
split_up( b, b->in(0), NULL );
}
}
}
//---------------------- stride_of_possible_iv -------------------------------------
// Looks for an iff/bool/comp with one operand of the compare
// being a cycle involving an add and a phi,
// with an optional truncation (left-shift followed by a right-shift)
// of the add. Returns zero if not an iv.
int PhaseIdealLoop::stride_of_possible_iv(Node* iff) {
Node* trunc1 = NULL;
Node* trunc2 = NULL;
const TypeInt* ttype = NULL;
if (!iff->is_If() || iff->in(1) == NULL || !iff->in(1)->is_Bool()) {
return 0;
}
BoolNode* bl = iff->in(1)->as_Bool();
Node* cmp = bl->in(1);
if (!cmp || cmp->Opcode() != Op_CmpI && cmp->Opcode() != Op_CmpU) {
return 0;
}
// Must have an invariant operand
if (is_member(get_loop(iff), get_ctrl(cmp->in(2)))) {
return 0;
}
Node* add2 = NULL;
Node* cmp1 = cmp->in(1);
if (cmp1->is_Phi()) {
// (If (Bool (CmpX phi:(Phi ...(Optional-trunc(AddI phi add2))) )))
Node* phi = cmp1;
for (uint i = 1; i < phi->req(); i++) {
Node* in = phi->in(i);
Node* add = CountedLoopNode::match_incr_with_optional_truncation(in,
&trunc1, &trunc2, &ttype);
if (add && add->in(1) == phi) {
add2 = add->in(2);
break;
}
}
} else {
// (If (Bool (CmpX addtrunc:(Optional-trunc((AddI (Phi ...addtrunc...) add2)) )))
Node* addtrunc = cmp1;
Node* add = CountedLoopNode::match_incr_with_optional_truncation(addtrunc,
&trunc1, &trunc2, &ttype);
if (add && add->in(1)->is_Phi()) {
Node* phi = add->in(1);
for (uint i = 1; i < phi->req(); i++) {
if (phi->in(i) == addtrunc) {
add2 = add->in(2);
break;
}
}
}
}
if (add2 != NULL) {
const TypeInt* add2t = _igvn.type(add2)->is_int();
if (add2t->is_con()) {
return add2t->get_con();
}
}
return 0;
}
//---------------------- stay_in_loop -------------------------------------
// Return the (unique) control output node that's in the loop (if it exists.)
Node* PhaseIdealLoop::stay_in_loop( Node* n, IdealLoopTree *loop) {
Node* unique = NULL;
if (!n) return NULL;
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
Node* use = n->fast_out(i);
if (!has_ctrl(use) && loop->is_member(get_loop(use))) {
if (unique != NULL) {
return NULL;
}
unique = use;
}
}
return unique;
}
//------------------------------ register_node -------------------------------------
// Utility to register node "n" with PhaseIdealLoop
void PhaseIdealLoop::register_node(Node* n, IdealLoopTree *loop, Node* pred, int ddepth) {
_igvn.register_new_node_with_optimizer(n);
loop->_body.push(n);
if (n->is_CFG()) {
set_loop(n, loop);
set_idom(n, pred, ddepth);
} else {
set_ctrl(n, pred);
}
}
//------------------------------ proj_clone -------------------------------------
// Utility to create an if-projection
ProjNode* PhaseIdealLoop::proj_clone(ProjNode* p, IfNode* iff) {
ProjNode* c = p->clone()->as_Proj();
c->set_req(0, iff);
return c;
}
//------------------------------ short_circuit_if -------------------------------------
// Force the iff control output to be the live_proj
Node* PhaseIdealLoop::short_circuit_if(IfNode* iff, ProjNode* live_proj) {
guarantee(live_proj != NULL, "null projection");
int proj_con = live_proj->_con;
assert(proj_con == 0 || proj_con == 1, "false or true projection");
Node *con = _igvn.intcon(proj_con);
set_ctrl(con, C->root());
if (iff) {
iff->set_req(1, con);
}
return con;
}
//------------------------------ insert_if_before_proj -------------------------------------
// Insert a new if before an if projection (* - new node)
//
// before
// if(test)
// / \
// v v
// other-proj proj (arg)
//
// after
// if(test)
// / \
// / v
// | * proj-clone
// v |
// other-proj v
// * new_if(relop(cmp[IU](left,right)))
// / \
// v v
// * new-proj proj
// (returned)
//
ProjNode* PhaseIdealLoop::insert_if_before_proj(Node* left, bool Signed, BoolTest::mask relop, Node* right, ProjNode* proj) {
IfNode* iff = proj->in(0)->as_If();
IdealLoopTree *loop = get_loop(proj);
ProjNode *other_proj = iff->proj_out(!proj->is_IfTrue())->as_Proj();
int ddepth = dom_depth(proj);
_igvn.rehash_node_delayed(iff);
_igvn.rehash_node_delayed(proj);
proj->set_req(0, NULL); // temporary disconnect
ProjNode* proj2 = proj_clone(proj, iff);
register_node(proj2, loop, iff, ddepth);
Node* cmp = Signed ? (Node*) new (C)CmpINode(left, right) : (Node*) new (C)CmpUNode(left, right);
register_node(cmp, loop, proj2, ddepth);
BoolNode* bol = new (C)BoolNode(cmp, relop);
register_node(bol, loop, proj2, ddepth);
IfNode* new_if = new (C)IfNode(proj2, bol, iff->_prob, iff->_fcnt);
register_node(new_if, loop, proj2, ddepth);
proj->set_req(0, new_if); // reattach
set_idom(proj, new_if, ddepth);
ProjNode* new_exit = proj_clone(other_proj, new_if)->as_Proj();
guarantee(new_exit != NULL, "null exit node");
register_node(new_exit, get_loop(other_proj), new_if, ddepth);
return new_exit;
}
//------------------------------ insert_region_before_proj -------------------------------------
// Insert a region before an if projection (* - new node)
//
// before
// if(test)
// / |
// v |
// proj v
// other-proj
//
// after
// if(test)
// / |
// v |
// * proj-clone v
// | other-proj
// v
// * new-region
// |
// v
// * dum_if
// / \
// v \
// * dum-proj v
// proj
//
RegionNode* PhaseIdealLoop::insert_region_before_proj(ProjNode* proj) {
IfNode* iff = proj->in(0)->as_If();
IdealLoopTree *loop = get_loop(proj);
ProjNode *other_proj = iff->proj_out(!proj->is_IfTrue())->as_Proj();
int ddepth = dom_depth(proj);
_igvn.rehash_node_delayed(iff);
_igvn.rehash_node_delayed(proj);
proj->set_req(0, NULL); // temporary disconnect
ProjNode* proj2 = proj_clone(proj, iff);
register_node(proj2, loop, iff, ddepth);
RegionNode* reg = new (C)RegionNode(2);
reg->set_req(1, proj2);
register_node(reg, loop, iff, ddepth);
IfNode* dum_if = new (C)IfNode(reg, short_circuit_if(NULL, proj), iff->_prob, iff->_fcnt);
register_node(dum_if, loop, reg, ddepth);
proj->set_req(0, dum_if); // reattach
set_idom(proj, dum_if, ddepth);
ProjNode* dum_proj = proj_clone(other_proj, dum_if);
register_node(dum_proj, loop, dum_if, ddepth);
return reg;
}
//------------------------------ insert_cmpi_loop_exit -------------------------------------
// Clone a signed compare loop exit from an unsigned compare and
// insert it before the unsigned cmp on the stay-in-loop path.
// All new nodes inserted in the dominator tree between the original
// if and it's projections. The original if test is replaced with
// a constant to force the stay-in-loop path.
//
// This is done to make sure that the original if and it's projections
// still dominate the same set of control nodes, that the ctrl() relation
// from data nodes to them is preserved, and that their loop nesting is
// preserved.
//
// before
// if(i <u limit) unsigned compare loop exit
// / |
// v v
// exit-proj stay-in-loop-proj
//
// after
// if(stay-in-loop-const) original if
// / |
// / v
// / if(i < limit) new signed test
// / / |
// / / v
// / / if(i <u limit) new cloned unsigned test
// / / / |
// v v v |
// region |
// | |
// dum-if |
// / | |
// ether | |
// v v
// exit-proj stay-in-loop-proj
//
IfNode* PhaseIdealLoop::insert_cmpi_loop_exit(IfNode* if_cmpu, IdealLoopTree *loop) {
const bool Signed = true;
const bool Unsigned = false;
BoolNode* bol = if_cmpu->in(1)->as_Bool();
if (bol->_test._test != BoolTest::lt) return NULL;
CmpNode* cmpu = bol->in(1)->as_Cmp();
if (cmpu->Opcode() != Op_CmpU) return NULL;
int stride = stride_of_possible_iv(if_cmpu);
if (stride == 0) return NULL;
Node* lp_proj = stay_in_loop(if_cmpu, loop);
guarantee(lp_proj != NULL, "null loop node");
ProjNode* lp_continue = lp_proj->as_Proj();
ProjNode* lp_exit = if_cmpu->proj_out(!lp_continue->is_IfTrue())->as_Proj();
Node* limit = NULL;
if (stride > 0) {
limit = cmpu->in(2);
} else {
limit = _igvn.makecon(TypeInt::ZERO);
set_ctrl(limit, C->root());
}
// Create a new region on the exit path
RegionNode* reg = insert_region_before_proj(lp_exit);
guarantee(reg != NULL, "null region node");
// Clone the if-cmpu-true-false using a signed compare
BoolTest::mask rel_i = stride > 0 ? bol->_test._test : BoolTest::ge;
ProjNode* cmpi_exit = insert_if_before_proj(cmpu->in(1), Signed, rel_i, limit, lp_continue);
reg->add_req(cmpi_exit);
// Clone the if-cmpu-true-false
BoolTest::mask rel_u = bol->_test._test;
ProjNode* cmpu_exit = insert_if_before_proj(cmpu->in(1), Unsigned, rel_u, cmpu->in(2), lp_continue);
reg->add_req(cmpu_exit);
// Force original if to stay in loop.
short_circuit_if(if_cmpu, lp_continue);
return cmpi_exit->in(0)->as_If();
}
//------------------------------ remove_cmpi_loop_exit -------------------------------------
// Remove a previously inserted signed compare loop exit.
void PhaseIdealLoop::remove_cmpi_loop_exit(IfNode* if_cmp, IdealLoopTree *loop) {
Node* lp_proj = stay_in_loop(if_cmp, loop);
assert(if_cmp->in(1)->in(1)->Opcode() == Op_CmpI &&
stay_in_loop(lp_proj, loop)->is_If() &&
stay_in_loop(lp_proj, loop)->in(1)->in(1)->Opcode() == Op_CmpU, "inserted cmpi before cmpu");
Node *con = _igvn.makecon(lp_proj->is_IfTrue() ? TypeInt::ONE : TypeInt::ZERO);
set_ctrl(con, C->root());
if_cmp->set_req(1, con);
}
//------------------------------ scheduled_nodelist -------------------------------------
// Create a post order schedule of nodes that are in the
// "member" set. The list is returned in "sched".
// The first node in "sched" is the loop head, followed by
// nodes which have no inputs in the "member" set, and then
// followed by the nodes that have an immediate input dependence
// on a node in "sched".
void PhaseIdealLoop::scheduled_nodelist( IdealLoopTree *loop, VectorSet& member, Node_List &sched ) {
assert(member.test(loop->_head->_idx), "loop head must be in member set");
Arena *a = Thread::current()->resource_area();
VectorSet visited(a);
Node_Stack nstack(a, loop->_body.size());
Node* n = loop->_head; // top of stack is cached in "n"
uint idx = 0;
visited.set(n->_idx);
// Initially push all with no inputs from within member set
for(uint i = 0; i < loop->_body.size(); i++ ) {
Node *elt = loop->_body.at(i);
if (member.test(elt->_idx)) {
bool found = false;
for (uint j = 0; j < elt->req(); j++) {
Node* def = elt->in(j);
if (def && member.test(def->_idx) && def != elt) {
found = true;
break;
}
}
if (!found && elt != loop->_head) {
nstack.push(n, idx);
n = elt;
assert(!visited.test(n->_idx), "not seen yet");
visited.set(n->_idx);
}
}
}
// traverse out's that are in the member set
while (true) {
if (idx < n->outcnt()) {
Node* use = n->raw_out(idx);
idx++;
if (!visited.test_set(use->_idx)) {
if (member.test(use->_idx)) {
nstack.push(n, idx);
n = use;
idx = 0;
}
}
} else {
// All outputs processed
sched.push(n);
if (nstack.is_empty()) break;
n = nstack.node();
idx = nstack.index();
nstack.pop();
}
}
}
//------------------------------ has_use_in_set -------------------------------------
// Has a use in the vector set
bool PhaseIdealLoop::has_use_in_set( Node* n, VectorSet& vset ) {
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* use = n->fast_out(j);
if (vset.test(use->_idx)) {
return true;
}
}
return false;
}
//------------------------------ has_use_internal_to_set -------------------------------------
// Has use internal to the vector set (ie. not in a phi at the loop head)
bool PhaseIdealLoop::has_use_internal_to_set( Node* n, VectorSet& vset, IdealLoopTree *loop ) {
Node* head = loop->_head;
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* use = n->fast_out(j);
if (vset.test(use->_idx) && !(use->is_Phi() && use->in(0) == head)) {
return true;
}
}
return false;
}
//------------------------------ clone_for_use_outside_loop -------------------------------------
// clone "n" for uses that are outside of loop
int PhaseIdealLoop::clone_for_use_outside_loop( IdealLoopTree *loop, Node* n, Node_List& worklist ) {
int cloned = 0;
assert(worklist.size() == 0, "should be empty");
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* use = n->fast_out(j);
if( !loop->is_member(get_loop(has_ctrl(use) ? get_ctrl(use) : use)) ) {
worklist.push(use);
}
}
while( worklist.size() ) {
Node *use = worklist.pop();
if (!has_node(use) || use->in(0) == C->top()) continue;
uint j;
for (j = 0; j < use->req(); j++) {
if (use->in(j) == n) break;
}
assert(j < use->req(), "must be there");
// clone "n" and insert it between the inputs of "n" and the use outside the loop
Node* n_clone = n->clone();
_igvn.replace_input_of(use, j, n_clone);
cloned++;
Node* use_c;
if (!use->is_Phi()) {
use_c = has_ctrl(use) ? get_ctrl(use) : use->in(0);
} else {
// Use in a phi is considered a use in the associated predecessor block
use_c = use->in(0)->in(j);
}
set_ctrl(n_clone, use_c);
assert(!loop->is_member(get_loop(use_c)), "should be outside loop");
get_loop(use_c)->_body.push(n_clone);
_igvn.register_new_node_with_optimizer(n_clone);
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("loop exit cloning old: %d new: %d newbb: %d", n->_idx, n_clone->_idx, get_ctrl(n_clone)->_idx);
}
#endif
}
return cloned;
}
//------------------------------ clone_for_special_use_inside_loop -------------------------------------
// clone "n" for special uses that are in the not_peeled region.
// If these def-uses occur in separate blocks, the code generator
// marks the method as not compilable. For example, if a "BoolNode"
// is in a different basic block than the "IfNode" that uses it, then
// the compilation is aborted in the code generator.
void PhaseIdealLoop::clone_for_special_use_inside_loop( IdealLoopTree *loop, Node* n,
VectorSet& not_peel, Node_List& sink_list, Node_List& worklist ) {
if (n->is_Phi() || n->is_Load()) {
return;
}
assert(worklist.size() == 0, "should be empty");
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* use = n->fast_out(j);
if ( not_peel.test(use->_idx) &&
(use->is_If() || use->is_CMove() || use->is_Bool()) &&
use->in(1) == n) {
worklist.push(use);
}
}
if (worklist.size() > 0) {
// clone "n" and insert it between inputs of "n" and the use
Node* n_clone = n->clone();
loop->_body.push(n_clone);
_igvn.register_new_node_with_optimizer(n_clone);
set_ctrl(n_clone, get_ctrl(n));
sink_list.push(n_clone);
not_peel <<= n_clone->_idx; // add n_clone to not_peel set.
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("special not_peeled cloning old: %d new: %d", n->_idx, n_clone->_idx);
}
#endif
while( worklist.size() ) {
Node *use = worklist.pop();
_igvn.rehash_node_delayed(use);
for (uint j = 1; j < use->req(); j++) {
if (use->in(j) == n) {
use->set_req(j, n_clone);
}
}
}
}
}
//------------------------------ insert_phi_for_loop -------------------------------------
// Insert phi(lp_entry_val, back_edge_val) at use->in(idx) for loop lp if phi does not already exist
void PhaseIdealLoop::insert_phi_for_loop( Node* use, uint idx, Node* lp_entry_val, Node* back_edge_val, LoopNode* lp ) {
Node *phi = PhiNode::make(lp, back_edge_val);
phi->set_req(LoopNode::EntryControl, lp_entry_val);
// Use existing phi if it already exists
Node *hit = _igvn.hash_find_insert(phi);
if( hit == NULL ) {
_igvn.register_new_node_with_optimizer(phi);
set_ctrl(phi, lp);
} else {
// Remove the new phi from the graph and use the hit
_igvn.remove_dead_node(phi);
phi = hit;
}
_igvn.replace_input_of(use, idx, phi);
}
#ifdef ASSERT
//------------------------------ is_valid_loop_partition -------------------------------------
// Validate the loop partition sets: peel and not_peel
bool PhaseIdealLoop::is_valid_loop_partition( IdealLoopTree *loop, VectorSet& peel, Node_List& peel_list,
VectorSet& not_peel ) {
uint i;
// Check that peel_list entries are in the peel set
for (i = 0; i < peel_list.size(); i++) {
if (!peel.test(peel_list.at(i)->_idx)) {
return false;
}
}
// Check at loop members are in one of peel set or not_peel set
for (i = 0; i < loop->_body.size(); i++ ) {
Node *def = loop->_body.at(i);
uint di = def->_idx;
// Check that peel set elements are in peel_list
if (peel.test(di)) {
if (not_peel.test(di)) {
return false;
}
// Must be in peel_list also
bool found = false;
for (uint j = 0; j < peel_list.size(); j++) {
if (peel_list.at(j)->_idx == di) {
found = true;
break;
}
}
if (!found) {
return false;
}
} else if (not_peel.test(di)) {
if (peel.test(di)) {
return false;
}
} else {
return false;
}
}
return true;
}
//------------------------------ is_valid_clone_loop_exit_use -------------------------------------
// Ensure a use outside of loop is of the right form
bool PhaseIdealLoop::is_valid_clone_loop_exit_use( IdealLoopTree *loop, Node* use, uint exit_idx) {
Node *use_c = has_ctrl(use) ? get_ctrl(use) : use;
return (use->is_Phi() &&
use_c->is_Region() && use_c->req() == 3 &&
(use_c->in(exit_idx)->Opcode() == Op_IfTrue ||
use_c->in(exit_idx)->Opcode() == Op_IfFalse ||
use_c->in(exit_idx)->Opcode() == Op_JumpProj) &&
loop->is_member( get_loop( use_c->in(exit_idx)->in(0) ) ) );
}
//------------------------------ is_valid_clone_loop_form -------------------------------------
// Ensure that all uses outside of loop are of the right form
bool PhaseIdealLoop::is_valid_clone_loop_form( IdealLoopTree *loop, Node_List& peel_list,
uint orig_exit_idx, uint clone_exit_idx) {
uint len = peel_list.size();
for (uint i = 0; i < len; i++) {
Node *def = peel_list.at(i);
for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) {
Node *use = def->fast_out(j);
Node *use_c = has_ctrl(use) ? get_ctrl(use) : use;
if (!loop->is_member(get_loop(use_c))) {
// use is not in the loop, check for correct structure
if (use->in(0) == def) {
// Okay
} else if (!is_valid_clone_loop_exit_use(loop, use, orig_exit_idx)) {
return false;
}
}
}
}
return true;
}
#endif
//------------------------------ partial_peel -------------------------------------
// Partially peel (aka loop rotation) the top portion of a loop (called
// the peel section below) by cloning it and placing one copy just before
// the new loop head and the other copy at the bottom of the new loop.
//
// before after where it came from
//
// stmt1 stmt1
// loop: stmt2 clone
// stmt2 if condA goto exitA clone
// if condA goto exitA new_loop: new
// stmt3 stmt3 clone
// if !condB goto loop if condB goto exitB clone
// exitB: stmt2 orig
// stmt4 if !condA goto new_loop orig
// exitA: goto exitA
// exitB:
// stmt4
// exitA:
//
// Step 1: find the cut point: an exit test on probable
// induction variable.
// Step 2: schedule (with cloning) operations in the peel
// section that can be executed after the cut into
// the section that is not peeled. This may need
// to clone operations into exit blocks. For
// instance, a reference to A[i] in the not-peel
// section and a reference to B[i] in an exit block
// may cause a left-shift of i by 2 to be placed
// in the peel block. This step will clone the left
// shift into the exit block and sink the left shift
// from the peel to the not-peel section.
// Step 3: clone the loop, retarget the control, and insert
// phis for values that are live across the new loop
// head. This is very dependent on the graph structure
// from clone_loop. It creates region nodes for
// exit control and associated phi nodes for values
// flow out of the loop through that exit. The region
// node is dominated by the clone's control projection.
// So the clone's peel section is placed before the
// new loop head, and the clone's not-peel section is
// forms the top part of the new loop. The original
// peel section forms the tail of the new loop.
// Step 4: update the dominator tree and recompute the
// dominator depth.
//
// orig
//
// stmt1
// |
// v
// loop predicate
// |
// v
// loop<----+
// | |
// stmt2 |
// | |
// v |
// ifA |
// / | |
// v v |
// false true ^ <-- last_peel
// / | |
// / ===|==cut |
// / stmt3 | <-- first_not_peel
// / | |
// | v |
// v ifB |
// exitA: / \ |
// / \ |
// v v |
// false true |
// / \ |
// / ----+
// |
// v
// exitB:
// stmt4
//
//
// after clone loop
//
// stmt1
// |
// v
// loop predicate
// / \
// clone / \ orig
// / \
// / \
// v v
// +---->loop loop<----+
// | | | |
// | stmt2 stmt2 |
// | | | |
// | v v |
// | ifA ifA |
// | | \ / | |
// | v v v v |
// ^ true false false true ^ <-- last_peel
// | | ^ \ / | |
// | cut==|== \ \ / ===|==cut |
// | stmt3 \ \ / stmt3 | <-- first_not_peel
// | | dom | | | |
// | v \ 1v v2 v |
// | ifB regionA ifB |
// | / \ | / \ |
// | / \ v / \ |
// | v v exitA: v v |
// | true false false true |
// | / ^ \ / \ |
// +---- \ \ / ----+
// dom \ /
// \ 1v v2
// regionB
// |
// v
// exitB:
// stmt4
//
//
// after partial peel
//
// stmt1
// |
// v
// loop predicate
// /
// clone / orig
// / TOP
// / \
// v v
// TOP->loop loop----+
// | | |
// stmt2 stmt2 |
// | | |
// v v |
// ifA ifA |
// | \ / | |
// v v v v |
// true false false true | <-- last_peel
// | ^ \ / +------|---+
// +->newloop \ \ / === ==cut | |
// | stmt3 \ \ / TOP | |
// | | dom | | stmt3 | | <-- first_not_peel
// | v \ 1v v2 v | |
// | ifB regionA ifB ^ v
// | / \ | / \ | |
// | / \ v / \ | |
// | v v exitA: v v | |
// | true false false true | |
// | / ^ \ / \ | |
// | | \ \ / v | |
// | | dom \ / TOP | |
// | | \ 1v v2 | |
// ^ v regionB | |
// | | | | |
// | | v ^ v
// | | exitB: | |
// | | stmt4 | |
// | +------------>-----------------+ |
// | |
// +-----------------<---------------------+
//
//
// final graph
//
// stmt1
// |
// v
// loop predicate
// |
// v
// stmt2 clone
// |
// v
// ........> ifA clone
// : / |
// dom / |
// : v v
// : false true
// : | |
// : | v
// : | newloop<-----+
// : | | |
// : | stmt3 clone |
// : | | |
// : | v |
// : | ifB |
// : | / \ |
// : | v v |
// : | false true |
// : | | | |
// : | v stmt2 |
// : | exitB: | |
// : | stmt4 v |
// : | ifA orig |
// : | / \ |
// : | / \ |
// : | v v |
// : | false true |
// : | / \ |
// : v v -----+
// RegionA
// |
// v
// exitA
//
bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) {
assert(!loop->_head->is_CountedLoop(), "Non-counted loop only");
if (!loop->_head->is_Loop()) {
return false; }
LoopNode *head = loop->_head->as_Loop();
if (head->is_partial_peel_loop() || head->partial_peel_has_failed()) {
return false;
}
// Check for complex exit control
for(uint ii = 0; ii < loop->_body.size(); ii++ ) {
Node *n = loop->_body.at(ii);
int opc = n->Opcode();
if (n->is_Call() ||
opc == Op_Catch ||
opc == Op_CatchProj ||
opc == Op_Jump ||
opc == Op_JumpProj) {
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("\nExit control too complex: lp: %d", head->_idx);
}
#endif
return false;
}
}
int dd = dom_depth(head);
// Step 1: find cut point
// Walk up dominators to loop head looking for first loop exit
// which is executed on every path thru loop.
IfNode *peel_if = NULL;
IfNode *peel_if_cmpu = NULL;
Node *iff = loop->tail();
while( iff != head ) {
if( iff->is_If() ) {
Node *ctrl = get_ctrl(iff->in(1));
if (ctrl->is_top()) return false; // Dead test on live IF.
// If loop-varying exit-test, check for induction variable
if( loop->is_member(get_loop(ctrl)) &&
loop->is_loop_exit(iff) &&
is_possible_iv_test(iff)) {
Node* cmp = iff->in(1)->in(1);
if (cmp->Opcode() == Op_CmpI) {
peel_if = iff->as_If();
} else {
assert(cmp->Opcode() == Op_CmpU, "must be CmpI or CmpU");
peel_if_cmpu = iff->as_If();
}
}
}
iff = idom(iff);
}
// Prefer signed compare over unsigned compare.
IfNode* new_peel_if = NULL;
if (peel_if == NULL) {
if (!PartialPeelAtUnsignedTests || peel_if_cmpu == NULL) {
return false; // No peel point found
}
new_peel_if = insert_cmpi_loop_exit(peel_if_cmpu, loop);
if (new_peel_if == NULL) {
return false; // No peel point found
}
peel_if = new_peel_if;
}
Node* last_peel = stay_in_loop(peel_if, loop);
Node* first_not_peeled = stay_in_loop(last_peel, loop);
if (first_not_peeled == NULL || first_not_peeled == head) {
return false;
}
#if !defined(PRODUCT)
if (TraceLoopOpts) {
tty->print("PartialPeel ");
loop->dump_head();
}
if (TracePartialPeeling) {
tty->print_cr("before partial peel one iteration");
Node_List wl;
Node* t = head->in(2);
while (true) {
wl.push(t);
if (t == head) break;
t = idom(t);
}
while (wl.size() > 0) {
Node* tt = wl.pop();
tt->dump();
if (tt == last_peel) tty->print_cr("-- cut --");
}
}
#endif
ResourceArea *area = Thread::current()->resource_area();
VectorSet peel(area);
VectorSet not_peel(area);
Node_List peel_list(area);
Node_List worklist(area);
Node_List sink_list(area);
// Set of cfg nodes to peel are those that are executable from
// the head through last_peel.
assert(worklist.size() == 0, "should be empty");
worklist.push(head);
peel.set(head->_idx);
while (worklist.size() > 0) {
Node *n = worklist.pop();
if (n != last_peel) {
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* use = n->fast_out(j);
if (use->is_CFG() &&
loop->is_member(get_loop(use)) &&
!peel.test_set(use->_idx)) {
worklist.push(use);
}
}
}
}
// Set of non-cfg nodes to peel are those that are control
// dependent on the cfg nodes.
uint i;
for(i = 0; i < loop->_body.size(); i++ ) {
Node *n = loop->_body.at(i);
Node *n_c = has_ctrl(n) ? get_ctrl(n) : n;
if (peel.test(n_c->_idx)) {
peel.set(n->_idx);
} else {
not_peel.set(n->_idx);
}
}
// Step 2: move operations from the peeled section down into the
// not-peeled section
// Get a post order schedule of nodes in the peel region
// Result in right-most operand.
scheduled_nodelist(loop, peel, peel_list );
assert(is_valid_loop_partition(loop, peel, peel_list, not_peel), "bad partition");
// For future check for too many new phis
uint old_phi_cnt = 0;
for (DUIterator_Fast jmax, j = head->fast_outs(jmax); j < jmax; j++) {
Node* use = head->fast_out(j);
if (use->is_Phi()) old_phi_cnt++;
}
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("\npeeled list");
}
#endif
// Evacuate nodes in peel region into the not_peeled region if possible
uint new_phi_cnt = 0;
uint cloned_for_outside_use = 0;
for (i = 0; i < peel_list.size();) {
Node* n = peel_list.at(i);
#if !defined(PRODUCT)
if (TracePartialPeeling) n->dump();
#endif
bool incr = true;
if ( !n->is_CFG() ) {
if ( has_use_in_set(n, not_peel) ) {
// If not used internal to the peeled region,
// move "n" from peeled to not_peeled region.
if ( !has_use_internal_to_set(n, peel, loop) ) {
// if not pinned and not a load (which maybe anti-dependent on a store)
// and not a CMove (Matcher expects only bool->cmove).
if ( n->in(0) == NULL && !n->is_Load() && !n->is_CMove() ) {
cloned_for_outside_use += clone_for_use_outside_loop( loop, n, worklist );
sink_list.push(n);
peel >>= n->_idx; // delete n from peel set.
not_peel <<= n->_idx; // add n to not_peel set.
peel_list.remove(i);
incr = false;
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("sink to not_peeled region: %d newbb: %d",
n->_idx, get_ctrl(n)->_idx);
}
#endif
}
} else {
// Otherwise check for special def-use cases that span
// the peel/not_peel boundary such as bool->if
clone_for_special_use_inside_loop( loop, n, not_peel, sink_list, worklist );
new_phi_cnt++;
}
}
}
if (incr) i++;
}
if (new_phi_cnt > old_phi_cnt + PartialPeelNewPhiDelta) {
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("\nToo many new phis: %d old %d new cmpi: %c",
new_phi_cnt, old_phi_cnt, new_peel_if != NULL?'T':'F');
}
#endif
if (new_peel_if != NULL) {
remove_cmpi_loop_exit(new_peel_if, loop);
}
// Inhibit more partial peeling on this loop
assert(!head->is_partial_peel_loop(), "not partial peeled");
head->mark_partial_peel_failed();
if (cloned_for_outside_use > 0) {
// Terminate this round of loop opts because
// the graph outside this loop was changed.
C->set_major_progress();
return true;
}
return false;
}
// Step 3: clone loop, retarget control, and insert new phis
// Create new loop head for new phis and to hang
// the nodes being moved (sinked) from the peel region.
LoopNode* new_head = new (C) LoopNode(last_peel, last_peel);
new_head->set_unswitch_count(head->unswitch_count()); // Preserve
_igvn.register_new_node_with_optimizer(new_head);
assert(first_not_peeled->in(0) == last_peel, "last_peel <- first_not_peeled");
first_not_peeled->set_req(0, new_head);
set_loop(new_head, loop);
loop->_body.push(new_head);
not_peel.set(new_head->_idx);
set_idom(new_head, last_peel, dom_depth(first_not_peeled));
set_idom(first_not_peeled, new_head, dom_depth(first_not_peeled));
while (sink_list.size() > 0) {
Node* n = sink_list.pop();
set_ctrl(n, new_head);
}
assert(is_valid_loop_partition(loop, peel, peel_list, not_peel), "bad partition");
clone_loop( loop, old_new, dd );
const uint clone_exit_idx = 1;
const uint orig_exit_idx = 2;
assert(is_valid_clone_loop_form( loop, peel_list, orig_exit_idx, clone_exit_idx ), "bad clone loop");
Node* head_clone = old_new[head->_idx];
LoopNode* new_head_clone = old_new[new_head->_idx]->as_Loop();
Node* orig_tail_clone = head_clone->in(2);
// Add phi if "def" node is in peel set and "use" is not
for(i = 0; i < peel_list.size(); i++ ) {
Node *def = peel_list.at(i);
if (!def->is_CFG()) {
for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) {
Node *use = def->fast_out(j);
if (has_node(use) && use->in(0) != C->top() &&
(!peel.test(use->_idx) ||
(use->is_Phi() && use->in(0) == head)) ) {
worklist.push(use);
}
}
while( worklist.size() ) {
Node *use = worklist.pop();
for (uint j = 1; j < use->req(); j++) {
Node* n = use->in(j);
if (n == def) {
// "def" is in peel set, "use" is not in peel set
// or "use" is in the entry boundary (a phi) of the peel set
Node* use_c = has_ctrl(use) ? get_ctrl(use) : use;
if ( loop->is_member(get_loop( use_c )) ) {
// use is in loop
if (old_new[use->_idx] != NULL) { // null for dead code
Node* use_clone = old_new[use->_idx];
_igvn.replace_input_of(use, j, C->top());
insert_phi_for_loop( use_clone, j, old_new[def->_idx], def, new_head_clone );
}
} else {
assert(is_valid_clone_loop_exit_use(loop, use, orig_exit_idx), "clone loop format");
// use is not in the loop, check if the live range includes the cut
Node* lp_if = use_c->in(orig_exit_idx)->in(0);
if (not_peel.test(lp_if->_idx)) {
assert(j == orig_exit_idx, "use from original loop");
insert_phi_for_loop( use, clone_exit_idx, old_new[def->_idx], def, new_head_clone );
}
}
}
}
}
}
}
// Step 3b: retarget control
// Redirect control to the new loop head if a cloned node in
// the not_peeled region has control that points into the peeled region.
// This necessary because the cloned peeled region will be outside
// the loop.
// from to
// cloned-peeled <---+
// new_head_clone: | <--+
// cloned-not_peeled in(0) in(0)
// orig-peeled
for(i = 0; i < loop->_body.size(); i++ ) {
Node *n = loop->_body.at(i);
if (!n->is_CFG() && n->in(0) != NULL &&
not_peel.test(n->_idx) && peel.test(n->in(0)->_idx)) {
Node* n_clone = old_new[n->_idx];
_igvn.replace_input_of(n_clone, 0, new_head_clone);
}
}
// Backedge of the surviving new_head (the clone) is original last_peel
_igvn.replace_input_of(new_head_clone, LoopNode::LoopBackControl, last_peel);
// Cut first node in original not_peel set
_igvn.rehash_node_delayed(new_head); // Multiple edge updates:
new_head->set_req(LoopNode::EntryControl, C->top()); // use rehash_node_delayed / set_req instead of
new_head->set_req(LoopNode::LoopBackControl, C->top()); // multiple replace_input_of calls
// Copy head_clone back-branch info to original head
// and remove original head's loop entry and
// clone head's back-branch
_igvn.rehash_node_delayed(head); // Multiple edge updates
head->set_req(LoopNode::EntryControl, head_clone->in(LoopNode::LoopBackControl));
head->set_req(LoopNode::LoopBackControl, C->top());
_igvn.replace_input_of(head_clone, LoopNode::LoopBackControl, C->top());
// Similarly modify the phis
for (DUIterator_Fast kmax, k = head->fast_outs(kmax); k < kmax; k++) {
Node* use = head->fast_out(k);
if (use->is_Phi() && use->outcnt() > 0) {
Node* use_clone = old_new[use->_idx];
_igvn.rehash_node_delayed(use); // Multiple edge updates
use->set_req(LoopNode::EntryControl, use_clone->in(LoopNode::LoopBackControl));
use->set_req(LoopNode::LoopBackControl, C->top());
_igvn.replace_input_of(use_clone, LoopNode::LoopBackControl, C->top());
}
}
// Step 4: update dominator tree and dominator depth
set_idom(head, orig_tail_clone, dd);
recompute_dom_depth();
// Inhibit more partial peeling on this loop
new_head_clone->set_partial_peel_loop();
C->set_major_progress();
loop->record_for_igvn();
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("\nafter partial peel one iteration");
Node_List wl(area);
Node* t = last_peel;
while (true) {
wl.push(t);
if (t == head_clone) break;
t = idom(t);
}
while (wl.size() > 0) {
Node* tt = wl.pop();
if (tt == head) tty->print_cr("orig head");
else if (tt == new_head_clone) tty->print_cr("new head");
else if (tt == head_clone) tty->print_cr("clone head");
tt->dump();
}
}
#endif
return true;
}
//------------------------------reorg_offsets----------------------------------
// Reorganize offset computations to lower register pressure. Mostly
// prevent loop-fallout uses of the pre-incremented trip counter (which are
// then alive with the post-incremented trip counter forcing an extra
// register move)
void PhaseIdealLoop::reorg_offsets(IdealLoopTree *loop) {
// Perform it only for canonical counted loops.
// Loop's shape could be messed up by iteration_split_impl.
if (!loop->_head->is_CountedLoop())
return;
if (!loop->_head->as_Loop()->is_valid_counted_loop())
return;
CountedLoopNode *cl = loop->_head->as_CountedLoop();
CountedLoopEndNode *cle = cl->loopexit();
Node *exit = cle->proj_out(false);
Node *phi = cl->phi();
// Check for the special case of folks using the pre-incremented
// trip-counter on the fall-out path (forces the pre-incremented
// and post-incremented trip counter to be live at the same time).
// Fix this by adjusting to use the post-increment trip counter.
bool progress = true;
while (progress) {
progress = false;
for (DUIterator_Fast imax, i = phi->fast_outs(imax); i < imax; i++) {
Node* use = phi->fast_out(i); // User of trip-counter
if (!has_ctrl(use)) continue;
Node *u_ctrl = get_ctrl(use);
if (use->is_Phi()) {
u_ctrl = NULL;
for (uint j = 1; j < use->req(); j++)
if (use->in(j) == phi)
u_ctrl = dom_lca(u_ctrl, use->in(0)->in(j));
}
IdealLoopTree *u_loop = get_loop(u_ctrl);
// Look for loop-invariant use
if (u_loop == loop) continue;
if (loop->is_member(u_loop)) continue;
// Check that use is live out the bottom. Assuming the trip-counter
// update is right at the bottom, uses of of the loop middle are ok.
if (dom_lca(exit, u_ctrl) != exit) continue;
// Hit! Refactor use to use the post-incremented tripcounter.
// Compute a post-increment tripcounter.
Node *opaq = new (C) Opaque2Node( C, cle->incr() );
register_new_node(opaq, exit);
Node *neg_stride = _igvn.intcon(-cle->stride_con());
set_ctrl(neg_stride, C->root());
Node *post = new (C) AddINode( opaq, neg_stride);
register_new_node(post, exit);
_igvn.rehash_node_delayed(use);
for (uint j = 1; j < use->req(); j++) {
if (use->in(j) == phi)
use->set_req(j, post);
}
// Since DU info changed, rerun loop
progress = true;
break;
}
}
}
C:\hotspot-69087d08d473\src\share\vm/opto/loopPredicate.cpp
/*
* Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "opto/loopnode.hpp"
#include "opto/addnode.hpp"
#include "opto/callnode.hpp"
#include "opto/connode.hpp"
#include "opto/loopnode.hpp"
#include "opto/matcher.hpp"
#include "opto/mulnode.hpp"
#include "opto/rootnode.hpp"
#include "opto/subnode.hpp"
/*
* The general idea of Loop Predication is to insert a predicate on the entry
* path to a loop, and raise a uncommon trap if the check of the condition fails.
* The condition checks are promoted from inside the loop body, and thus
* the checks inside the loop could be eliminated. Currently, loop predication
* optimization has been applied to remove array range check and loop invariant
* checks (such as null checks).
*/
//-------------------------------register_control-------------------------
void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred) {
assert(n->is_CFG(), "must be control node");
_igvn.register_new_node_with_optimizer(n);
loop->_body.push(n);
set_loop(n, loop);
// When called from beautify_loops() idom is not constructed yet.
if (_idom != NULL) {
set_idom(n, pred, dom_depth(pred));
}
}
//------------------------------create_new_if_for_predicate------------------------
// create a new if above the uct_if_pattern for the predicate to be promoted.
//
// before after
// ---------- ----------
// ctrl ctrl
// | |
// | |
// v v
// iff new_iff
// / \ / \
// / \ / \
// v v v v
// uncommon_proj cont_proj if_uct if_cont
// \ | | | |
// \ | | | |
// v v v | v
// rgn loop | iff
// | | / \
// | | / \
// v | v v
// uncommon_trap | uncommon_proj cont_proj
// \ \ | |
// \ \ | |
// v v v v
// rgn loop
// |
// |
// v
// uncommon_trap
//
//
// We will create a region to guard the uct call if there is no one there.
// The true projecttion (if_cont) of the new_iff is returned.
// This code is also used to clone predicates to clonned loops.
ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
Deoptimization::DeoptReason reason) {
assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
IfNode* iff = cont_proj->in(0)->as_If();
ProjNode *uncommon_proj = iff->proj_out(1 - cont_proj->_con);
Node *rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
uint proj_index = 1; // region's edge corresponding to uncommon_proj
if (!rgn->is_Region()) { // create a region to guard the call
assert(rgn->is_Call(), "must be call uct");
CallNode* call = rgn->as_Call();
IdealLoopTree* loop = get_loop(call);
rgn = new (C) RegionNode(1);
rgn->add_req(uncommon_proj);
register_control(rgn, loop, uncommon_proj);
_igvn.hash_delete(call);
call->set_req(0, rgn);
// When called from beautify_loops() idom is not constructed yet.
if (_idom != NULL) {
set_idom(call, rgn, dom_depth(rgn));
}
} else {
// Find region's edge corresponding to uncommon_proj
for (; proj_index < rgn->req(); proj_index++)
if (rgn->in(proj_index) == uncommon_proj) break;
assert(proj_index < rgn->req(), "sanity");
}
Node* entry = iff->in(0);
if (new_entry != NULL) {
// Clonning the predicate to new location.
entry = new_entry;
}
// Create new_iff
IdealLoopTree* lp = get_loop(entry);
IfNode *new_iff = iff->clone()->as_If();
new_iff->set_req(0, entry);
register_control(new_iff, lp, entry);
Node *if_cont = new (C) IfTrueNode(new_iff);
Node *if_uct = new (C) IfFalseNode(new_iff);
if (cont_proj->is_IfFalse()) {
// Swap
Node* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
}
register_control(if_cont, lp, new_iff);
register_control(if_uct, get_loop(rgn), new_iff);
// if_uct to rgn
_igvn.hash_delete(rgn);
rgn->add_req(if_uct);
// When called from beautify_loops() idom is not constructed yet.
if (_idom != NULL) {
Node* ridom = idom(rgn);
Node* nrdom = dom_lca(ridom, new_iff);
set_idom(rgn, nrdom, dom_depth(rgn));
}
// If rgn has phis add new edges which has the same
// value as on original uncommon_proj pass.
assert(rgn->in(rgn->req() -1) == if_uct, "new edge should be last");
bool has_phi = false;
for (DUIterator_Fast imax, i = rgn->fast_outs(imax); i < imax; i++) {
Node* use = rgn->fast_out(i);
if (use->is_Phi() && use->outcnt() > 0) {
assert(use->in(0) == rgn, "");
_igvn.rehash_node_delayed(use);
use->add_req(use->in(proj_index));
has_phi = true;
}
}
assert(!has_phi || rgn->req() > 3, "no phis when region is created");
if (new_entry == NULL) {
// Attach if_cont to iff
_igvn.hash_delete(iff);
iff->set_req(0, if_cont);
if (_idom != NULL) {
set_idom(iff, if_cont, dom_depth(iff));
}
}
return if_cont->as_Proj();
}
//------------------------------create_new_if_for_predicate------------------------
// Create a new if below new_entry for the predicate to be cloned (IGVN optimization)
ProjNode* PhaseIterGVN::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
Deoptimization::DeoptReason reason) {
assert(new_entry != 0, "only used for clone predicate");
assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
IfNode* iff = cont_proj->in(0)->as_If();
ProjNode *uncommon_proj = iff->proj_out(1 - cont_proj->_con);
Node *rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
uint proj_index = 1; // region's edge corresponding to uncommon_proj
if (!rgn->is_Region()) { // create a region to guard the call
assert(rgn->is_Call(), "must be call uct");
CallNode* call = rgn->as_Call();
rgn = new (C) RegionNode(1);
register_new_node_with_optimizer(rgn);
rgn->add_req(uncommon_proj);
hash_delete(call);
call->set_req(0, rgn);
} else {
// Find region's edge corresponding to uncommon_proj
for (; proj_index < rgn->req(); proj_index++)
if (rgn->in(proj_index) == uncommon_proj) break;
assert(proj_index < rgn->req(), "sanity");
}
// Create new_iff in new location.
IfNode *new_iff = iff->clone()->as_If();
new_iff->set_req(0, new_entry);
register_new_node_with_optimizer(new_iff);
Node *if_cont = new (C) IfTrueNode(new_iff);
Node *if_uct = new (C) IfFalseNode(new_iff);
if (cont_proj->is_IfFalse()) {
// Swap
Node* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
}
register_new_node_with_optimizer(if_cont);
register_new_node_with_optimizer(if_uct);
// if_uct to rgn
hash_delete(rgn);
rgn->add_req(if_uct);
// If rgn has phis add corresponding new edges which has the same
// value as on original uncommon_proj pass.
assert(rgn->in(rgn->req() -1) == if_uct, "new edge should be last");
bool has_phi = false;
for (DUIterator_Fast imax, i = rgn->fast_outs(imax); i < imax; i++) {
Node* use = rgn->fast_out(i);
if (use->is_Phi() && use->outcnt() > 0) {
rehash_node_delayed(use);
use->add_req(use->in(proj_index));
has_phi = true;
}
}
assert(!has_phi || rgn->req() > 3, "no phis when region is created");
return if_cont->as_Proj();
}
//--------------------------clone_predicate-----------------------
ProjNode* PhaseIdealLoop::clone_predicate(ProjNode* predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason,
PhaseIdealLoop* loop_phase,
PhaseIterGVN* igvn) {
ProjNode* new_predicate_proj;
if (loop_phase != NULL) {
new_predicate_proj = loop_phase->create_new_if_for_predicate(predicate_proj, new_entry, reason);
} else {
new_predicate_proj = igvn->create_new_if_for_predicate(predicate_proj, new_entry, reason);
}
IfNode* iff = new_predicate_proj->in(0)->as_If();
Node* ctrl = iff->in(0);
// Match original condition since predicate's projections could be swapped.
assert(predicate_proj->in(0)->in(1)->in(1)->Opcode()==Op_Opaque1, "must be");
Node* opq = new (igvn->C) Opaque1Node(igvn->C, predicate_proj->in(0)->in(1)->in(1)->in(1));
igvn->C->add_predicate_opaq(opq);
Node* bol = new (igvn->C) Conv2BNode(opq);
if (loop_phase != NULL) {
loop_phase->register_new_node(opq, ctrl);
loop_phase->register_new_node(bol, ctrl);
} else {
igvn->register_new_node_with_optimizer(opq);
igvn->register_new_node_with_optimizer(bol);
}
igvn->hash_delete(iff);
iff->set_req(1, bol);
return new_predicate_proj;
}
//--------------------------clone_loop_predicates-----------------------
// Interface from IGVN
Node* PhaseIterGVN::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) {
return PhaseIdealLoop::clone_loop_predicates(old_entry, new_entry, clone_limit_check, NULL, this);
}
// Interface from PhaseIdealLoop
Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) {
return clone_loop_predicates(old_entry, new_entry, clone_limit_check, this, &this->_igvn);
}
// Clone loop predicates to cloned loops (peeled, unswitched, split_if).
Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry,
bool clone_limit_check,
PhaseIdealLoop* loop_phase,
PhaseIterGVN* igvn) {
#ifdef ASSERT
if (new_entry == NULL || !(new_entry->is_Proj() || new_entry->is_Region() || new_entry->is_SafePoint())) {
if (new_entry != NULL)
new_entry->dump();
assert(false, "not IfTrue, IfFalse, Region or SafePoint");
}
#endif
// Search original predicates
Node* entry = old_entry;
ProjNode* limit_check_proj = NULL;
if (LoopLimitCheck) {
limit_check_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (limit_check_proj != NULL) {
entry = entry->in(0)->in(0);
}
}
if (UseLoopPredicate) {
ProjNode* predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (predicate_proj != NULL) { // right pattern that can be used by loop predication
// clone predicate
new_entry = clone_predicate(predicate_proj, new_entry,
Deoptimization::Reason_predicate,
loop_phase, igvn);
assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone predicate");
if (TraceLoopPredicate) {
tty->print("Loop Predicate cloned: ");
debug_only( new_entry->in(0)->dump(); )
}
}
}
if (limit_check_proj != NULL && clone_limit_check) {
// Clone loop limit check last to insert it before loop.
// Don't clone a limit check which was already finalized
// for this counted loop (only one limit check is needed).
new_entry = clone_predicate(limit_check_proj, new_entry,
Deoptimization::Reason_loop_limit_check,
loop_phase, igvn);
assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone limit check");
if (TraceLoopLimitCheck) {
tty->print("Loop Limit Check cloned: ");
debug_only( new_entry->in(0)->dump(); )
}
}
return new_entry;
}
//--------------------------skip_loop_predicates------------------------------
// Skip related predicates.
Node* PhaseIdealLoop::skip_loop_predicates(Node* entry) {
Node* predicate = NULL;
if (LoopLimitCheck) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate != NULL) {
entry = entry->in(0)->in(0);
}
}
if (UseLoopPredicate) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (predicate != NULL) { // right pattern that can be used by loop predication
IfNode* iff = entry->in(0)->as_If();
ProjNode* uncommon_proj = iff->proj_out(1 - entry->as_Proj()->_con);
Node* rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
entry = entry->in(0)->in(0);
while (entry != NULL && entry->is_Proj() && entry->in(0)->is_If()) {
uncommon_proj = entry->in(0)->as_If()->proj_out(1 - entry->as_Proj()->_con);
if (uncommon_proj->unique_ctrl_out() != rgn)
break;
entry = entry->in(0)->in(0);
}
}
}
return entry;
}
//--------------------------find_predicate_insertion_point-------------------
// Find a good location to insert a predicate
ProjNode* PhaseIdealLoop::find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason) {
if (start_c == NULL || !start_c->is_Proj())
return NULL;
if (start_c->as_Proj()->is_uncommon_trap_if_pattern(reason)) {
return start_c->as_Proj();
}
return NULL;
}
//--------------------------find_predicate------------------------------------
// Find a predicate
Node* PhaseIdealLoop::find_predicate(Node* entry) {
Node* predicate = NULL;
if (LoopLimitCheck) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate != NULL) { // right pattern that can be used by loop predication
return entry;
}
}
if (UseLoopPredicate) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (predicate != NULL) { // right pattern that can be used by loop predication
return entry;
}
}
return NULL;
}
//------------------------------Invariance-----------------------------------
// Helper class for loop_predication_impl to compute invariance on the fly and
// clone invariants.
class Invariance : public StackObj {
VectorSet _visited, _invariant;
Node_Stack _stack;
VectorSet _clone_visited;
Node_List _old_new; // map of old to new (clone)
IdealLoopTree* _lpt;
PhaseIdealLoop* _phase;
// Helper function to set up the invariance for invariance computation
// If n is a known invariant, set up directly. Otherwise, look up the
// the possibility to push n onto the stack for further processing.
void visit(Node* use, Node* n) {
if (_lpt->is_invariant(n)) { // known invariant
_invariant.set(n->_idx);
} else if (!n->is_CFG()) {
Node *n_ctrl = _phase->ctrl_or_self(n);
Node *u_ctrl = _phase->ctrl_or_self(use); // self if use is a CFG
if (_phase->is_dominator(n_ctrl, u_ctrl)) {
_stack.push(n, n->in(0) == NULL ? 1 : 0);
}
}
}
// Compute invariance for "the_node" and (possibly) all its inputs recursively
// on the fly
void compute_invariance(Node* n) {
assert(_visited.test(n->_idx), "must be");
visit(n, n);
while (_stack.is_nonempty()) {
Node* n = _stack.node();
uint idx = _stack.index();
if (idx == n->req()) { // all inputs are processed
_stack.pop();
// n is invariant if it's inputs are all invariant
bool all_inputs_invariant = true;
for (uint i = 0; i < n->req(); i++) {
Node* in = n->in(i);
if (in == NULL) continue;
assert(_visited.test(in->_idx), "must have visited input");
if (!_invariant.test(in->_idx)) { // bad guy
all_inputs_invariant = false;
break;
}
}
if (all_inputs_invariant) {
// If n's control is a predicate that was moved out of the
// loop, it was marked invariant but n is only invariant if
// it depends only on that test. Otherwise, unless that test
// is out of the loop, it's not invariant.
if (n->is_CFG() || n->depends_only_on_test() || n->in(0) == NULL || !_phase->is_member(_lpt, n->in(0))) {
_invariant.set(n->_idx); // I am a invariant too
}
}
} else { // process next input
_stack.set_index(idx + 1);
Node* m = n->in(idx);
if (m != NULL && !_visited.test_set(m->_idx)) {
visit(n, m);
}
}
}
}
// Helper function to set up _old_new map for clone_nodes.
// If n is a known invariant, set up directly ("clone" of n == n).
// Otherwise, push n onto the stack for real cloning.
void clone_visit(Node* n) {
assert(_invariant.test(n->_idx), "must be invariant");
if (_lpt->is_invariant(n)) { // known invariant
_old_new.map(n->_idx, n);
} else { // to be cloned
assert(!n->is_CFG(), "should not see CFG here");
_stack.push(n, n->in(0) == NULL ? 1 : 0);
}
}
// Clone "n" and (possibly) all its inputs recursively
void clone_nodes(Node* n, Node* ctrl) {
clone_visit(n);
while (_stack.is_nonempty()) {
Node* n = _stack.node();
uint idx = _stack.index();
if (idx == n->req()) { // all inputs processed, clone n!
_stack.pop();
// clone invariant node
Node* n_cl = n->clone();
_old_new.map(n->_idx, n_cl);
_phase->register_new_node(n_cl, ctrl);
for (uint i = 0; i < n->req(); i++) {
Node* in = n_cl->in(i);
if (in == NULL) continue;
n_cl->set_req(i, _old_new[in->_idx]);
}
} else { // process next input
_stack.set_index(idx + 1);
Node* m = n->in(idx);
if (m != NULL && !_clone_visited.test_set(m->_idx)) {
clone_visit(m); // visit the input
}
}
}
}
public:
Invariance(Arena* area, IdealLoopTree* lpt) :
_lpt(lpt), _phase(lpt->_phase),
_visited(area), _invariant(area), _stack(area, 10 /* guess */),
_clone_visited(area), _old_new(area)
{
Node* head = _lpt->_head;
Node* entry = head->in(LoopNode::EntryControl);
if (entry->outcnt() != 1) {
// If a node is pinned between the predicates and the loop
// entry, we won't be able to move any node in the loop that
// depends on it above it in a predicate. Mark all those nodes
// as non loop invariatnt.
Unique_Node_List wq;
wq.push(entry);
for (uint next = 0; next < wq.size(); ++next) {
Node *n = wq.at(next);
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
Node* u = n->fast_out(i);
if (!u->is_CFG()) {
Node* c = _phase->get_ctrl(u);
if (_lpt->is_member(_phase->get_loop(c)) || _phase->is_dominator(c, head)) {
_visited.set(u->_idx);
wq.push(u);
}
}
}
}
}
}
// Map old to n for invariance computation and clone
void map_ctrl(Node* old, Node* n) {
assert(old->is_CFG() && n->is_CFG(), "must be");
_old_new.map(old->_idx, n); // "clone" of old is n
_invariant.set(old->_idx); // old is invariant
_clone_visited.set(old->_idx);
}
// Driver function to compute invariance
bool is_invariant(Node* n) {
if (!_visited.test_set(n->_idx))
compute_invariance(n);
return (_invariant.test(n->_idx) != 0);
}
// Driver function to clone invariant
Node* clone(Node* n, Node* ctrl) {
assert(ctrl->is_CFG(), "must be");
assert(_invariant.test(n->_idx), "must be an invariant");
if (!_clone_visited.test(n->_idx))
clone_nodes(n, ctrl);
return _old_new[n->_idx];
}
};
//------------------------------is_range_check_if -----------------------------------
// Returns true if the predicate of iff is in "scale*iv + offset u< load_range(ptr)" format
// Note: this function is particularly designed for loop predication. We require load_range
// and offset to be loop invariant computed on the fly by "invar"
bool IdealLoopTree::is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar) const {
if (!is_loop_exit(iff)) {
return false;
}
if (!iff->in(1)->is_Bool()) {
return false;
}
const BoolNode *bol = iff->in(1)->as_Bool();
if (bol->_test._test != BoolTest::lt) {
return false;
}
if (!bol->in(1)->is_Cmp()) {
return false;
}
const CmpNode *cmp = bol->in(1)->as_Cmp();
if (cmp->Opcode() != Op_CmpU) {
return false;
}
Node* range = cmp->in(2);
if (range->Opcode() != Op_LoadRange) {
const TypeInt* tint = phase->_igvn.type(range)->isa_int();
if (tint == NULL || tint->empty() || tint->_lo < 0) {
// Allow predication on positive values that aren't LoadRanges.
// This allows optimization of loops where the length of the
// array is a known value and doesn't need to be loaded back
// from the array.
return false;
}
}
if (!invar.is_invariant(range)) {
return false;
}
Node *iv = _head->as_CountedLoop()->phi();
int scale = 0;
Node *offset = NULL;
if (!phase->is_scaled_iv_plus_offset(cmp->in(1), iv, &scale, &offset)) {
return false;
}
if (offset && !invar.is_invariant(offset)) { // offset must be invariant
return false;
}
return true;
}
//------------------------------rc_predicate-----------------------------------
// Create a range check predicate
//
// for (i = init; i < limit; i += stride) {
// a[scale*i+offset]
// }
//
// Compute max(scale*i + offset) for init <= i < limit and build the predicate
// as "max(scale*i + offset) u< a.length".
//
// There are two cases for max(scale*i + offset):
// (1) stride*scale > 0
// max(scale*i + offset) = scale*(limit-stride) + offset
// (2) stride*scale < 0
// max(scale*i + offset) = scale*init + offset
BoolNode* PhaseIdealLoop::rc_predicate(IdealLoopTree *loop, Node* ctrl,
int scale, Node* offset,
Node* init, Node* limit, jint stride,
Node* range, bool upper, bool &overflow) {
jint con_limit = limit->is_Con() ? limit->get_int() : 0;
jint con_init = init->is_Con() ? init->get_int() : 0;
jint con_offset = offset->is_Con() ? offset->get_int() : 0;
stringStream* predString = NULL;
if (TraceLoopPredicate) {
predString = new stringStream();
predString->print("rc_predicate ");
}
overflow = false;
Node* max_idx_expr = NULL;
const TypeInt* idx_type = TypeInt::INT;
if ((stride > 0) == (scale > 0) == upper) {
if (TraceLoopPredicate) {
if (limit->is_Con()) {
predString->print("(%d ", con_limit);
} else {
predString->print("(limit ");
}
predString->print("- %d) ", stride);
}
// Check if (limit - stride) may overflow
const TypeInt* limit_type = _igvn.type(limit)->isa_int();
jint limit_lo = limit_type->_lo;
jint limit_hi = limit_type->_hi;
jint res_lo = limit_lo - stride;
jint res_hi = limit_hi - stride;
if ((stride > 0 && (res_lo < limit_lo)) ||
(stride < 0 && (res_hi > limit_hi))) {
// No overflow possible
ConINode* con_stride = _igvn.intcon(stride);
set_ctrl(con_stride, C->root());
max_idx_expr = new (C) SubINode(limit, con_stride);
idx_type = TypeInt::make(limit_lo - stride, limit_hi - stride, limit_type->_widen);
} else {
// May overflow
overflow = true;
limit = new (C) ConvI2LNode(limit);
register_new_node(limit, ctrl);
ConLNode* con_stride = _igvn.longcon(stride);
set_ctrl(con_stride, C->root());
max_idx_expr = new (C) SubLNode(limit, con_stride);
}
register_new_node(max_idx_expr, ctrl);
} else {
if (TraceLoopPredicate) {
if (init->is_Con()) {
predString->print("%d ", con_init);
} else {
predString->print("init ");
}
}
idx_type = _igvn.type(init)->isa_int();
max_idx_expr = init;
}
if (scale != 1) {
ConNode* con_scale = _igvn.intcon(scale);
set_ctrl(con_scale, C->root());
if (TraceLoopPredicate) {
predString->print("* %d ", scale);
}
// Check if (scale * max_idx_expr) may overflow
const TypeInt* scale_type = TypeInt::make(scale);
MulINode* mul = new (C) MulINode(max_idx_expr, con_scale);
idx_type = (TypeInt*)mul->mul_ring(idx_type, scale_type);
if (overflow || TypeInt::INT->higher_equal(idx_type)) {
// May overflow
mul->destruct();
if (!overflow) {
max_idx_expr = new (C) ConvI2LNode(max_idx_expr);
register_new_node(max_idx_expr, ctrl);
}
overflow = true;
con_scale = _igvn.longcon(scale);
set_ctrl(con_scale, C->root());
max_idx_expr = new (C) MulLNode(max_idx_expr, con_scale);
} else {
// No overflow possible
max_idx_expr = mul;
}
register_new_node(max_idx_expr, ctrl);
}
if (offset && (!offset->is_Con() || con_offset != 0)){
if (TraceLoopPredicate) {
if (offset->is_Con()) {
predString->print("+ %d ", con_offset);
} else {
predString->print("+ offset");
}
}
// Check if (max_idx_expr + offset) may overflow
const TypeInt* offset_type = _igvn.type(offset)->isa_int();
jint lo = idx_type->_lo + offset_type->_lo;
jint hi = idx_type->_hi + offset_type->_hi;
if (overflow || (lo > hi) ||
((idx_type->_lo & offset_type->_lo) < 0 && lo >= 0) ||
((~(idx_type->_hi | offset_type->_hi)) < 0 && hi < 0)) {
// May overflow
if (!overflow) {
max_idx_expr = new (C) ConvI2LNode(max_idx_expr);
register_new_node(max_idx_expr, ctrl);
}
overflow = true;
offset = new (C) ConvI2LNode(offset);
register_new_node(offset, ctrl);
max_idx_expr = new (C) AddLNode(max_idx_expr, offset);
} else {
// No overflow possible
max_idx_expr = new (C) AddINode(max_idx_expr, offset);
}
register_new_node(max_idx_expr, ctrl);
}
CmpNode* cmp = NULL;
if (overflow) {
// Integer expressions may overflow, do long comparison
range = new (C) ConvI2LNode(range);
register_new_node(range, ctrl);
if (!Matcher::has_match_rule(Op_CmpUL)) {
// We don't support unsigned long comparisons. Set 'max_idx_expr'
// to max_julong if < 0 to make the signed comparison fail.
ConINode* sign_pos = _igvn.intcon(BitsPerLong - 1);
set_ctrl(sign_pos, C->root());
Node* sign_bit_mask = new (C) RShiftLNode(max_idx_expr, sign_pos);
register_new_node(sign_bit_mask, ctrl);
// OR with sign bit to set all bits to 1 if negative (otherwise no change)
max_idx_expr = new (C) OrLNode(max_idx_expr, sign_bit_mask);
register_new_node(max_idx_expr, ctrl);
// AND with 0x7ff... to unset the sign bit
ConLNode* remove_sign_mask = _igvn.longcon(max_jlong);
set_ctrl(remove_sign_mask, C->root());
max_idx_expr = new (C) AndLNode(max_idx_expr, remove_sign_mask);
register_new_node(max_idx_expr, ctrl);
cmp = new (C) CmpLNode(max_idx_expr, range);
} else {
cmp = new (C) CmpULNode(max_idx_expr, range);
}
} else {
cmp = new (C) CmpUNode(max_idx_expr, range);
}
register_new_node(cmp, ctrl);
BoolNode* bol = new (C) BoolNode(cmp, BoolTest::lt);
register_new_node(bol, ctrl);
if (TraceLoopPredicate) {
predString->print_cr("<u range");
tty->print("%s", predString->as_string());
}
return bol;
}
//------------------------------ loop_predication_impl--------------------------
// Insert loop predicates for null checks and range checks
bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
if (!UseLoopPredicate) return false;
if (!loop->_head->is_Loop()) {
// Could be a simple region when irreducible loops are present.
return false;
}
LoopNode* head = loop->_head->as_Loop();
if (head->unique_ctrl_out()->Opcode() == Op_NeverBranch) {
// do nothing for infinite loops
return false;
}
CountedLoopNode *cl = NULL;
if (head->is_valid_counted_loop()) {
cl = head->as_CountedLoop();
// do nothing for iteration-splitted loops
if (!cl->is_normal_loop()) return false;
// Avoid RCE if Counted loop's test is '!='.
BoolTest::mask bt = cl->loopexit()->test_trip();
if (bt != BoolTest::lt && bt != BoolTest::gt)
cl = NULL;
}
Node* entry = head->in(LoopNode::EntryControl);
ProjNode *predicate_proj = NULL;
// Loop limit check predicate should be near the loop.
if (LoopLimitCheck) {
predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate_proj != NULL)
entry = predicate_proj->in(0)->in(0);
}
predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (!predicate_proj) {
#ifndef PRODUCT
if (TraceLoopPredicate) {
tty->print("missing predicate:");
loop->dump_head();
head->dump(1);
}
#endif
return false;
}
ConNode* zero = _igvn.intcon(0);
set_ctrl(zero, C->root());
ResourceArea *area = Thread::current()->resource_area();
Invariance invar(area, loop);
// Create list of if-projs such that a newer proj dominates all older
// projs in the list, and they all dominate loop->tail()
Node_List if_proj_list(area);
Node *current_proj = loop->tail(); //start from tail
while (current_proj != head) {
if (loop == get_loop(current_proj) && // still in the loop ?
current_proj->is_Proj() && // is a projection ?
current_proj->in(0)->Opcode() == Op_If) { // is a if projection ?
if_proj_list.push(current_proj);
}
current_proj = idom(current_proj);
}
bool hoisted = false; // true if at least one proj is promoted
while (if_proj_list.size() > 0) {
// Following are changed to nonnull when a predicate can be hoisted
ProjNode* new_predicate_proj = NULL;
ProjNode* proj = if_proj_list.pop()->as_Proj();
IfNode* iff = proj->in(0)->as_If();
if (!proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) {
if (loop->is_loop_exit(iff)) {
// stop processing the remaining projs in the list because the execution of them
// depends on the condition of "iff" (iff->in(1)).
break;
} else {
// Both arms are inside the loop. There are two cases:
// (1) there is one backward branch. In this case, any remaining proj
// in the if_proj list post-dominates "iff". So, the condition of "iff"
// does not determine the execution the remining projs directly, and we
// can safely continue.
// (2) both arms are forwarded, i.e. a diamond shape. In this case, "proj"
// does not dominate loop->tail(), so it can not be in the if_proj list.
continue;
}
}
Node* test = iff->in(1);
if (!test->is_Bool()){ //Conv2B, ...
continue;
}
BoolNode* bol = test->as_Bool();
if (invar.is_invariant(bol)) {
// Invariant test
new_predicate_proj = create_new_if_for_predicate(predicate_proj, NULL,
Deoptimization::Reason_predicate);
Node* ctrl = new_predicate_proj->in(0)->as_If()->in(0);
BoolNode* new_predicate_bol = invar.clone(bol, ctrl)->as_Bool();
// Negate test if necessary
bool negated = false;
if (proj->_con != predicate_proj->_con) {
new_predicate_bol = new (C) BoolNode(new_predicate_bol->in(1), new_predicate_bol->_test.negate());
register_new_node(new_predicate_bol, ctrl);
negated = true;
}
IfNode* new_predicate_iff = new_predicate_proj->in(0)->as_If();
_igvn.hash_delete(new_predicate_iff);
new_predicate_iff->set_req(1, new_predicate_bol);
#ifndef PRODUCT
if (TraceLoopPredicate) {
tty->print("Predicate invariant if%s: %d ", negated ? " negated" : "", new_predicate_iff->_idx);
loop->dump_head();
} else if (TraceLoopOpts) {
tty->print("Predicate IC ");
loop->dump_head();
}
#endif
} else if ((cl != NULL) && (proj->_con == predicate_proj->_con) &&
loop->is_range_check_if(iff, this, invar)) {
// Range check for counted loops
const Node* cmp = bol->in(1)->as_Cmp();
Node* idx = cmp->in(1);
assert(!invar.is_invariant(idx), "index is variant");
Node* rng = cmp->in(2);
assert(rng->Opcode() == Op_LoadRange || _igvn.type(rng)->is_int()->_lo >= 0, "must be");
assert(invar.is_invariant(rng), "range must be invariant");
int scale = 1;
Node* offset = zero;
bool ok = is_scaled_iv_plus_offset(idx, cl->phi(), &scale, &offset);
assert(ok, "must be index expression");
Node* init = cl->init_trip();
// Limit is not exact.
// Calculate exact limit here.
// Note, counted loop's test is '<' or '>'.
Node* limit = exact_limit(loop);
int stride = cl->stride()->get_int();
// Build if's for the upper and lower bound tests. The
// lower_bound test will dominate the upper bound test and all
// cloned or created nodes will use the lower bound test as
// their declared control.
ProjNode* lower_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate);
ProjNode* upper_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate);
assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate");
Node *ctrl = lower_bound_proj->in(0)->as_If()->in(0);
// Perform cloning to keep Invariance state correct since the
// late schedule will place invariant things in the loop.
rng = invar.clone(rng, ctrl);
if (offset && offset != zero) {
assert(invar.is_invariant(offset), "offset must be loop invariant");
offset = invar.clone(offset, ctrl);
}
// If predicate expressions may overflow in the integer range, longs are used.
bool overflow = false;
// Test the lower bound
Node* lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false, overflow);
IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If();
_igvn.hash_delete(lower_bound_iff);
lower_bound_iff->set_req(1, lower_bound_bol);
if (TraceLoopPredicate) tty->print_cr("lower bound check if: %d", lower_bound_iff->_idx);
// Test the upper bound
Node* upper_bound_bol = rc_predicate(loop, lower_bound_proj, scale, offset, init, limit, stride, rng, true, overflow);
IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If();
_igvn.hash_delete(upper_bound_iff);
upper_bound_iff->set_req(1, upper_bound_bol);
if (TraceLoopPredicate) tty->print_cr("upper bound check if: %d", lower_bound_iff->_idx);
// Fall through into rest of the clean up code which will move
// any dependent nodes onto the upper bound test.
new_predicate_proj = upper_bound_proj;
#ifndef PRODUCT
if (TraceLoopOpts && !TraceLoopPredicate) {
tty->print("Predicate RC ");
loop->dump_head();
}
#endif
} else {
// Loop variant check (for example, range check in non-counted loop)
// with uncommon trap.
continue;
}
assert(new_predicate_proj != NULL, "sanity");
// Success - attach condition (new_predicate_bol) to predicate if
invar.map_ctrl(proj, new_predicate_proj); // so that invariance test can be appropriate
// Eliminate the old If in the loop body
dominated_by( new_predicate_proj, iff, proj->_con != new_predicate_proj->_con );
hoisted = true;
C->set_major_progress();
} // end while
#ifndef PRODUCT
// report that the loop predication has been actually performed
// for this loop
if (TraceLoopPredicate && hoisted) {
tty->print("Loop Predication Performed:");
loop->dump_head();
}
#endif
return hoisted;
}
//------------------------------loop_predication--------------------------------
// driver routine for loop predication optimization
bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) {
bool hoisted = false;
// Recursively promote predicates
if (_child) {
hoisted = _child->loop_predication( phase);
}
// self
if (!_irreducible && !tail()->is_top()) {
hoisted |= phase->loop_predication_impl(this);
}
if (_next) { //sibling
hoisted |= _next->loop_predication( phase);
}
return hoisted;
}
C:\hotspot-69087d08d473\src\share\vm/opto/loopTransform.cpp
/*
* Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "compiler/compileLog.hpp"
#include "memory/allocation.inline.hpp"
#include "opto/addnode.hpp"
#include "opto/callnode.hpp"
#include "opto/connode.hpp"
#include "opto/divnode.hpp"
#include "opto/loopnode.hpp"
#include "opto/mulnode.hpp"
#include "opto/rootnode.hpp"
#include "opto/runtime.hpp"
#include "opto/subnode.hpp"
//------------------------------is_loop_exit-----------------------------------
// Given an IfNode, return the loop-exiting projection or NULL if both
// arms remain in the loop.
Node *IdealLoopTree::is_loop_exit(Node *iff) const {
if( iff->outcnt() != 2 ) return NULL; // Ignore partially dead tests
PhaseIdealLoop *phase = _phase;
// Test is an IfNode, has 2 projections. If BOTH are in the loop
// we need loop unswitching instead of peeling.
if( !is_member(phase->get_loop( iff->raw_out(0) )) )
return iff->raw_out(0);
if( !is_member(phase->get_loop( iff->raw_out(1) )) )
return iff->raw_out(1);
return NULL;
}
//=============================================================================
//------------------------------record_for_igvn----------------------------
// Put loop body on igvn work list
void IdealLoopTree::record_for_igvn() {
for( uint i = 0; i < _body.size(); i++ ) {
Node *n = _body.at(i);
_phase->_igvn._worklist.push(n);
}
}
//------------------------------compute_exact_trip_count-----------------------
// Compute loop exact trip count if possible. Do not recalculate trip count for
// split loops (pre-main-post) which have their limits and inits behind Opaque node.
void IdealLoopTree::compute_exact_trip_count( PhaseIdealLoop *phase ) {
if (!_head->as_Loop()->is_valid_counted_loop()) {
return;
}
CountedLoopNode* cl = _head->as_CountedLoop();
// Trip count may become nonexact for iteration split loops since
// RCE modifies limits. Note, _trip_count value is not reset since
// it is used to limit unrolling of main loop.
cl->set_nonexact_trip_count();
// Loop's test should be part of loop.
if (!phase->is_member(this, phase->get_ctrl(cl->loopexit()->in(CountedLoopEndNode::TestValue))))
return; // Infinite loop
#ifdef ASSERT
BoolTest::mask bt = cl->loopexit()->test_trip();
assert(bt == BoolTest::lt || bt == BoolTest::gt ||
bt == BoolTest::ne, "canonical test is expected");
#endif
Node* init_n = cl->init_trip();
Node* limit_n = cl->limit();
if (init_n != NULL && init_n->is_Con() &&
limit_n != NULL && limit_n->is_Con()) {
// Use longs to avoid integer overflow.
int stride_con = cl->stride_con();
jlong init_con = cl->init_trip()->get_int();
jlong limit_con = cl->limit()->get_int();
int stride_m = stride_con - (stride_con > 0 ? 1 : -1);
jlong trip_count = (limit_con - init_con + stride_m)/stride_con;
if (trip_count > 0 && (julong)trip_count < (julong)max_juint) {
// Set exact trip count.
cl->set_exact_trip_count((uint)trip_count);
}
}
}
//------------------------------compute_profile_trip_cnt----------------------------
// Compute loop trip count from profile data as
// (backedge_count + loop_exit_count) / loop_exit_count
void IdealLoopTree::compute_profile_trip_cnt( PhaseIdealLoop *phase ) {
if (!_head->is_CountedLoop()) {
return;
}
CountedLoopNode* head = _head->as_CountedLoop();
if (head->profile_trip_cnt() != COUNT_UNKNOWN) {
return; // Already computed
}
float trip_cnt = (float)max_jint; // default is big
Node* back = head->in(LoopNode::LoopBackControl);
while (back != head) {
if ((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
back->in(0) &&
back->in(0)->is_If() &&
back->in(0)->as_If()->_fcnt != COUNT_UNKNOWN &&
back->in(0)->as_If()->_prob != PROB_UNKNOWN) {
break;
}
back = phase->idom(back);
}
if (back != head) {
assert((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
back->in(0), "if-projection exists");
IfNode* back_if = back->in(0)->as_If();
float loop_back_cnt = back_if->_fcnt * back_if->_prob;
// Now compute a loop exit count
float loop_exit_cnt = 0.0f;
for( uint i = 0; i < _body.size(); i++ ) {
Node *n = _body[i];
if( n->is_If() ) {
IfNode *iff = n->as_If();
if( iff->_fcnt != COUNT_UNKNOWN && iff->_prob != PROB_UNKNOWN ) {
Node *exit = is_loop_exit(iff);
if( exit ) {
float exit_prob = iff->_prob;
if (exit->Opcode() == Op_IfFalse) exit_prob = 1.0 - exit_prob;
if (exit_prob > PROB_MIN) {
float exit_cnt = iff->_fcnt * exit_prob;
loop_exit_cnt += exit_cnt;
}
}
}
}
}
if (loop_exit_cnt > 0.0f) {
trip_cnt = (loop_back_cnt + loop_exit_cnt) / loop_exit_cnt;
} else {
// No exit count so use
trip_cnt = loop_back_cnt;
}
}
#ifndef PRODUCT
if (TraceProfileTripCount) {
tty->print_cr("compute_profile_trip_cnt lp: %d cnt: %f\n", head->_idx, trip_cnt);
}
#endif
head->set_profile_trip_cnt(trip_cnt);
}
//---------------------is_invariant_addition-----------------------------
// Return nonzero index of invariant operand for an Add or Sub
// of (nonconstant) invariant and variant values. Helper for reassociate_invariants.
int IdealLoopTree::is_invariant_addition(Node* n, PhaseIdealLoop *phase) {
int op = n->Opcode();
if (op == Op_AddI || op == Op_SubI) {
bool in1_invar = this->is_invariant(n->in(1));
bool in2_invar = this->is_invariant(n->in(2));
if (in1_invar && !in2_invar) return 1;
if (!in1_invar && in2_invar) return 2;
}
return 0;
}
//---------------------reassociate_add_sub-----------------------------
// Reassociate invariant add and subtract expressions:
//
// inv1 + (x + inv2) => ( inv1 + inv2) + x
// (x + inv2) + inv1 => ( inv1 + inv2) + x
// inv1 + (x - inv2) => ( inv1 - inv2) + x
// inv1 - (inv2 - x) => ( inv1 - inv2) + x
// (x + inv2) - inv1 => (-inv1 + inv2) + x
// (x - inv2) + inv1 => ( inv1 - inv2) + x
// (x - inv2) - inv1 => (-inv1 - inv2) + x
// inv1 + (inv2 - x) => ( inv1 + inv2) - x
// inv1 - (x - inv2) => ( inv1 + inv2) - x
// (inv2 - x) + inv1 => ( inv1 + inv2) - x
// (inv2 - x) - inv1 => (-inv1 + inv2) - x
// inv1 - (x + inv2) => ( inv1 - inv2) - x
//
Node* IdealLoopTree::reassociate_add_sub(Node* n1, PhaseIdealLoop *phase) {
if (!n1->is_Add() && !n1->is_Sub() || n1->outcnt() == 0) return NULL;
if (is_invariant(n1)) return NULL;
int inv1_idx = is_invariant_addition(n1, phase);
if (!inv1_idx) return NULL;
// Don't mess with add of constant (igvn moves them to expression tree root.)
if (n1->is_Add() && n1->in(2)->is_Con()) return NULL;
Node* inv1 = n1->in(inv1_idx);
Node* n2 = n1->in(3 - inv1_idx);
int inv2_idx = is_invariant_addition(n2, phase);
if (!inv2_idx) return NULL;
Node* x = n2->in(3 - inv2_idx);
Node* inv2 = n2->in(inv2_idx);
bool neg_x = n2->is_Sub() && inv2_idx == 1;
bool neg_inv2 = n2->is_Sub() && inv2_idx == 2;
bool neg_inv1 = n1->is_Sub() && inv1_idx == 2;
if (n1->is_Sub() && inv1_idx == 1) {
neg_x = !neg_x;
neg_inv2 = !neg_inv2;
}
Node* inv1_c = phase->get_ctrl(inv1);
Node* inv2_c = phase->get_ctrl(inv2);
Node* n_inv1;
if (neg_inv1) {
Node *zero = phase->_igvn.intcon(0);
phase->set_ctrl(zero, phase->C->root());
n_inv1 = new (phase->C) SubINode(zero, inv1);
phase->register_new_node(n_inv1, inv1_c);
} else {
n_inv1 = inv1;
}
Node* inv;
if (neg_inv2) {
inv = new (phase->C) SubINode(n_inv1, inv2);
} else {
inv = new (phase->C) AddINode(n_inv1, inv2);
}
phase->register_new_node(inv, phase->get_early_ctrl(inv));
Node* addx;
if (neg_x) {
addx = new (phase->C) SubINode(inv, x);
} else {
addx = new (phase->C) AddINode(x, inv);
}
phase->register_new_node(addx, phase->get_ctrl(x));
phase->_igvn.replace_node(n1, addx);
assert(phase->get_loop(phase->get_ctrl(n1)) == this, "");
_body.yank(n1);
return addx;
}
//---------------------reassociate_invariants-----------------------------
// Reassociate invariant expressions:
void IdealLoopTree::reassociate_invariants(PhaseIdealLoop *phase) {
for (int i = _body.size() - 1; i >= 0; i--) {
Node *n = _body.at(i);
for (int j = 0; j < 5; j++) {
Node* nn = reassociate_add_sub(n, phase);
if (nn == NULL) break;
n = nn; // again
};
}
}
//------------------------------policy_peeling---------------------------------
// Return TRUE or FALSE if the loop should be peeled or not. Peel if we can
// make some loop-invariant test (usually a null-check) happen before the loop.
bool IdealLoopTree::policy_peeling( PhaseIdealLoop *phase ) const {
Node *test = ((IdealLoopTree*)this)->tail();
int body_size = ((IdealLoopTree*)this)->_body.size();
// Peeling does loop cloning which can result in O(N^2) node construction
if( body_size > 255 /* Prevent overflow for large body_size */
|| (body_size * body_size + phase->C->live_nodes()) > phase->C->max_node_limit() ) {
return false; // too large to safely clone
}
while( test != _head ) { // Scan till run off top of loop
if( test->is_If() ) { // Test?
Node *ctrl = phase->get_ctrl(test->in(1));
if (ctrl->is_top())
return false; // Found dead test on live IF? No peeling!
// Standard IF only has one input value to check for loop invariance
assert( test->Opcode() == Op_If || test->Opcode() == Op_CountedLoopEnd, "Check this code when new subtype is added");
// Condition is not a member of this loop?
if( !is_member(phase->get_loop(ctrl)) &&
is_loop_exit(test) )
return true; // Found reason to peel!
}
// Walk up dominators to loop _head looking for test which is
// executed on every path thru loop.
test = phase->idom(test);
}
return false;
}
//------------------------------peeled_dom_test_elim---------------------------
// If we got the effect of peeling, either by actually peeling or by making
// a pre-loop which must execute at least once, we can remove all
// loop-invariant dominated tests in the main body.
void PhaseIdealLoop::peeled_dom_test_elim( IdealLoopTree *loop, Node_List &old_new ) {
bool progress = true;
while( progress ) {
progress = false; // Reset for next iteration
Node *prev = loop->_head->in(LoopNode::LoopBackControl);//loop->tail();
Node *test = prev->in(0);
while( test != loop->_head ) { // Scan till run off top of loop
int p_op = prev->Opcode();
if( (p_op == Op_IfFalse || p_op == Op_IfTrue) &&
test->is_If() && // Test?
!test->in(1)->is_Con() && // And not already obvious?
// Condition is not a member of this loop?
!loop->is_member(get_loop(get_ctrl(test->in(1))))){
// Walk loop body looking for instances of this test
for( uint i = 0; i < loop->_body.size(); i++ ) {
Node *n = loop->_body.at(i);
if( n->is_If() && n->in(1) == test->in(1) /*&& n != loop->tail()->in(0)*/ ) {
// IfNode was dominated by version in peeled loop body
progress = true;
dominated_by( old_new[prev->_idx], n );
}
}
}
prev = test;
test = idom(test);
} // End of scan tests in loop
} // End of while( progress )
}
//------------------------------do_peeling-------------------------------------
// Peel the first iteration of the given loop.
// Step 1: Clone the loop body. The clone becomes the peeled iteration.
// The pre-loop illegally has 2 control users (old & new loops).
// Step 2: Make the old-loop fall-in edges point to the peeled iteration.
// Do this by making the old-loop fall-in edges act as if they came
// around the loopback from the prior iteration (follow the old-loop
// backedges) and then map to the new peeled iteration. This leaves
// the pre-loop with only 1 user (the new peeled iteration), but the
// peeled-loop backedge has 2 users.
// Step 3: Cut the backedge on the clone (so its not a loop) and remove the
// extra backedge user.
//
// orig
//
// stmt1
// |
// v
// loop predicate
// |
// v
// loop<----+
// | |
// stmt2 |
// | |
// v |
// if ^
// / \ |
// / \ |
// v v |
// false true |
// / \ |
// / ----+
// |
// v
// exit
//
//
// after clone loop
//
// stmt1
// |
// v
// loop predicate
// / \
// clone / \ orig
// / \
// / \
// v v
// +---->loop clone loop<----+
// | | | |
// | stmt2 clone stmt2 |
// | | | |
// | v v |
// ^ if clone If ^
// | / \ / \ |
// | / \ / \ |
// | v v v v |
// | true false false true |
// | / \ / \ |
// +---- \ / ----+
// \ /
// 1v v2
// region
// |
// v
// exit
//
//
// after peel and predicate move
//
// stmt1
// /
// /
// clone / orig
// /
// / +----------+
// / | |
// / loop predicate |
// / | |
// v v |
// TOP-->loop clone loop<----+ |
// | | | |
// stmt2 clone stmt2 | |
// | | | ^
// v v | |
// if clone If ^ |
// / \ / \ | |
// / \ / \ | |
// v v v v | |
// true false false true | |
// | \ / \ | |
// | \ / ----+ ^
// | \ / |
// | 1v v2 |
// v region |
// | | |
// | v |
// | exit |
// | |
// +--------------->-----------------+
//
//
// final graph
//
// stmt1
// |
// v
// stmt2 clone
// |
// v
// if clone
// / |
// / |
// v v
// false true
// | |
// | v
// | loop predicate
// | |
// | v
// | loop<----+
// | | |
// | stmt2 |
// | | |
// | v |
// v if ^
// | / \ |
// | / \ |
// | v v |
// | false true |
// | | \ |
// v v --+
// region
// |
// v
// exit
//
void PhaseIdealLoop::do_peeling( IdealLoopTree *loop, Node_List &old_new ) {
C->set_major_progress();
// Peeling a 'main' loop in a pre/main/post situation obfuscates the
// 'pre' loop from the main and the 'pre' can no longer have it's
// iterations adjusted. Therefore, we need to declare this loop as
// no longer a 'main' loop; it will need new pre and post loops before
// we can do further RCE.
#ifndef PRODUCT
if (TraceLoopOpts) {
tty->print("Peel ");
loop->dump_head();
}
#endif
Node* head = loop->_head;
bool counted_loop = head->is_CountedLoop();
if (counted_loop) {
CountedLoopNode *cl = head->as_CountedLoop();
assert(cl->trip_count() > 0, "peeling a fully unrolled loop");
cl->set_trip_count(cl->trip_count() - 1);
if (cl->is_main_loop()) {
cl->set_normal_loop();
#ifndef PRODUCT
if (PrintOpto && VerifyLoopOptimizations) {
tty->print("Peeling a 'main' loop; resetting to 'normal' ");
loop->dump_head();
}
#endif
}
}
Node* entry = head->in(LoopNode::EntryControl);
// Step 1: Clone the loop body. The clone becomes the peeled iteration.
// The pre-loop illegally has 2 control users (old & new loops).
clone_loop( loop, old_new, dom_depth(head) );
// Step 2: Make the old-loop fall-in edges point to the peeled iteration.
// Do this by making the old-loop fall-in edges act as if they came
// around the loopback from the prior iteration (follow the old-loop
// backedges) and then map to the new peeled iteration. This leaves
// the pre-loop with only 1 user (the new peeled iteration), but the
// peeled-loop backedge has 2 users.
Node* new_entry = old_new[head->in(LoopNode::LoopBackControl)->_idx];
_igvn.hash_delete(head);
head->set_req(LoopNode::EntryControl, new_entry);
for (DUIterator_Fast jmax, j = head->fast_outs(jmax); j < jmax; j++) {
Node* old = head->fast_out(j);
if (old->in(0) == loop->_head && old->req() == 3 && old->is_Phi()) {
Node* new_exit_value = old_new[old->in(LoopNode::LoopBackControl)->_idx];
if (!new_exit_value ) // Backedge value is ALSO loop invariant?
// Then loop body backedge value remains the same.
new_exit_value = old->in(LoopNode::LoopBackControl);
_igvn.hash_delete(old);
old->set_req(LoopNode::EntryControl, new_exit_value);
}
}
// Step 3: Cut the backedge on the clone (so its not a loop) and remove the
// extra backedge user.
Node* new_head = old_new[head->_idx];
_igvn.hash_delete(new_head);
new_head->set_req(LoopNode::LoopBackControl, C->top());
for (DUIterator_Fast j2max, j2 = new_head->fast_outs(j2max); j2 < j2max; j2++) {
Node* use = new_head->fast_out(j2);
if (use->in(0) == new_head && use->req() == 3 && use->is_Phi()) {
_igvn.hash_delete(use);
use->set_req(LoopNode::LoopBackControl, C->top());
}
}
// Step 4: Correct dom-depth info. Set to loop-head depth.
int dd = dom_depth(head);
set_idom(head, head->in(1), dd);
for (uint j3 = 0; j3 < loop->_body.size(); j3++) {
Node *old = loop->_body.at(j3);
Node *nnn = old_new[old->_idx];
if (!has_ctrl(nnn))
set_idom(nnn, idom(nnn), dd-1);
}
// Now force out all loop-invariant dominating tests. The optimizer
// finds some, but we _know_ they are all useless.
peeled_dom_test_elim(loop,old_new);
loop->record_for_igvn();
}
#define EMPTY_LOOP_SIZE 7 // number of nodes in an empty loop
//------------------------------policy_maximally_unroll------------------------
// Calculate exact loop trip count and return true if loop can be maximally
// unrolled.
bool IdealLoopTree::policy_maximally_unroll( PhaseIdealLoop *phase ) const {
CountedLoopNode *cl = _head->as_CountedLoop();
assert(cl->is_normal_loop(), "");
if (!cl->is_valid_counted_loop())
return false; // Malformed counted loop
if (!cl->has_exact_trip_count()) {
// Trip count is not exact.
return false;
}
uint trip_count = cl->trip_count();
// Note, max_juint is used to indicate unknown trip count.
assert(trip_count > 1, "one iteration loop should be optimized out already");
assert(trip_count < max_juint, "exact trip_count should be less than max_uint.");
// Real policy: if we maximally unroll, does it get too big?
// Allow the unrolled mess to get larger than standard loop
// size. After all, it will no longer be a loop.
uint body_size = _body.size();
uint unroll_limit = (uint)LoopUnrollLimit * 4;
assert( (intx)unroll_limit == LoopUnrollLimit * 4, "LoopUnrollLimit must fit in 32bits");
if (trip_count > unroll_limit || body_size > unroll_limit) {
return false;
}
// Fully unroll a loop with few iterations regardless next
// conditions since following loop optimizations will split
// such loop anyway (pre-main-post).
if (trip_count <= 3)
return true;
// Take into account that after unroll conjoined heads and tails will fold,
// otherwise policy_unroll() may allow more unrolling than max unrolling.
uint new_body_size = EMPTY_LOOP_SIZE + (body_size - EMPTY_LOOP_SIZE) * trip_count;
uint tst_body_size = (new_body_size - EMPTY_LOOP_SIZE) / trip_count + EMPTY_LOOP_SIZE;
if (body_size != tst_body_size) // Check for int overflow
return false;
if (new_body_size > unroll_limit ||
// Unrolling can result in a large amount of node construction
new_body_size >= phase->C->max_node_limit() - phase->C->live_nodes()) {
return false;
}
// Do not unroll a loop with String intrinsics code.
// String intrinsics are large and have loops.
for (uint k = 0; k < _body.size(); k++) {
Node* n = _body.at(k);
switch (n->Opcode()) {
case Op_StrComp:
case Op_StrEquals:
case Op_StrIndexOf:
case Op_EncodeISOArray:
case Op_AryEq: {
return false;
}
#if INCLUDE_RTM_OPT
case Op_FastLock:
case Op_FastUnlock: {
// Don't unroll RTM locking code because it is large.
if (UseRTMLocking) {
return false;
}
}
#endif
} // switch
}
return true; // Do maximally unroll
}
//------------------------------policy_unroll----------------------------------
// Return TRUE or FALSE if the loop should be unrolled or not. Unroll if
// the loop is a CountedLoop and the body is small enough.
bool IdealLoopTree::policy_unroll( PhaseIdealLoop *phase ) const {
CountedLoopNode *cl = _head->as_CountedLoop();
assert(cl->is_normal_loop() || cl->is_main_loop(), "");
if (!cl->is_valid_counted_loop())
return false; // Malformed counted loop
// Protect against over-unrolling.
// After split at least one iteration will be executed in pre-loop.
if (cl->trip_count() <= (uint)(cl->is_normal_loop() ? 2 : 1)) return false;
int future_unroll_ct = cl->unrolled_count() * 2;
if (future_unroll_ct > LoopMaxUnroll) return false;
// Check for initial stride being a small enough constant
if (abs(cl->stride_con()) > (1<<2)*future_unroll_ct) return false;
// Don't unroll if the next round of unrolling would push us
// over the expected trip count of the loop. One is subtracted
// from the expected trip count because the pre-loop normally
// executes 1 iteration.
if (UnrollLimitForProfileCheck > 0 &&
cl->profile_trip_cnt() != COUNT_UNKNOWN &&
future_unroll_ct > UnrollLimitForProfileCheck &&
(float)future_unroll_ct > cl->profile_trip_cnt() - 1.0) {
return false;
}
// When unroll count is greater than LoopUnrollMin, don't unroll if:
// the residual iterations are more than 10% of the trip count
// and rounds of "unroll,optimize" are not making significant progress
// Progress defined as current size less than 20% larger than previous size.
if (UseSuperWord && cl->node_count_before_unroll() > 0 &&
future_unroll_ct > LoopUnrollMin &&
(future_unroll_ct - 1) * 10.0 > cl->profile_trip_cnt() &&
1.2 * cl->node_count_before_unroll() < (double)_body.size()) {
return false;
}
Node *init_n = cl->init_trip();
Node *limit_n = cl->limit();
int stride_con = cl->stride_con();
// Non-constant bounds.
// Protect against over-unrolling when init or/and limit are not constant
// (so that trip_count's init value is maxint) but iv range is known.
if (init_n == NULL || !init_n->is_Con() ||
limit_n == NULL || !limit_n->is_Con()) {
Node* phi = cl->phi();
if (phi != NULL) {
assert(phi->is_Phi() && phi->in(0) == _head, "Counted loop should have iv phi.");
const TypeInt* iv_type = phase->_igvn.type(phi)->is_int();
int next_stride = stride_con * 2; // stride after this unroll
if (next_stride > 0) {
if (iv_type->_lo + next_stride <= iv_type->_lo || // overflow
iv_type->_lo + next_stride > iv_type->_hi) {
return false; // over-unrolling
}
} else if (next_stride < 0) {
if (iv_type->_hi + next_stride >= iv_type->_hi || // overflow
iv_type->_hi + next_stride < iv_type->_lo) {
return false; // over-unrolling
}
}
}
}
// After unroll limit will be adjusted: new_limit = limit-stride.
// Bailout if adjustment overflow.
const TypeInt* limit_type = phase->_igvn.type(limit_n)->is_int();
if (stride_con > 0 && ((limit_type->_hi - stride_con) >= limit_type->_hi) ||
stride_con < 0 && ((limit_type->_lo - stride_con) <= limit_type->_lo))
return false; // overflow
// Adjust body_size to determine if we unroll or not
uint body_size = _body.size();
// Key test to unroll loop in CRC32 java code
int xors_in_loop = 0;
// Also count ModL, DivL and MulL which expand mightly
for (uint k = 0; k < _body.size(); k++) {
Node* n = _body.at(k);
switch (n->Opcode()) {
case Op_XorI: xors_in_loop++; break; // CRC32 java code
case Op_ModL: body_size += 30; break;
case Op_DivL: body_size += 30; break;
case Op_MulL: body_size += 10; break;
case Op_StrComp:
case Op_StrEquals:
case Op_StrIndexOf:
case Op_EncodeISOArray:
case Op_AryEq: {
// Do not unroll a loop with String intrinsics code.
// String intrinsics are large and have loops.
return false;
}
#if INCLUDE_RTM_OPT
case Op_FastLock:
case Op_FastUnlock: {
// Don't unroll RTM locking code because it is large.
if (UseRTMLocking) {
return false;
}
}
#endif
} // switch
}
// Check for being too big
if (body_size > (uint)LoopUnrollLimit) {
if (xors_in_loop >= 4 && body_size < (uint)LoopUnrollLimit*4) return true;
// Normal case: loop too big
return false;
}
// Unroll once! (Each trip will soon do double iterations)
return true;
}
//------------------------------policy_align-----------------------------------
// Return TRUE or FALSE if the loop should be cache-line aligned. Gather the
// expression that does the alignment. Note that only one array base can be
// aligned in a loop (unless the VM guarantees mutual alignment). Note that
// if we vectorize short memory ops into longer memory ops, we may want to
// increase alignment.
bool IdealLoopTree::policy_align( PhaseIdealLoop *phase ) const {
return false;
}
//------------------------------policy_range_check-----------------------------
// Return TRUE or FALSE if the loop should be range-check-eliminated.
// Actually we do iteration-splitting, a more powerful form of RCE.
bool IdealLoopTree::policy_range_check( PhaseIdealLoop *phase ) const {
if (!RangeCheckElimination) return false;
CountedLoopNode *cl = _head->as_CountedLoop();
// If we unrolled with no intention of doing RCE and we later
// changed our minds, we got no pre-loop. Either we need to
// make a new pre-loop, or we gotta disallow RCE.
if (cl->is_main_no_pre_loop()) return false; // Disallowed for now.
Node *trip_counter = cl->phi();
10-31
128
128
07-08
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
