if (_child != NULL) _child->verify_tree(loop->_child, this);
if (_next != NULL) _next ->verify_tree(loop->_next, parent);
int fail = 0;
if (!Compile::current()->major_progress() && _child == NULL) {
for( uint i = 0; i < _body.size(); i++ ) {
Node *n = _body.at(i);
if (n->outcnt() == 0) continue; // Ignore dead
uint j;
for( j = 0; j < loop->_body.size(); j++ )
if( loop->_body.at(j) == n )
break;
if( j == loop->_body.size() ) { // Not found in loop body
if (Compile::current()->root()->find(n->_idx)) {
fail++;
tty->print("We have that verify does not: ");
n->dump();
}
}
}
for( uint i2 = 0; i2 < loop->_body.size(); i2++ ) {
Node *n = loop->_body.at(i2);
if (n->outcnt() == 0) continue; // Ignore dead
uint j;
for( j = 0; j < _body.size(); j++ )
if( _body.at(j) == n )
break;
if( j == _body.size() ) { // Not found in loop body
if (Compile::current()->root()->find(n->_idx)) {
fail++;
tty->print("Verify has that we do not: ");
n->dump();
}
}
}
assert( !fail, "loop body mismatch" );
}
}
#endif
void PhaseIdealLoop::set_idom(Node* d, Node* n, uint dom_depth) {
uint idx = d->_idx;
if (idx >= _idom_size) {
uint newsize = _idom_size<<1;
while( idx >= newsize ) {
newsize <<= 1;
}
_idom = REALLOC_RESOURCE_ARRAY( Node*, _idom,_idom_size,newsize);
_dom_depth = REALLOC_RESOURCE_ARRAY( uint, _dom_depth,_idom_size,newsize);
memset( _dom_depth + _idom_size, 0, (newsize - _idom_size) * sizeof(uint) );
_idom_size = newsize;
}
_idom[idx] = n;
_dom_depth[idx] = dom_depth;
}
void PhaseIdealLoop::recompute_dom_depth() {
uint no_depth_marker = C->unique();
uint i;
for (i = 0; i < _idom_size; i++) {
if (_dom_depth[i] > 0 && _idom[i] != NULL) {
_dom_depth[i] = no_depth_marker;
}
}
if (_dom_stk == NULL) {
uint init_size = C->live_nodes() / 100; // Guess that 1/100 is a reasonable initial size.
if (init_size < 10) init_size = 10;
_dom_stk = new GrowableArray<uint>(init_size);
}
for (i = 0; i < _idom_size; i++) {
uint j = i;
while (_dom_depth[j] == no_depth_marker) {
_dom_stk->push(j);
j = _idom[j]->_idx;
}
uint dd = _dom_depth[j] + 1;
while (_dom_stk->length() > 0) {
uint j = _dom_stk->pop();
_dom_depth[j] = dd;
dd++;
}
}
}
IdealLoopTree *PhaseIdealLoop::sort( IdealLoopTree *loop, IdealLoopTree *innermost ) {
if( !innermost ) return loop; // New innermost loop
int loop_preorder = get_preorder(loop->_head); // Cache pre-order number
assert( loop_preorder, "not yet post-walked loop" );
IdealLoopTree **pp = &innermost; // Pointer to previous next-pointer
IdealLoopTree *l = *pp; // Do I go before or after 'l'?
while( l ) { // Insertion sort based on pre-order
if( l == loop ) return innermost; // Already on list!
int l_preorder = get_preorder(l->_head); // Cache pre-order number
assert( l_preorder, "not yet post-walked l" );
if( loop_preorder > l_preorder )
break; // End of insertion
if( loop_preorder == l_preorder &&
get_preorder(loop->_tail) < get_preorder(l->_tail) )
break; // Also check for shared headers (same pre#)
pp = &l->_parent; // Chain up list
l = *pp;
}
IdealLoopTree *p = loop->_parent;
loop->_parent = l; // Point me to successor
if( p ) sort( p, innermost ); // Insert my parents into list as well
return innermost;
}
void PhaseIdealLoop::build_loop_tree() {
GrowableArray <Node *> bltstack(C->live_nodes() >> 1);
Node *n = C->root();
bltstack.push(n);
int pre_order = 1;
int stack_size;
while ( ( stack_size = bltstack.length() ) != 0 ) {
n = bltstack.top(); // Leave node on stack
if ( !is_visited(n) ) {
set_preorder_visited( n, pre_order ); // set as visited
for ( int i = n->outcnt() - 1; i >= 0; --i ) {
Node* m = n->raw_out(i); // Child
if( m->is_CFG() && !is_visited(m) ) { // Only for CFG children
for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) {
Node* l = m->fast_out(j);
if( is_visited(l) && // Been visited?
!is_postvisited(l) && // But not post-visited
get_preorder(l) < pre_order ) { // And smaller pre-order
bltstack.push(m);
break;
}
}
}
}
pre_order++;
}
else if ( !is_postvisited(n) ) {
for ( int k = n->outcnt() - 1; k >= 0; --k ) {
Node* u = n->raw_out(k);
if ( u->is_CFG() && !is_visited(u) ) {
bltstack.push(u);
}
}
if ( bltstack.length() == stack_size ) {
(void)bltstack.pop(); // Remove node from stack
pre_order = build_loop_tree_impl( n, pre_order );
if (C->failing()) {
return;
}
check_grow_preorders();
}
}
else {
(void)bltstack.pop(); // Remove post-visited node from stack
}
}
}
int PhaseIdealLoop::build_loop_tree_impl( Node *n, int pre_order ) {
IdealLoopTree *innermost = NULL;
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
Node* m = n->fast_out(i); // Child
if( n == m ) continue; // Ignore control self-cycles
if( !m->is_CFG() ) continue;// Ignore non-CFG edges
IdealLoopTree *l; // Child's loop
if( !is_postvisited(m) ) { // Child visited but not post-visited?
assert( get_preorder(m) < pre_order, "should be backedge" );
if( m == C->root()) { // Found the root?
l = _ltree_root; // Root is the outermost LoopNode
} else { // Else found a nested loop
l = new IdealLoopTree(this, m, n);
} // End of Else found a nested loop
if( !has_loop(m) ) // If 'm' does not already have a loop set
set_loop(m, l); // Set loop header to loop now
} else { // Else not a nested loop
if( !_nodes[m->_idx] ) continue; // Dead code has no loop
l = get_loop(m); // Get previously determined loop
while( l && l->_head == m ) // Successor heads loop?
l = l->_parent; // Move up 1 for me
if( !l ) {
l = _ltree_root; // Oops, found infinite loop
if (!_verify_only) {
NeverBranchNode *iff = new (C) NeverBranchNode( m );
_igvn.register_new_node_with_optimizer(iff);
set_loop(iff, l);
Node *if_t = new (C) CProjNode( iff, 0 );
_igvn.register_new_node_with_optimizer(if_t);
set_loop(if_t, l);
Node* cfg = NULL; // Find the One True Control User of m
for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) {
Node* x = m->fast_out(j);
if (x->is_CFG() && x != m && x != iff)
{ cfg = x; break; }
}
assert(cfg != NULL, "must find the control user of m");
uint k = 0; // Probably cfg->in(0)
while( cfg->in(k) != m ) k++; // But check incase cfg is a Region
cfg->set_req( k, if_t ); // Now point to NeverBranch
Node *if_f = new (C) CProjNode( iff, 1 );
_igvn.register_new_node_with_optimizer(if_f);
set_loop(if_f, l);
Node *frame = new (C) ParmNode( C->start(), TypeFunc::FramePtr );
_igvn.register_new_node_with_optimizer(frame);
Node *halt = new (C) HaltNode( if_f, frame );
_igvn.register_new_node_with_optimizer(halt);
set_loop(halt, l);
C->root()->add_req(halt);
}
set_loop(C->root(), _ltree_root);
}
}
if (!_verify_only) {
while( is_postvisited(l->_head) ) {
l->_irreducible = 1; // = true
l = l->_parent;
_has_irreducible_loops = true;
if (l == NULL) {
C->record_method_not_compilable("unhandled CFG detected during loop optimization");
return pre_order;
}
}
C->set_has_irreducible_loop(_has_irreducible_loops);
}
innermost = sort( l, innermost );
}
if( innermost && innermost->_head == n ) {
assert( get_loop(n) == innermost, "" );
IdealLoopTree *p = innermost->_parent;
IdealLoopTree *l = innermost;
while( p && l->_head == n ) {
l->_next = p->_child; // Put self on parents 'next child'
p->_child = l; // Make self as first child of parent
l = p; // Now walk up the parent chain
p = l->_parent;
}
} else {
set_loop(n, innermost);
if( innermost ) {
if( n->is_Call() && !n->is_CallLeaf() && !n->is_macro() ) {
if( !n->is_CallStaticJava() || !n->as_CallStaticJava()->_name ) {
Node *iff = n->in(0)->in(0);
if( UseSuperWord || !iff->is_If() ||
(n->in(0)->Opcode() == Op_IfFalse &&
(1.0 - iff->as_If()->_prob) >= 0.01) ||
(iff->as_If()->_prob >= 0.01) )
innermost->_has_call = 1;
}
} else if( n->is_Allocate() && n->as_Allocate()->_is_scalar_replaceable ) {
innermost->_allow_optimizations = false;
innermost->_has_call = 1; // = true
} else if (n->Opcode() == Op_SafePoint) {
if (innermost->_safepts == NULL) innermost->_safepts = new Node_List();
innermost->_safepts->push(n);
}
}
}
set_postvisited(n);
return pre_order;
}
void PhaseIdealLoop::build_loop_early( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ) {
while (worklist.size() != 0) {
Node *nstack_top_n = worklist.pop();
uint nstack_top_i = 0;
while (true) {
Node *n = nstack_top_n;
uint i = nstack_top_i;
uint cnt = n->req(); // Count of inputs
if (i == 0) { // Pre-process the node.
if( has_node(n) && // Have either loop or control already?
!has_ctrl(n) ) { // Have loop picked out already?
IdealLoopTree *ilt;
while( !(ilt = get_loop(n))->_head ) {
_nodes.map(n->_idx, (Node*)(ilt->_parent) );
}
if( !_verify_only && !_verify_me && ilt->_has_sfpt && n->Opcode() == Op_SafePoint &&
is_deleteable_safept(n)) {
Node *in = n->in(TypeFunc::Control);
lazy_replace(n,in); // Pull safepoint now
if (ilt->_safepts != NULL) {
ilt->_safepts->yank(n);
}
if( !visited.test_set( in->_idx ) ) {
worklist.push(in); // Visit this guy later, using worklist
}
i = cnt + 1;
}
}
} // if (i == 0)
bool done = true; // Assume all n's inputs will be processed
while (i < cnt) {
Node *in = n->in(i);
++i;
if (in == NULL) continue;
if (in->pinned() && !in->is_CFG())
set_ctrl(in, in->in(0));
int is_visited = visited.test_set( in->_idx );
if (!has_node(in)) { // No controlling input yet?
assert( !in->is_CFG(), "CFG Node with no controlling input?" );
assert( !is_visited, "visit only once" );
nstack.push(n, i); // Save parent node and next input's index.
nstack_top_n = in; // Process current input now.
nstack_top_i = 0;
done = false; // Not all n's inputs processed.
break; // continue while_nstack_nonempty;
} else if (!is_visited) {
worklist.push(in); // Visit this guy later, using worklist
}
}
if (done) {
if (!has_node(n)) {
set_early_ctrl( n );
}
if (nstack.is_empty()) {
break;
}
nstack_top_n = nstack.node();
nstack_top_i = nstack.index();
nstack.pop();
}
} // while (true)
}
}
Node *PhaseIdealLoop::dom_lca_internal( Node *n1, Node *n2 ) const {
if( !n1 ) return n2; // Handle NULL original LCA
assert( n1->is_CFG(), "" );
assert( n2->is_CFG(), "" );
uint d1 = dom_depth(n1);
uint d2 = dom_depth(n2);
while (n1 != n2) {
if (d1 > d2) {
n1 = idom(n1);
d1 = dom_depth(n1);
} else if (d1 < d2) {
n2 = idom(n2);
d2 = dom_depth(n2);
} else {
Node *t1 = idom(n1);
while (dom_depth(t1) == d1) {
if (t1 == n2) return n2;
t1 = idom(t1);
}
Node *t2 = idom(n2);
while (dom_depth(t2) == d2) {
if (t2 == n1) return n1;
t2 = idom(t2);
}
n1 = t1;
n2 = t2;
d1 = dom_depth(n1);
d2 = dom_depth(n2);
}
}
return n1;
}
Node *PhaseIdealLoop::compute_idom( Node *region ) const {
assert( region->is_Region(), "" );
Node *LCA = NULL;
for( uint i = 1; i < region->req(); i++ ) {
if( region->in(i) != C->top() )
LCA = dom_lca( LCA, region->in(i) );
}
return LCA;
}
bool PhaseIdealLoop::verify_dominance(Node* n, Node* use, Node* LCA, Node* early) {
bool had_error = false;
#ifdef ASSERT
if (early != C->root()) {
Node* d = LCA;
while (d != early) {
if (d == C->root()) {
dump_bad_graph("Bad graph detected in compute_lca_of_uses", n, early, LCA);
tty->print_cr("*** Use %d isn't dominated by def %d ***", use->_idx, n->_idx);
had_error = true;
break;
}
d = idom(d);
}
}
#endif
return had_error;
}
Node* PhaseIdealLoop::compute_lca_of_uses(Node* n, Node* early, bool verify) {
bool had_error = false;
Node *LCA = NULL;
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && LCA != early; i++) {
Node* c = n->fast_out(i);
if (_nodes[c->_idx] == NULL)
continue; // Skip the occasional dead node
if( c->is_Phi() ) { // For Phis, we must land above on the path
for( uint j=1; j<c->req(); j++ ) {// For all inputs
if( c->in(j) == n ) { // Found matching input?
Node *use = c->in(0)->in(j);
if (_verify_only && use->is_top()) continue;
LCA = dom_lca_for_get_late_ctrl( LCA, use, n );
if (verify) had_error = verify_dominance(n, use, LCA, early) || had_error;
}
}
} else {
Node *use = has_ctrl(c) ? get_ctrl(c) : c->in(0);
LCA = dom_lca_for_get_late_ctrl( LCA, use, n );
if (verify) had_error = verify_dominance(n, use, LCA, early) || had_error;
}
}
assert(!had_error, "bad dominance");
return LCA;
}
bool PhaseIdealLoop::is_canonical_main_loop_entry(CountedLoopNode* cl) {
assert(cl->is_main_loop(), "check should be applied to main loops");
Node* ctrl = cl->in(LoopNode::EntryControl);
if (ctrl == NULL || (!ctrl->is_IfTrue() && !ctrl->is_IfFalse())) {
return false;
}
Node* iffm = ctrl->in(0);
if (iffm == NULL || !iffm->is_If()) {
return false;
}
Node* bolzm = iffm->in(1);
if (bolzm == NULL || !bolzm->is_Bool()) {
return false;
}
Node* cmpzm = bolzm->in(1);
if (cmpzm == NULL || !cmpzm->is_Cmp()) {
return false;
}
Node* opqzm = cmpzm->in(2);
if (opqzm == NULL || opqzm->Opcode() != Op_Opaque1) {
return false;
}
return true;
}
Node *PhaseIdealLoop::get_late_ctrl( Node *n, Node *early ) {
assert(early != NULL, "early control should not be NULL");
Node* LCA = compute_lca_of_uses(n, early);
#ifdef ASSERT
if (LCA == C->root() && LCA != early) {
compute_lca_of_uses(n, early, true);
}
#endif
if (n->is_Load() && LCA != early) {
Node_List worklist;
Node *mem = n->in(MemNode::Memory);
for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
Node* s = mem->fast_out(i);
worklist.push(s);
}
while(worklist.size() != 0 && LCA != early) {
Node* s = worklist.pop();
if (s->is_Load()) {
continue;
} else if (s->is_MergeMem()) {
for (DUIterator_Fast imax, i = s->fast_outs(imax); i < imax; i++) {
Node* s1 = s->fast_out(i);
worklist.push(s1);
}
} else {
Node *sctrl = has_ctrl(s) ? get_ctrl(s) : s->in(0);
assert(sctrl != NULL || s->outcnt() == 0, "must have control");
if (sctrl != NULL && !sctrl->is_top() && is_dominator(early, sctrl)) {
LCA = dom_lca_for_get_late_ctrl(LCA, sctrl, n);
}
}
}
}
assert(LCA == find_non_split_ctrl(LCA), "unexpected late control");
return LCA;
}
bool PhaseIdealLoop::is_dominator(Node *d, Node *n) {
if (d == n)
return true;
assert(d->is_CFG() && n->is_CFG(), "must have CFG nodes");
uint dd = dom_depth(d);
while (dom_depth(n) >= dd) {
if (n == d)
return true;
n = idom(n);
}
return false;
}
Node *PhaseIdealLoop::dom_lca_for_get_late_ctrl_internal( Node *n1, Node *n2, Node *tag ) {
uint d1 = dom_depth(n1);
uint d2 = dom_depth(n2);
do {
if (d1 > d2) {
_dom_lca_tags.map(n1->_idx, tag);
n1 = idom(n1);
d1 = dom_depth(n1);
} else if (d1 < d2) {
Node *memo = _dom_lca_tags[n2->_idx];
if( memo == tag ) {
return n1; // Return the current LCA
}
_dom_lca_tags.map(n2->_idx, tag);
n2 = idom(n2);
d2 = dom_depth(n2);
} else {
_dom_lca_tags.map(n1->_idx, tag);
Node *t1 = idom(n1);
while (dom_depth(t1) == d1) {
if (t1 == n2) return n2;
_dom_lca_tags.map(t1->_idx, tag);
t1 = idom(t1);
}
_dom_lca_tags.map(n2->_idx, tag);
Node *t2 = idom(n2);
while (dom_depth(t2) == d2) {
if (t2 == n1) return n1;
_dom_lca_tags.map(t2->_idx, tag);
t2 = idom(t2);
}
n1 = t1;
n2 = t2;
d1 = dom_depth(n1);
d2 = dom_depth(n2);
}
} while (n1 != n2);
return n1;
}
void PhaseIdealLoop::init_dom_lca_tags() {
uint limit = C->unique() + 1;
_dom_lca_tags.map( limit, NULL );
#ifdef ASSERT
for( uint i = 0; i < limit; ++i ) {
assert(_dom_lca_tags[i] == NULL, "Must be distinct from each node pointer");
}
#endif // ASSERT
}
void PhaseIdealLoop::clear_dom_lca_tags() {
uint limit = C->unique() + 1;
_dom_lca_tags.map( limit, NULL );
_dom_lca_tags.clear();
#ifdef ASSERT
for( uint i = 0; i < limit; ++i ) {
assert(_dom_lca_tags[i] == NULL, "Must be distinct from each node pointer");
}
#endif // ASSERT
}
void PhaseIdealLoop::build_loop_late( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ) {
while (worklist.size() != 0) {
Node *n = worklist.pop();
if (visited.test_set(n->_idx)) continue;
uint cnt = n->outcnt();
uint i = 0;
while (true) {
assert( _nodes[n->_idx], "no dead nodes" );
if (i < cnt) {
Node* use = n->raw_out(i);
++i;
if( _nodes[use->_idx] != NULL || use->is_top() ) { // Not dead?
if( use->in(0) && (use->is_CFG() || use->is_Phi()) ) {
if( !visited.test(use->_idx) )
worklist.push(use);
} else if( !visited.test_set(use->_idx) ) {
nstack.push(n, i); // Save parent and next use's index.
n = use; // Process all children of current use.
cnt = use->outcnt();
i = 0;
}
} else {
_deadlist.push(use);
}
} else {
build_loop_late_post(n);
if (nstack.is_empty()) {
break;
}
n = nstack.node();
cnt = n->outcnt();
i = nstack.index();
nstack.pop();
}
}
}
}
void PhaseIdealLoop::build_loop_late_post( Node *n ) {
if (n->req() == 2 && n->Opcode() == Op_ConvI2L && !C->major_progress() && !_verify_only) {
_igvn._worklist.push(n); // Maybe we'll normalize it, if no more loops.
}
#ifdef ASSERT
if (_verify_only && !n->is_CFG()) {
compute_lca_of_uses(n, get_ctrl(n), true /* verify */);
}
#endif
if( n->in(0) ) {
if( n->in(0)->is_top() ) return; // Dead?
bool pinned = true;
switch( n->Opcode() ) {
case Op_DivI:
case Op_DivF:
case Op_DivD:
case Op_ModI:
case Op_ModF:
case Op_ModD:
case Op_LoadB: // Same with Loads; they can sink
case Op_LoadUB: // during loop optimizations.
case Op_LoadUS:
case Op_LoadD:
case Op_LoadF:
case Op_LoadI:
case Op_LoadKlass:
case Op_LoadNKlass:
case Op_LoadL:
case Op_LoadS:
case Op_LoadP:
case Op_LoadN:
case Op_LoadRange:
case Op_LoadD_unaligned:
case Op_LoadL_unaligned:
case Op_StrComp: // Does a bunch of load-like effects
case Op_StrEquals:
case Op_StrIndexOf:
case Op_AryEq:
pinned = false;
}
if( pinned ) {
IdealLoopTree *chosen_loop = get_loop(n->is_CFG() ? n : get_ctrl(n));
if( !chosen_loop->_child ) // Inner loop?
chosen_loop->_body.push(n); // Collect inner loops
return;
}
} else { // No slot zero
if( n->is_CFG() ) { // CFG with no slot 0 is dead
_nodes.map(n->_idx,0); // No block setting, it's globally dead
return;
}
assert(!n->is_CFG() || n->outcnt() == 0, "");
}
Node *early = get_ctrl(n);// Early location already computed
Node *LCA = get_late_ctrl( n, early );
if( LCA == NULL ) {
#ifdef ASSERT
for (DUIterator i1 = n->outs(); n->has_out(i1); i1++) {
assert( _nodes[n->out(i1)->_idx] == NULL, "all uses must also be dead");
}
#endif
_nodes.map(n->_idx, 0); // This node is useless
_deadlist.push(n);
return;
}
assert(LCA != NULL && !LCA->is_top(), "no dead nodes");
Node *legal = LCA; // Walk 'legal' up the IDOM chain
Node *least = legal; // Best legal position so far
while( early != legal ) { // While not at earliest legal
#ifdef ASSERT
if (legal->is_Start() && !early->is_Root()) {
dump_bad_graph("Bad graph detected in build_loop_late", n, early, LCA);
assert(false, "Bad graph detected in build_loop_late");
}
#endif
legal = idom(legal); // Bump up the IDOM tree
if( get_loop(legal)->_nest < get_loop(least)->_nest )
least = legal;
}
assert(early == legal || legal != C->root(), "bad dominance of inputs");
if (least != early) {
Node* ctrl_out = least->unique_ctrl_out();
if (ctrl_out && ctrl_out->is_CountedLoop() &&
least == ctrl_out->in(LoopNode::EntryControl)) {
Node* least_dom = idom(least);
if (get_loop(least_dom)->is_member(get_loop(least))) {
least = least_dom;
}
}
}
#ifdef ASSERT
if( _verify_me ) {
Node *v_ctrl = _verify_me->get_ctrl_no_update(n);
Node *legal = LCA;
while( early != legal ) { // While not at earliest legal
if( legal == v_ctrl ) break; // Check for prior good location
legal = idom(legal) ;// Bump up the IDOM tree
}
if( legal == v_ctrl ) least = legal; // Keep prior if found
}
#endif
least = find_non_split_ctrl(least);
set_ctrl(n, least);
IdealLoopTree *chosen_loop = get_loop(least);
if( !chosen_loop->_child ) // Inner loop?
chosen_loop->_body.push(n);// Collect inner loops
}
#ifdef ASSERT
void PhaseIdealLoop::dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA) {
tty->print_cr("%s", msg);
tty->print("n: "); n->dump();
tty->print("early(n): "); early->dump();
if (n->in(0) != NULL && !n->in(0)->is_top() &&
n->in(0) != early && !n->in(0)->is_Root()) {
tty->print("n->in(0): "); n->in(0)->dump();
}
for (uint i = 1; i < n->req(); i++) {
Node* in1 = n->in(i);
if (in1 != NULL && in1 != n && !in1->is_top()) {
tty->print("n->in(%d): ", i); in1->dump();
Node* in1_early = get_ctrl(in1);
tty->print("early(n->in(%d)): ", i); in1_early->dump();
if (in1->in(0) != NULL && !in1->in(0)->is_top() &&
in1->in(0) != in1_early && !in1->in(0)->is_Root()) {
tty->print("n->in(%d)->in(0): ", i); in1->in(0)->dump();
}
for (uint j = 1; j < in1->req(); j++) {
Node* in2 = in1->in(j);
if (in2 != NULL && in2 != n && in2 != in1 && !in2->is_top()) {
tty->print("n->in(%d)->in(%d): ", i, j); in2->dump();
Node* in2_early = get_ctrl(in2);
tty->print("early(n->in(%d)->in(%d)): ", i, j); in2_early->dump();
if (in2->in(0) != NULL && !in2->in(0)->is_top() &&
in2->in(0) != in2_early && !in2->in(0)->is_Root()) {
tty->print("n->in(%d)->in(%d)->in(0): ", i, j); in2->in(0)->dump();
}
}
}
}
}
tty->cr();
tty->print("LCA(n): "); LCA->dump();
for (uint i = 0; i < n->outcnt(); i++) {
Node* u1 = n->raw_out(i);
if (u1 == n)
continue;
tty->print("n->out(%d): ", i); u1->dump();
if (u1->is_CFG()) {
for (uint j = 0; j < u1->outcnt(); j++) {
Node* u2 = u1->raw_out(j);
if (u2 != u1 && u2 != n && u2->is_CFG()) {
tty->print("n->out(%d)->out(%d): ", i, j); u2->dump();
}
}
} else {
Node* u1_later = get_ctrl(u1);
tty->print("later(n->out(%d)): ", i); u1_later->dump();
if (u1->in(0) != NULL && !u1->in(0)->is_top() &&
u1->in(0) != u1_later && !u1->in(0)->is_Root()) {
tty->print("n->out(%d)->in(0): ", i); u1->in(0)->dump();
}
for (uint j = 0; j < u1->outcnt(); j++) {
Node* u2 = u1->raw_out(j);
if (u2 == n || u2 == u1)
continue;
tty->print("n->out(%d)->out(%d): ", i, j); u2->dump();
if (!u2->is_CFG()) {
Node* u2_later = get_ctrl(u2);
tty->print("later(n->out(%d)->out(%d)): ", i, j); u2_later->dump();
if (u2->in(0) != NULL && !u2->in(0)->is_top() &&
u2->in(0) != u2_later && !u2->in(0)->is_Root()) {
tty->print("n->out(%d)->in(0): ", i); u2->in(0)->dump();
}
}
}
}
}
tty->cr();
int ct = 0;
Node *dbg_legal = LCA;
while(!dbg_legal->is_Start() && ct < 100) {
tty->print("idom[%d] ",ct); dbg_legal->dump();
ct++;
dbg_legal = idom(dbg_legal);
}
tty->cr();
}
#endif
#ifndef PRODUCT
void PhaseIdealLoop::dump( ) const {
ResourceMark rm;
Arena* arena = Thread::current()->resource_area();
Node_Stack stack(arena, C->live_nodes() >> 2);
Node_List rpo_list;
VectorSet visited(arena);
visited.set(C->top()->_idx);
rpo( C->root(), stack, visited, rpo_list );
dump( _ltree_root, rpo_list.size(), rpo_list );
}
void PhaseIdealLoop::dump( IdealLoopTree *loop, uint idx, Node_List &rpo_list ) const {
loop->dump_head();
for( uint j=idx; j > 0; j-- ) {
Node *n = rpo_list[j-1];
if( !_nodes[n->_idx] ) // Skip dead nodes
continue;
if( get_loop(n) != loop ) { // Wrong loop nest
if( get_loop(n)->_head == n && // Found nested loop?
get_loop(n)->_parent == loop )
dump(get_loop(n),rpo_list.size(),rpo_list); // Print it nested-ly
continue;
}
for( uint x = 0; x < loop->_nest; x++ )
tty->print(" ");
tty->print("C");
if( n == C->root() ) {
n->dump();
} else {
Node* cached_idom = idom_no_update(n);
Node *computed_idom = n->in(0);
if( n->is_Region() ) {
computed_idom = compute_idom(n);
cached_idom = find_non_split_ctrl(cached_idom);
}
tty->print(" ID:%d",computed_idom->_idx);
n->dump();
if( cached_idom != computed_idom ) {
tty->print_cr("*** BROKEN IDOM! Computed as: %d, cached as: %d",
computed_idom->_idx, cached_idom->_idx);
}
}
for( uint k = 0; k < _nodes.Size(); k++ ) {
if (k < C->unique() && _nodes[k] == (Node*)((intptr_t)n + 1)) {
Node *m = C->root()->find(k);
if( m && m->outcnt() > 0 ) {
if (!(has_ctrl(m) && get_ctrl_no_update(m) == n)) {
tty->print_cr("*** BROKEN CTRL ACCESSOR! _nodes[k] is %p, ctrl is %p",
_nodes[k], has_ctrl(m) ? get_ctrl_no_update(m) : NULL);
}
for( uint j = 0; j < loop->_nest; j++ )
tty->print(" ");
tty->print(" ");
m->dump();
}
}
}
}
}
void PhaseIdealLoop::rpo( Node *start, Node_Stack &stk, VectorSet &visited, Node_List &rpo_list ) const {
stk.push(start, 0);
visited.set(start->_idx);
while (stk.is_nonempty()) {
Node* m = stk.node();
uint idx = stk.index();
if (idx < m->outcnt()) {
stk.set_index(idx + 1);
Node* n = m->raw_out(idx);
if (n->is_CFG() && !visited.test_set(n->_idx)) {
stk.push(n, 0);
}
} else {
rpo_list.push(m);
stk.pop();
}
}
}
#endif
void LoopTreeIterator::next() {
assert(!done(), "must not be done.");
if (_curnt->_child != NULL) {
_curnt = _curnt->_child;
} else if (_curnt->_next != NULL) {
_curnt = _curnt->_next;
} else {
while (_curnt != _root && _curnt->_next == NULL) {
_curnt = _curnt->_parent;
}
if (_curnt == _root) {
_curnt = NULL;
assert(done(), "must be done.");
} else {
assert(_curnt->_next != NULL, "must be more to do");
_curnt = _curnt->_next;
}
}
}
C:\hotspot-69087d08d473\src\share\vm/opto/loopnode.hpp
#ifndef SHARE_VM_OPTO_LOOPNODE_HPP
#define SHARE_VM_OPTO_LOOPNODE_HPP
#include "opto/cfgnode.hpp"
#include "opto/multnode.hpp"
#include "opto/phaseX.hpp"
#include "opto/subnode.hpp"
#include "opto/type.hpp"
class CmpNode;
class CountedLoopEndNode;
class CountedLoopNode;
class IdealLoopTree;
class LoopNode;
class Node;
class PhaseIdealLoop;
class VectorSet;
class Invariance;
struct small_cache;
class LoopNode : public RegionNode {
virtual uint size_of() const { return sizeof(*this); }
protected:
short _loop_flags;
enum { Normal=0, Pre=1, Main=2, Post=3, PreMainPostFlagsMask=3,
MainHasNoPreLoop=4,
HasExactTripCount=8,
InnerLoop=16,
PartialPeelLoop=32,
PartialPeelFailed=64 };
char _unswitch_count;
enum { _unswitch_max=3 };
public:
enum { Self=0, EntryControl, LoopBackControl };
int is_inner_loop() const { return _loop_flags & InnerLoop; }
void set_inner_loop() { _loop_flags |= InnerLoop; }
int is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; }
void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; }
int partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; }
void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; }
int unswitch_max() { return _unswitch_max; }
int unswitch_count() { return _unswitch_count; }
void set_unswitch_count(int val) {
assert (val <= unswitch_max(), "too many unswitches");
_unswitch_count = val;
}
LoopNode( Node *entry, Node *backedge ) : RegionNode(3), _loop_flags(0), _unswitch_count(0) {
init_class_id(Class_Loop);
init_req(EntryControl, entry);
init_req(LoopBackControl, backedge);
}
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
virtual int Opcode() const;
bool can_be_counted_loop(PhaseTransform* phase) const {
return req() == 3 && in(0) != NULL &&
in(1) != NULL && phase->type(in(1)) != Type::TOP &&
in(2) != NULL && phase->type(in(2)) != Type::TOP;
}
bool is_valid_counted_loop() const;
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
class CountedLoopNode : public LoopNode {
virtual uint size_of() const { return sizeof(*this); }
node_idx_t _main_idx;
uint _trip_count;
float _profile_trip_cnt;
int _unrolled_count_log2;
int _node_count_before_unroll;
public:
CountedLoopNode( Node *entry, Node *backedge )
: LoopNode(entry, backedge), _main_idx(0), _trip_count(max_juint),
_profile_trip_cnt(COUNT_UNKNOWN), _unrolled_count_log2(0),
_node_count_before_unroll(0) {
init_class_id(Class_CountedLoop);
}
virtual int Opcode() const;
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
Node *init_control() const { return in(EntryControl); }
Node *back_control() const { return in(LoopBackControl); }
CountedLoopEndNode *loopexit() const;
Node *init_trip() const;
Node *stride() const;
int stride_con() const;
bool stride_is_con() const;
Node *limit() const;
Node *incr() const;
Node *phi() const;
static Node* match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, const TypeInt** trunc_type);
int is_normal_loop() const { return (_loop_flags&PreMainPostFlagsMask) == Normal; }
int is_pre_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Pre; }
int is_main_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Main; }
int is_post_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Post; }
int is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; }
void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; }
int main_idx() const { return _main_idx; }
void set_pre_loop (CountedLoopNode *main) { assert(is_normal_loop(),""); _loop_flags |= Pre ; _main_idx = main->_idx; }
void set_main_loop ( ) { assert(is_normal_loop(),""); _loop_flags |= Main; }
void set_post_loop (CountedLoopNode *main) { assert(is_normal_loop(),""); _loop_flags |= Post; _main_idx = main->_idx; }
void set_normal_loop( ) { _loop_flags &= ~PreMainPostFlagsMask; }
void set_trip_count(uint tc) { _trip_count = tc; }
uint trip_count() { return _trip_count; }
bool has_exact_trip_count() const { return (_loop_flags & HasExactTripCount) != 0; }
void set_exact_trip_count(uint tc) {
_trip_count = tc;
_loop_flags |= HasExactTripCount;
}
void set_nonexact_trip_count() {
_loop_flags &= ~HasExactTripCount;
}
void set_profile_trip_cnt(float ptc) { _profile_trip_cnt = ptc; }
float profile_trip_cnt() { return _profile_trip_cnt; }
void double_unrolled_count() { _unrolled_count_log2++; }
int unrolled_count() { return 1 << MIN2(_unrolled_count_log2, BitsPerInt-3); }
void set_node_count_before_unroll(int ct) { _node_count_before_unroll = ct; }
int node_count_before_unroll() { return _node_count_before_unroll; }
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
class CountedLoopEndNode : public IfNode {
public:
enum { TestControl, TestValue };
CountedLoopEndNode( Node *control, Node *test, float prob, float cnt )
: IfNode( control, test, prob, cnt) {
init_class_id(Class_CountedLoopEnd);
}
virtual int Opcode() const;
Node *cmp_node() const { return (in(TestValue)->req() >=2) ? in(TestValue)->in(1) : NULL; }
Node *incr() const { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }
Node *limit() const { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }
Node *stride() const { Node *tmp = incr (); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }
Node *init_trip() const { Node *tmp = phi (); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }
int stride_con() const;
bool stride_is_con() const { Node *tmp = stride (); return (tmp != NULL && tmp->is_Con()); }
BoolTest::mask test_trip() const { return in(TestValue)->as_Bool()->_test._test; }
PhiNode *phi() const {
Node *tmp = incr();
if (tmp && tmp->req() == 3) {
Node* phi = tmp->in(1);
if (phi->is_Phi()) {
return phi->as_Phi();
}
}
return NULL;
}
CountedLoopNode *loopnode() const {
PhiNode* iv_phi = phi();
if (iv_phi == NULL) {
return NULL;
}
assert(iv_phi->is_Phi(), "should be PhiNode");
Node *ln = iv_phi->in(0);
if (ln->is_CountedLoop() && ln->as_CountedLoop()->loopexit() == this) {
return (CountedLoopNode*)ln;
}
return NULL;
}
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
inline CountedLoopEndNode *CountedLoopNode::loopexit() const {
Node *bc = back_control();
if( bc == NULL ) return NULL;
Node *le = bc->in(0);
if( le->Opcode() != Op_CountedLoopEnd )
return NULL;
return (CountedLoopEndNode*)le;
}
inline Node *CountedLoopNode::init_trip() const { return loopexit() ? loopexit()->init_trip() : NULL; }
inline Node *CountedLoopNode::stride() const { return loopexit() ? loopexit()->stride() : NULL; }
inline int CountedLoopNode::stride_con() const { return loopexit() ? loopexit()->stride_con() : 0; }
inline bool CountedLoopNode::stride_is_con() const { return loopexit() && loopexit()->stride_is_con(); }
inline Node *CountedLoopNode::limit() const { return loopexit() ? loopexit()->limit() : NULL; }
inline Node *CountedLoopNode::incr() const { return loopexit() ? loopexit()->incr() : NULL; }
inline Node *CountedLoopNode::phi() const { return loopexit() ? loopexit()->phi() : NULL; }
class LoopLimitNode : public Node {
enum { Init=1, Limit=2, Stride=3 };
public:
LoopLimitNode( Compile* C, Node *init, Node *limit, Node *stride ) : Node(0,init,limit,stride) {
init_flags(Flag_is_macro);
C->add_macro_node(this);
}
virtual int Opcode() const;
virtual const Type *bottom_type() const { return TypeInt::INT; }
virtual uint ideal_reg() const { return Op_RegI; }
virtual const Type *Value( PhaseTransform *phase ) const;
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
virtual Node *Identity( PhaseTransform *phase );
};
class IdealLoopTree : public ResourceObj {
public:
IdealLoopTree *_parent; // Parent in loop tree
IdealLoopTree *_next; // Next sibling in loop tree
IdealLoopTree *_child; // First child in loop tree
Node *_head; // Head of loop
Node *_tail; // Tail of loop
inline Node *tail(); // Handle lazy update of _tail field
PhaseIdealLoop* _phase;
Node_List _body; // Loop body for inner loops
uint8 _nest; // Nesting depth
uint8 _irreducible:1, // True if irreducible
_has_call:1, // True if has call safepoint
_has_sfpt:1, // True if has non-call safepoint
_rce_candidate:1; // True if candidate for range check elimination
Node_List* _safepts; // List of safepoints in this loop
Node_List* _required_safept; // A inner loop cannot delete these safepts;
bool _allow_optimizations; // Allow loop optimizations
IdealLoopTree( PhaseIdealLoop* phase, Node *head, Node *tail )
: _parent(0), _next(0), _child(0),
_head(head), _tail(tail),
_phase(phase),
_safepts(NULL),
_required_safept(NULL),
_allow_optimizations(true),
_nest(0), _irreducible(0), _has_call(0), _has_sfpt(0), _rce_candidate(0)
{ }
int is_member( const IdealLoopTree *l ) const; // Test for nested membership
int set_nest( uint depth );
void split_fall_in( PhaseIdealLoop *phase, int fall_in_cnt );
void split_outer_loop( PhaseIdealLoop *phase );
void merge_many_backedges( PhaseIdealLoop *phase );
bool beautify_loops( PhaseIdealLoop *phase );
bool loop_predication( PhaseIdealLoop *phase);
bool iteration_split( PhaseIdealLoop *phase, Node_List &old_new );
bool iteration_split_impl( PhaseIdealLoop *phase, Node_List &old_new );
void check_safepts(VectorSet &visited, Node_List &stack);
void allpaths_check_safepts(VectorSet &visited, Node_List &stack);
void remove_safepoints(PhaseIdealLoop* phase, bool keep_one);
void counted_loop( PhaseIdealLoop *phase );
Node *is_loop_exit(Node *iff) const;
bool dominates_backedge(Node* ctrl);
void DCE_loop_body();
void adjust_loop_exit_prob( PhaseIdealLoop *phase );
bool policy_peel_only( PhaseIdealLoop *phase ) const;
bool policy_unswitching( PhaseIdealLoop *phase ) const;
bool policy_do_remove_empty_loop( PhaseIdealLoop *phase );
bool policy_do_one_iteration_loop( PhaseIdealLoop *phase );
bool policy_peeling( PhaseIdealLoop *phase ) const;
bool policy_maximally_unroll( PhaseIdealLoop *phase ) const;
bool policy_unroll( PhaseIdealLoop *phase ) const;
bool policy_range_check( PhaseIdealLoop *phase ) const;
bool policy_align( PhaseIdealLoop *phase ) const;
bool is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar) const;
void compute_exact_trip_count( PhaseIdealLoop *phase );
void compute_profile_trip_cnt( PhaseIdealLoop *phase );
void reassociate_invariants(PhaseIdealLoop *phase);
Node* reassociate_add_sub(Node* n1, PhaseIdealLoop *phase);
int is_invariant_addition(Node* n, PhaseIdealLoop *phase);
bool is_invariant(Node* n) const;
void record_for_igvn();
bool is_loop() { return !_irreducible && _tail && !_tail->is_top(); }
bool is_inner() { return is_loop() && _child == NULL; }
bool is_counted() { return is_loop() && _head != NULL && _head->is_CountedLoop(); }
#ifndef PRODUCT
void dump_head( ) const; // Dump loop head only
void dump() const; // Dump this loop recursively
void verify_tree(IdealLoopTree *loop, const IdealLoopTree *parent) const;
#endif
};
class PhaseIdealLoop : public PhaseTransform {
friend class IdealLoopTree;
friend class SuperWord;
PhaseIterGVN &_igvn;
IdealLoopTree *_ltree_root;
uint *_preorders;
uint _max_preorder;
const PhaseIdealLoop* _verify_me;
bool _verify_only;
void allocate_preorders() {
_max_preorder = C->unique()+8;
_preorders = NEW_RESOURCE_ARRAY(uint, _max_preorder);
memset(_preorders, 0, sizeof(uint) * _max_preorder);
}
void reallocate_preorders() {
if ( _max_preorder < C->unique() ) {
_preorders = REALLOC_RESOURCE_ARRAY(uint, _preorders, _max_preorder, C->unique());
_max_preorder = C->unique();
}
memset(_preorders, 0, sizeof(uint) * _max_preorder);
}
void check_grow_preorders( ) {
if ( _max_preorder < C->unique() ) {
uint newsize = _max_preorder<<1; // double size of array
_preorders = REALLOC_RESOURCE_ARRAY(uint, _preorders, _max_preorder, newsize);
memset(&_preorders[_max_preorder],0,sizeof(uint)*(newsize-_max_preorder));
_max_preorder = newsize;
}
}
int is_visited( Node *n ) const { return _preorders[n->_idx]; }
void set_preorder_visited( Node *n, int pre_order ) {
assert( !is_visited( n ), "already set" );
_preorders[n->_idx] = (pre_order<<1);
};
int get_preorder( Node *n ) const { assert( is_visited(n), "" ); return _preorders[n->_idx]>>1; }
int is_postvisited( Node *n ) const { assert( is_visited(n), "" ); return _preorders[n->_idx]&1; }
void set_postvisited( Node *n ) { assert( !is_postvisited( n ), "" ); _preorders[n->_idx] |= 1; }
bool has_ctrl( Node *n ) const { return ((intptr_t)_nodes[n->_idx]) & 1; }
Node_List _deadlist;
Node_Array _dom_lca_tags;
void init_dom_lca_tags();
void clear_dom_lca_tags();
bool verify_dominance(Node* n, Node* use, Node* LCA, Node* early);
Node* compute_lca_of_uses(Node* n, Node* early, bool verify = false);
Node *dom_lca_for_get_late_ctrl( Node *lca, Node *n, Node *tag ) {
assert( n->is_CFG(), "" );
if( lca != NULL && lca != n ) {
assert( lca->is_CFG(), "" );
n = dom_lca_for_get_late_ctrl_internal( lca, n, tag );
}
return find_non_split_ctrl(n);
}
Node *dom_lca_for_get_late_ctrl_internal( Node *lca, Node *n, Node *tag );
Node *find_non_split_ctrl( Node *ctrl ) const {
if (ctrl != NULL) {
if (ctrl->is_MultiBranch()) {
ctrl = ctrl->in(0);
}
assert(ctrl->is_CFG(), "CFG");
}
return ctrl;
}
bool cast_incr_before_loop(Node* incr, Node* ctrl, Node* loop);
public:
static bool is_canonical_main_loop_entry(CountedLoopNode* cl);
bool has_node( Node* n ) const {
guarantee(n != NULL, "No Node.");
return _nodes[n->_idx] != NULL;
}
Node *get_late_ctrl( Node *n, Node *early );
Node *get_early_ctrl( Node *n );
Node *get_early_ctrl_for_expensive(Node *n, Node* earliest);
void set_early_ctrl( Node *n );
void set_subtree_ctrl( Node *root );
void set_ctrl( Node *n, Node *ctrl ) {
assert( !has_node(n) || has_ctrl(n), "" );
assert( ctrl->in(0), "cannot set dead control node" );
assert( ctrl == find_non_split_ctrl(ctrl), "must set legal crtl" );
_nodes.map( n->_idx, (Node*)((intptr_t)ctrl + 1) );
}
void set_ctrl_and_loop(Node* n, Node* ctrl) {
IdealLoopTree* old_loop = get_loop(get_ctrl(n));
IdealLoopTree* new_loop = get_loop(ctrl);
if (old_loop != new_loop) {
if (old_loop->_child == NULL) old_loop->_body.yank(n);
if (new_loop->_child == NULL) new_loop->_body.push(n);
}
set_ctrl(n, ctrl);
}
Node *get_ctrl( Node *i ) {
assert(has_node(i), "");
Node *n = get_ctrl_no_update(i);
_nodes.map( i->_idx, (Node*)((intptr_t)n + 1) );
assert(has_node(i) && has_ctrl(i), "");
assert(n == find_non_split_ctrl(n), "must return legal ctrl" );
return n;
}
bool is_dominator(Node *d, Node *n);
Node* ctrl_or_self(Node* n) {
if (has_ctrl(n))
return get_ctrl(n);
else {
assert (n->is_CFG(), "must be a CFG node");
return n;
}
}
private:
Node *get_ctrl_no_update_helper(Node *i) const {
assert(has_ctrl(i), "should be control, not loop");
return (Node*)(((intptr_t)_nodes[i->_idx]) & ~1);
}
Node *get_ctrl_no_update(Node *i) const {
assert( has_ctrl(i), "" );
Node *n = get_ctrl_no_update_helper(i);
if (!n->in(0)) {
do {
n = get_ctrl_no_update_helper(n);
} while (!n->in(0));
n = find_non_split_ctrl(n);
}
return n;
}
bool has_loop( Node *n ) const {
assert(!has_node(n) || !has_ctrl(n), "");
return has_node(n);
}
void set_loop( Node *n, IdealLoopTree *loop ) {
_nodes.map(n->_idx, (Node*)loop);
}
public:
void lazy_update(Node *old_node, Node *new_node) {
assert(old_node != new_node, "no cycles please");
_nodes.map(old_node->_idx, (Node*)((intptr_t)new_node + 1));
}
void lazy_replace(Node *old_node, Node *new_node) {
_igvn.replace_node(old_node, new_node);
lazy_update(old_node, new_node);
}
private:
void build_loop_tree();
int build_loop_tree_impl( Node *n, int pre_order );
IdealLoopTree *sort( IdealLoopTree *loop, IdealLoopTree *innermost );
void build_loop_early( VectorSet &visited, Node_List &worklist, Node_Stack &nstack );
void build_loop_late ( VectorSet &visited, Node_List &worklist, Node_Stack &nstack );
void build_loop_late_post ( Node* n );
private:
uint _idom_size;
Node **_idom; // Array of immediate dominators
uint *_dom_depth; // Used for fast LCA test
GrowableArray<uint>* _dom_stk; // For recomputation of dom depth
Node* idom_no_update(Node* d) const {
assert(d->_idx < _idom_size, "oob");
Node* n = _idom[d->_idx];
assert(n != NULL,"Bad immediate dominator info.");
while (n->in(0) == NULL) { // Skip dead CFG nodes
n = (Node*)(((intptr_t)_nodes[n->_idx]) & ~1);
assert(n != NULL,"Bad immediate dominator info.");
}
return n;
}
Node *idom(Node* d) const {
uint didx = d->_idx;
Node *n = idom_no_update(d);
_idom[didx] = n; // Lazily remove dead CFG nodes from table.
return n;
}
uint dom_depth(Node* d) const {
guarantee(d != NULL, "Null dominator info.");
guarantee(d->_idx < _idom_size, "");
return _dom_depth[d->_idx];
}
void set_idom(Node* d, Node* n, uint dom_depth);
Node *compute_idom( Node *region ) const;
void recompute_dom_depth();
bool is_deleteable_safept(Node* sfpt);
void replace_parallel_iv(IdealLoopTree *loop);
PhaseIdealLoop( PhaseIterGVN &igvn) :
PhaseTransform(Ideal_Loop),
_igvn(igvn),
_dom_lca_tags(arena()), // Thread::resource_area
_verify_me(NULL),
_verify_only(true) {
build_and_optimize(false, false);
}
void build_and_optimize(bool do_split_if, bool skip_loop_opts);
public:
void Dominators();
Node *dom_lca( Node *n1, Node *n2 ) const {
return find_non_split_ctrl(dom_lca_internal(n1, n2));
}
Node *dom_lca_internal( Node *n1, Node *n2 ) const;
PhaseIdealLoop( PhaseIterGVN &igvn, bool do_split_ifs, bool skip_loop_opts = false) :
PhaseTransform(Ideal_Loop),
_igvn(igvn),
_dom_lca_tags(arena()), // Thread::resource_area
_verify_me(NULL),
_verify_only(false) {
build_and_optimize(do_split_ifs, skip_loop_opts);
}
PhaseIdealLoop( PhaseIterGVN &igvn, const PhaseIdealLoop *verify_me) :
PhaseTransform(Ideal_Loop),
_igvn(igvn),
_dom_lca_tags(arena()), // Thread::resource_area
_verify_me(verify_me),
_verify_only(false) {
build_and_optimize(false, false);
}
static void verify(PhaseIterGVN& igvn) {
#ifdef ASSERT
PhaseIdealLoop v(igvn);
#endif
}
bool _has_irreducible_loops;
virtual Node *transform( Node *a_node ) { return 0; }
bool is_counted_loop( Node *x, IdealLoopTree *loop );
Node* exact_limit( IdealLoopTree *loop );
IdealLoopTree *get_loop( Node *n ) const {
if (!has_node(n)) return _ltree_root;
assert(!has_ctrl(n), "");
return (IdealLoopTree*)_nodes[n->_idx];
}
int is_member( const IdealLoopTree *loop, Node *n ) const {
return loop->is_member(get_loop(n)); }
void clone_loop( IdealLoopTree *loop, Node_List &old_new, int dom_depth,
Node* side_by_side_idom = NULL);
void peeled_dom_test_elim( IdealLoopTree *loop, Node_List &old_new );
void do_peeling( IdealLoopTree *loop, Node_List &old_new );
void insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_new, bool peel_only );
Node *clone_up_backedge_goo( Node *back_ctrl, Node *preheader_ctrl, Node *n, VectorSet &visited, Node_Stack &clones );
void do_maximally_unroll( IdealLoopTree *loop, Node_List &old_new );
void do_unroll( IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip );
bool is_scaled_iv(Node* exp, Node* iv, int* p_scale);
bool is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset, int depth = 0);
ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
Deoptimization::DeoptReason reason);
void register_control(Node* n, IdealLoopTree *loop, Node* pred);
static ProjNode* clone_predicate(ProjNode* predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason,
PhaseIdealLoop* loop_phase,
PhaseIterGVN* igvn);
static Node* clone_loop_predicates(Node* old_entry, Node* new_entry,
bool clone_limit_check,
PhaseIdealLoop* loop_phase,
PhaseIterGVN* igvn);
Node* clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check);
static Node* skip_loop_predicates(Node* entry);
static ProjNode* find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason);
static Node* find_predicate(Node* entry);
BoolNode* rc_predicate(IdealLoopTree *loop, Node* ctrl,
int scale, Node* offset,
Node* init, Node* limit, jint stride,
Node* range, bool upper, bool &overflow);
bool loop_predication_impl(IdealLoopTree *loop);
void collect_potentially_useful_predicates(IdealLoopTree *loop, Unique_Node_List &predicate_opaque1);
void eliminate_useless_predicates();
bool process_expensive_nodes();
bool is_node_unreachable(Node *n) const {
return !has_node(n) || n->is_unreachable(_igvn);
}
void do_range_check( IdealLoopTree *loop, Node_List &old_new );
ProjNode* create_slow_version_of_loop(IdealLoopTree *loop,
Node_List &old_new);
void do_unswitching (IdealLoopTree *loop, Node_List &old_new);
IfNode* find_unswitching_candidate(const IdealLoopTree *loop) const;
void add_constraint(jlong stride_con, jlong scale_con, Node* offset, Node* low_limit, Node* upper_limit, Node* pre_ctrl, Node** pre_limit, Node** main_limit);
Node* adjust_limit(bool reduce, Node* scale, Node* offset, Node* rc_limit, Node* old_limit, Node* pre_ctrl, bool round);
bool partial_peel( IdealLoopTree *loop, Node_List &old_new );
void scheduled_nodelist( IdealLoopTree *loop, VectorSet& ctrl, Node_List &sched );
bool has_use_in_set( Node* n, VectorSet& vset );
bool has_use_internal_to_set( Node* n, VectorSet& vset, IdealLoopTree *loop );
int clone_for_use_outside_loop( IdealLoopTree *loop, Node* n, Node_List& worklist );
void clone_for_special_use_inside_loop( IdealLoopTree *loop, Node* n,
VectorSet& not_peel, Node_List& sink_list, Node_List& worklist );
void insert_phi_for_loop( Node* use, uint idx, Node* lp_entry_val, Node* back_edge_val, LoopNode* lp );
#ifdef ASSERT
bool is_valid_loop_partition( IdealLoopTree *loop, VectorSet& peel, Node_List& peel_list, VectorSet& not_peel );
bool is_valid_clone_loop_form( IdealLoopTree *loop, Node_List& peel_list,
uint orig_exit_idx, uint clone_exit_idx);
bool is_valid_clone_loop_exit_use( IdealLoopTree *loop, Node* use, uint exit_idx);
#endif
int stride_of_possible_iv( Node* iff );
bool is_possible_iv_test( Node* iff ) { return stride_of_possible_iv(iff) != 0; }
Node* stay_in_loop( Node* n, IdealLoopTree *loop);
IfNode* insert_cmpi_loop_exit(IfNode* if_cmpu, IdealLoopTree *loop);
void remove_cmpi_loop_exit(IfNode* if_cmp, IdealLoopTree *loop);
void register_node(Node* n, IdealLoopTree *loop, Node* pred, int ddepth);
ProjNode* proj_clone(ProjNode* p, IfNode* iff);
Node* short_circuit_if(IfNode* iff, ProjNode* live_proj);
RegionNode* insert_region_before_proj(ProjNode* proj);
ProjNode* insert_if_before_proj(Node* left, bool Signed, BoolTest::mask relop, Node* right, ProjNode* proj);
BoolNode *clone_iff( PhiNode *phi, IdealLoopTree *loop );
CmpNode *clone_bool( PhiNode *phi, IdealLoopTree *loop );
Node *remix_address_expressions( Node *n );
Node *conditional_move( Node *n );
void reorg_offsets( IdealLoopTree *loop );
void split_if_with_blocks ( VectorSet &visited, Node_Stack &nstack );
Node *split_if_with_blocks_pre ( Node *n );
void split_if_with_blocks_post( Node *n );
Node *has_local_phi_input( Node *n );
void dominated_by( Node *prevdom, Node *iff, bool flip = false, bool exclude_loop_predicate = false );
Node *split_thru_region( Node *n, Node *region );
Node *split_thru_phi( Node *n, Node *region, int policy );
void do_split_if( Node *iff );
bool do_intrinsify_fill();
bool intrinsify_fill(IdealLoopTree* lpt);
bool match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& store_value,
Node*& shift, Node*& offset);
private:
const TypeInt* filtered_type( Node *n, Node* n_ctrl);
const TypeInt* filtered_type( Node *n ) { return filtered_type(n, NULL); }
const TypeInt* filtered_type_from_dominators( Node* val, Node *val_ctrl);
Node *spinup( Node *iff, Node *new_false, Node *new_true, Node *region, Node *phi, small_cache *cache );
Node *find_use_block( Node *use, Node *def, Node *old_false, Node *new_false, Node *old_true, Node *new_true );
void handle_use( Node *use, Node *def, small_cache *cache, Node *region_dom, Node *new_false, Node *new_true, Node *old_false, Node *old_true );
bool split_up( Node *n, Node *blk1, Node *blk2 );
void sink_use( Node *use, Node *post_loop );
Node *place_near_use( Node *useblock ) const;
bool _created_loop_node;
public:
void set_created_loop_node() { _created_loop_node = true; }
bool created_loop_node() { return _created_loop_node; }
void register_new_node( Node *n, Node *blk );
#ifdef ASSERT
void dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA);
#endif
#ifndef PRODUCT
void dump( ) const;
void dump( IdealLoopTree *loop, uint rpo_idx, Node_List &rpo_list ) const;
void rpo( Node *start, Node_Stack &stk, VectorSet &visited, Node_List &rpo_list ) const;
void verify() const; // Major slow :-)
void verify_compare( Node *n, const PhaseIdealLoop *loop_verify, VectorSet &visited ) const;
IdealLoopTree *get_loop_idx(Node* n) const {
return _nodes[n->_idx] ? (IdealLoopTree*)_nodes[n->_idx] : _ltree_root;
}
static void print_statistics();
static int _loop_invokes; // Count of PhaseIdealLoop invokes
static int _loop_work; // Sum of PhaseIdealLoop x _unique
#endif
};
inline Node* IdealLoopTree::tail() {
Node *n = _tail;
if (n->in(0) == NULL)
n = _phase->get_ctrl(n);
_tail = n;
return n;
}
class LoopTreeIterator : public StackObj {
private:
IdealLoopTree* _root;
IdealLoopTree* _curnt;
public:
LoopTreeIterator(IdealLoopTree* root) : _root(root), _curnt(root) {}
bool done() { return _curnt == NULL; } // Finished iterating?
void next(); // Advance to next loop tree
IdealLoopTree* current() { return _curnt; } // Return current value of iterator.
};
#endif // SHARE_VM_OPTO_LOOPNODE_HPP
C:\hotspot-69087d08d473\src\share\vm/opto/loopopts.cpp
#include "precompiled.hpp"
#include "memory/allocation.inline.hpp"
#include "opto/addnode.hpp"
#include "opto/connode.hpp"
#include "opto/divnode.hpp"
#include "opto/loopnode.hpp"
#include "opto/matcher.hpp"
#include "opto/mulnode.hpp"
#include "opto/rootnode.hpp"
#include "opto/subnode.hpp"
Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {
if (n->Opcode() == Op_ConvI2L && n->bottom_type() != TypeLong::LONG) {
return NULL;
}
if (n->Opcode() == Op_CastII && n->as_CastII()->has_range_check() &&
region->is_CountedLoop() && n->in(1) == region->as_CountedLoop()->phi()) {
return NULL;
}
int wins = 0;
assert(!n->is_CFG(), "");
assert(region->is_Region(), "");
const Type* type = n->bottom_type();
const TypeOopPtr *t_oop = _igvn.type(n)->isa_oopptr();
Node *phi;
if (t_oop != NULL && t_oop->is_known_instance_field()) {
int iid = t_oop->instance_id();
int index = C->get_alias_index(t_oop);
int offset = t_oop->offset();
phi = new (C) PhiNode(region, type, NULL, iid, index, offset);
} else {
phi = PhiNode::make_blank(region, n);
}
uint old_unique = C->unique();
for (uint i = 1; i < region->req(); i++) {
Node *x;
Node* the_clone = NULL;
if (region->in(i) == C->top()) {
x = C->top(); // Dead path? Use a dead data op
} else {
x = n->clone(); // Else clone up the data op
the_clone = x; // Remember for possible deletion.
if (n->in(0) == region)
x->set_req( 0, region->in(i) );
for (uint j = 1; j < n->req(); j++) {
Node *in = n->in(j);
if (in->is_Phi() && in->in(0) == region)
x->set_req( j, in->in(i) ); // Use pre-Phi input for the clone
}
}
const Type *t = x->Value(&_igvn);
bool singleton = t->singleton();
if (singleton && t == Type::TOP) {
singleton &= region->is_Loop() && (i != LoopNode::EntryControl);
}
if (singleton) {
wins++;
x = ((PhaseGVN&)_igvn).makecon(t);
} else {
_igvn.set_type(x, t);
x->raise_bottom_type(t);
Node *y = x->Identity(&_igvn);
if (y != x) {
wins++;
x = y;
} else {
y = _igvn.hash_find(x);
if (y) {
wins++;
x = y;
} else {
_igvn._worklist.push(x);
}
}
}
if (x != the_clone && the_clone != NULL)
_igvn.remove_dead_node(the_clone);
phi->set_req( i, x );
}
if (wins <= policy) {
_igvn.remove_dead_node(phi);
return NULL;
}
register_new_node( phi, region );
for (uint i2 = 1; i2 < phi->req(); i2++) {
Node *x = phi->in(i2);
Node *old_ctrl;
IdealLoopTree *old_loop;
if (x->is_Con()) {
set_ctrl(x, C->root());
continue;
}
if (x->_idx >= old_unique) { // Found a new, unplaced node?
old_ctrl = NULL;
old_loop = NULL; // Not in any prior loop
} else {
old_ctrl = get_ctrl(x);
old_loop = get_loop(old_ctrl); // Get prior loop
}
Node *new_ctrl = dom_lca(old_ctrl, region->in(i2));
if (new_ctrl == old_ctrl) // Nothing is changed
continue;
IdealLoopTree *new_loop = get_loop(new_ctrl);
IdealLoopTree *use_loop = get_loop(region);
if (!new_loop->is_member(use_loop) &&
(old_loop == NULL || !new_loop->is_member(old_loop))) {
new_ctrl = get_early_ctrl(x);
new_loop = get_loop(new_ctrl);
}
set_ctrl(x, new_ctrl);
if (old_loop != new_loop) {
if (old_loop && !old_loop->_child)
old_loop->_body.yank(x);
if (!new_loop->_child)
new_loop->_body.push(x); // Collect body info
}
}
return phi;
}
void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff, bool flip, bool exclude_loop_predicate ) {
#ifndef PRODUCT
if (VerifyLoopOptimizations && PrintOpto) tty->print_cr("dominating test");
#endif
assert( iff->is_If(), "" );
assert( iff->Opcode() == Op_If || iff->Opcode() == Op_CountedLoopEnd, "Check this code when new subtype is added");
int pop = prevdom->Opcode();
assert( pop == Op_IfFalse || pop == Op_IfTrue, "" );
if (flip) {
if (pop == Op_IfTrue)
pop = Op_IfFalse;
else
pop = Op_IfTrue;
}
Node *con = _igvn.makecon(pop == Op_IfTrue ? TypeInt::ONE : TypeInt::ZERO);
set_ctrl(con, C->root()); // Constant gets a new use
_igvn.replace_input_of(iff, 1, con);
if (iff->outcnt() != 2) return;
Node* dp = iff->as_If()->proj_out(pop == Op_IfTrue);
if (dp == NULL)
return;
ProjNode* dp_proj = dp->as_Proj();
ProjNode* unc_proj = iff->as_If()->proj_out(1 - dp_proj->_con)->as_Proj();
if (exclude_loop_predicate &&
(unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_predicate) ||
unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_range_check))) {
return; // Let IGVN transformation change control dependence.
}
IdealLoopTree *old_loop = get_loop(dp);
for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) {
Node* cd = dp->fast_out(i); // Control-dependent node
if (cd->depends_only_on_test()) {
assert(cd->in(0) == dp, "");
_igvn.replace_input_of(cd, 0, prevdom);
set_early_ctrl(cd);
IdealLoopTree *new_loop = get_loop(get_ctrl(cd));
if (old_loop != new_loop) {
if (!old_loop->_child) old_loop->_body.yank(cd);
if (!new_loop->_child) new_loop->_body.push(cd);
}
--i;
--imax;
}
}
}
Node *PhaseIdealLoop::has_local_phi_input( Node *n ) {
Node *n_ctrl = get_ctrl(n);
uint i;
for( i = 1; i < n->req(); i++ ) {
Node *phi = n->in(i);
if( phi->is_Phi() && phi->in(0) == n_ctrl )
break;
}
if( i >= n->req() )
return NULL; // No Phi inputs; nowhere to clone thru
for( i = 1; i < n->req(); i++ ) {
Node *m = n->in(i);
if( get_ctrl(m) == n_ctrl && !m->is_Phi() ) {
if (m->is_AddP() &&
get_ctrl(m->in(2)) != n_ctrl &&
get_ctrl(m->in(3)) != n_ctrl) {
set_ctrl_and_loop(m, find_non_split_ctrl(idom(n_ctrl)));
continue;
}
return NULL;
}
assert(n->is_Phi() || m->is_Phi() || is_dominator(get_ctrl(m), n_ctrl), "m has strange control");
}
return n_ctrl;
}
Node *PhaseIdealLoop::remix_address_expressions( Node *n ) {
if (!has_ctrl(n)) return NULL;
Node *n_ctrl = get_ctrl(n);
IdealLoopTree *n_loop = get_loop(n_ctrl);
if( n->req() < 3 || n->req() > 4 ) return NULL;
Node *n1_ctrl = get_ctrl(n->in( 1));
Node *n2_ctrl = get_ctrl(n->in( 2));
Node *n3_ctrl = get_ctrl(n->in(n->req() == 3 ? 2 : 3));
IdealLoopTree *n1_loop = get_loop( n1_ctrl );
IdealLoopTree *n2_loop = get_loop( n2_ctrl );
IdealLoopTree *n3_loop = get_loop( n3_ctrl );
if( (n_loop->is_member( n1_loop ) && n_loop != n1_loop) ||
(n_loop->is_member( n2_loop ) && n_loop != n2_loop) ||
(n_loop->is_member( n3_loop ) && n_loop != n3_loop) )
return NULL; // Leave well enough alone
if( n1_loop == n_loop &&
n2_loop == n_loop &&
n3_loop == n_loop )
return NULL; // No loop-invariant inputs
int n_op = n->Opcode();
if( n_op == Op_LShiftI ) {
Node *scale = n->in(2);
Node *scale_ctrl = get_ctrl(scale);
IdealLoopTree *scale_loop = get_loop(scale_ctrl );
if( n_loop == scale_loop || !scale_loop->is_member( n_loop ) )
return NULL;
const TypeInt *scale_t = scale->bottom_type()->isa_int();
if( scale_t && scale_t->is_con() && scale_t->get_con() >= 16 )
return NULL; // Dont bother with byte/short masking
Node *add = n->in(1);
Node *add_ctrl = get_ctrl(add);
IdealLoopTree *add_loop = get_loop(add_ctrl);
if( n_loop != add_loop ) return NULL; // happens w/ evil ZKM loops
if( add->Opcode() == Op_SubI &&
_igvn.type( add->in(1) ) != TypeInt::ZERO ) {
Node *zero = _igvn.intcon(0);
set_ctrl(zero, C->root());
Node *neg = new (C) SubINode( _igvn.intcon(0), add->in(2) );
register_new_node( neg, get_ctrl(add->in(2) ) );
add = new (C) AddINode( add->in(1), neg );
register_new_node( add, add_ctrl );
}
if( add->Opcode() != Op_AddI ) return NULL;
Node *add_var = add->in(1);
Node *add_var_ctrl = get_ctrl(add_var);
IdealLoopTree *add_var_loop = get_loop(add_var_ctrl );
Node *add_invar = add->in(2);
Node *add_invar_ctrl = get_ctrl(add_invar);
IdealLoopTree *add_invar_loop = get_loop(add_invar_ctrl );
if( add_var_loop == n_loop ) {
} else if( add_invar_loop == n_loop ) {
add_invar = add_var;
add_invar_ctrl = add_var_ctrl;
add_invar_loop = add_var_loop;
add_var = add->in(2);
Node *add_var_ctrl = get_ctrl(add_var);
IdealLoopTree *add_var_loop = get_loop(add_var_ctrl );
} else // Else neither input is loop invariant
return NULL;
if( n_loop == add_invar_loop || !add_invar_loop->is_member( n_loop ) )
return NULL; // No invariant part of the add?
Node *inv_scale = new (C) LShiftINode( add_invar, scale );
Node *inv_scale_ctrl =
dom_depth(add_invar_ctrl) > dom_depth(scale_ctrl) ?
add_invar_ctrl : scale_ctrl;
register_new_node( inv_scale, inv_scale_ctrl );
Node *var_scale = new (C) LShiftINode( add_var, scale );
register_new_node( var_scale, n_ctrl );
Node *var_add = new (C) AddINode( var_scale, inv_scale );
register_new_node( var_add, n_ctrl );
_igvn.replace_node( n, var_add );
return var_add;
}
if( n_op == Op_AddI ||
n_op == Op_AddL ||
n_op == Op_AddF ||
n_op == Op_AddD ||
n_op == Op_MulI ||
n_op == Op_MulL ||
n_op == Op_MulF ||
n_op == Op_MulD ) {
if( n2_loop == n_loop ) {
assert( n1_loop != n_loop, "" );
n->swap_edges(1, 2);
}
}
if( n_op == Op_AddP ) {
if( n2_loop == n_loop && n3_loop != n_loop ) {
if( n->in(2)->Opcode() == Op_AddP && !n->in(3)->is_Con() ) {
Node *n22_ctrl = get_ctrl(n->in(2)->in(2));
Node *n23_ctrl = get_ctrl(n->in(2)->in(3));
IdealLoopTree *n22loop = get_loop( n22_ctrl );
IdealLoopTree *n23_loop = get_loop( n23_ctrl );
if( n22loop != n_loop && n22loop->is_member(n_loop) &&
n23_loop == n_loop ) {
Node *add1 = new (C) AddPNode( n->in(1), n->in(2)->in(2), n->in(3) );
register_new_node( add1, n_loop->_head->in(LoopNode::EntryControl) );
Node *add2 = new (C) AddPNode( n->in(1), add1, n->in(2)->in(3) );
register_new_node( add2, n_ctrl );
_igvn.replace_node( n, add2 );
return add2;
}
}
}
if( n2_loop != n_loop && n3_loop == n_loop ) {
if( n->in(3)->Opcode() == Op_AddI ) {
Node *V = n->in(3)->in(1);
Node *I = n->in(3)->in(2);
if( is_member(n_loop,get_ctrl(V)) ) {
} else {
Node *tmp = V; V = I; I = tmp;
}
if( !is_member(n_loop,get_ctrl(I)) ) {
Node *add1 = new (C) AddPNode( n->in(1), n->in(2), I );
register_new_node( add1, n_loop->_head->in(LoopNode::EntryControl) );
Node *add2 = new (C) AddPNode( n->in(1), add1, V );
register_new_node( add2, n_ctrl );
_igvn.replace_node( n, add2 );
return add2;
}
}
}
}
return NULL;
}
Node *PhaseIdealLoop::conditional_move( Node *region ) {
assert(region->is_Region(), "sanity check");
if (region->req() != 3) return NULL;
Node *lp = region->in(1);
Node *rp = region->in(2);
if (!lp || !rp) return NULL;
Node *lp_c = lp->in(0);
if (lp_c == NULL || lp_c != rp->in(0) || !lp_c->is_If()) return NULL;
IfNode *iff = lp_c->as_If();
if (lp->outcnt() > 1) return NULL;
if (rp->outcnt() > 1) return NULL;
IdealLoopTree* r_loop = get_loop(region);
assert(r_loop == get_loop(iff), "sanity");
bool used_inside_loop = (r_loop == _ltree_root);
int cost = 0;
int phis = 0;
for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
Node *out = region->fast_out(i);
if (!out->is_Phi()) continue; // Ignore other control edges, etc
phis++;
PhiNode* phi = out->as_Phi();
BasicType bt = phi->type()->basic_type();
switch (bt) {
case T_FLOAT:
case T_DOUBLE: {
cost += Matcher::float_cmove_cost(); // Could be very expensive
break;
}
case T_LONG: {
cost += Matcher::long_cmove_cost(); // May encodes as 2 CMOV's
}
case T_INT: // These all CMOV fine
case T_ADDRESS: { // (RawPtr)
cost++;
break;
}
case T_NARROWOOP: // Fall through
case T_OBJECT: { // Base oops are OK, but not derived oops
const TypeOopPtr *tp = phi->type()->make_ptr()->isa_oopptr();
if (tp && tp->offset() != 0)
return NULL;
cost++;
break;
}
default:
return NULL; // In particular, can't do memory or I/O
}
for (uint j = 1; j < region->req(); j++) {
Node *proj = region->in(j);
Node *inp = phi->in(j);
if (get_ctrl(inp) == proj) { // Found local op
cost++;
for (uint k = 1; k < inp->req(); k++)
if (get_ctrl(inp->in(k)) == proj)
cost += ConditionalMoveLimit; // Too much speculative goo
}
}
for (DUIterator_Fast kmax, k = phi->fast_outs(kmax); k < kmax; k++) {
Node* use = phi->fast_out(k);
if (use->is_Cmp() || use->is_DecodeNarrowPtr() || use->is_EncodeNarrowPtr())
cost += ConditionalMoveLimit;
if (!used_inside_loop && is_java_primitive(bt)) {
IdealLoopTree* u_loop = get_loop(has_ctrl(use) ? get_ctrl(use) : use);
if (r_loop == u_loop || r_loop->is_member(u_loop)) {
used_inside_loop = true;
}
}
}
}
Node* bol = iff->in(1);
assert(bol->Opcode() == Op_Bool, "");
int cmp_op = bol->in(1)->Opcode();
if (phis > 1 && (cmp_op == Op_CmpF || cmp_op == Op_CmpD)) return NULL;
float infrequent_prob = PROB_UNLIKELY_MAG(3);
if (used_inside_loop) {
if (cost >= ConditionalMoveLimit) return NULL; // Too much goo
if (BlockLayoutByFrequency) {
infrequent_prob = MAX2(infrequent_prob, (float)BlockLayoutMinDiamondPercentage/110.0f);
}
}
if (iff->_prob < infrequent_prob ||
iff->_prob > (1.0f - infrequent_prob))
return NULL;
Node *cmov_ctrl = iff->in(0);
uint flip = (lp->Opcode() == Op_IfTrue);
Node_List wq;
while (1) {
PhiNode* phi = NULL;
for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
Node *out = region->fast_out(i);
if (out->is_Phi()) {
phi = out->as_Phi();
break;
}
}
if (phi == NULL) break;
#ifndef PRODUCT
if (PrintOpto && VerifyLoopOptimizations) tty->print_cr("CMOV");
#endif
wq.push(phi);
while (wq.size() > 0) {
Node *n = wq.pop();
for (uint j = 1; j < n->req(); j++) {
Node* m = n->in(j);
if (m != NULL && !is_dominator(get_ctrl(m), cmov_ctrl)) {
#ifndef PRODUCT
if (PrintOpto && VerifyLoopOptimizations) {
tty->print(" speculate: ");
m->dump();
}
#endif
set_ctrl(m, cmov_ctrl);
wq.push(m);
}
}
}
Node *cmov = CMoveNode::make( C, cmov_ctrl, iff->in(1), phi->in(1+flip), phi->in(2-flip), _igvn.type(phi) );
register_new_node( cmov, cmov_ctrl );
_igvn.replace_node( phi, cmov );
#ifndef PRODUCT
if (TraceLoopOpts) {
tty->print("CMOV ");
r_loop->dump_head();
if (Verbose) {
bol->in(1)->dump(1);
cmov->dump(1);
}
}
if (VerifyLoopOptimizations) verify();
#endif
}
_igvn._worklist.push(region);
return iff->in(1);
}
Node *PhaseIdealLoop::split_if_with_blocks_pre( Node *n ) {
int n_op = n->Opcode();
if( n_op == Op_MergeMem ) return n;
if( n->is_Proj() ) return n;
if( n->is_Cmp() ) return n;
if( ConditionalMoveLimit > 0 && n_op == Op_Region ) {
Node *cmov = conditional_move( n );
if( cmov ) return cmov;
}
if( n->is_CFG() || n->is_LoadStore() )
return n;
if( n_op == Op_Opaque1 || // Opaque nodes cannot be mod'd
n_op == Op_Opaque2 ) {
if( !C->major_progress() ) // If chance of no more loop opts...
_igvn._worklist.push(n); // maybe we'll remove them
return n;
}
if( n->is_Con() ) return n; // No cloning for Con nodes
Node *n_ctrl = get_ctrl(n);
if( !n_ctrl ) return n; // Dead node
Node *m = remix_address_expressions( n );
if( m ) return m;
Node *n_blk = has_local_phi_input( n );
if( !n_blk ) return n;
if( n_blk->is_CountedLoop() && n->Opcode() == Op_AddI ) return n;
if( n->in(0) ) {
Node *dom = idom(n_blk);
if( dom_lca( n->in(0), dom ) != n->in(0) )
return n;
}
int policy = n_blk->req() >> 2;
if (n_blk->is_CountedLoop()) {
IdealLoopTree *lp = get_loop(n_blk);
if (lp && lp->_rce_candidate) {
return n;
}
}
if( C->unique() > 35000 ) return n; // Method too big
Node *phi = split_thru_phi( n, n_blk, policy );
if (!phi) return n;
_igvn.replace_node( n, phi );
if (n_blk->is_Loop() && n->is_Load() &&
!phi->in(LoopNode::LoopBackControl)->is_Load())
C->set_major_progress();
return phi;
}
static bool merge_point_too_heavy(Compile* C, Node* region) {
int weight = 0;
for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
weight += region->fast_out(i)->outcnt();
}
int nodes_left = C->max_node_limit() - C->live_nodes();
if (weight * 8 > nodes_left) {
#ifndef PRODUCT
if (PrintOpto)
tty->print_cr("*** Split-if bails out: %d nodes, region weight %d", C->unique(), weight);
#endif
return true;
} else {
return false;
}
}
static bool merge_point_safe(Node* region) {
for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
Node* n = region->fast_out(i);
if (n->is_Phi()) {
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* m = n->fast_out(j);
if (m->is_FastLock())
return false;
#ifdef _LP64
if (m->Opcode() == Op_ConvI2L)
return false;
if (m->is_CastII() && m->isa_CastII()->has_range_check()) {
return false;
}
#endif
}
}
}
return true;
}
Node *PhaseIdealLoop::place_near_use( Node *useblock ) const {
IdealLoopTree *u_loop = get_loop( useblock );
return (u_loop->_irreducible || u_loop->_child)
? useblock
: u_loop->_head->in(LoopNode::EntryControl);
}
void PhaseIdealLoop::split_if_with_blocks_post( Node *n ) {
if( n->is_Cmp() && !n->is_FastLock() ) {
if( C->unique() > 35000 ) return; // Method too big
if( _has_irreducible_loops )
return;
Node *n_ctrl = get_ctrl(n);
Node *n_blk = has_local_phi_input( n );
if( n_blk != n_ctrl ) return;
if( merge_point_too_heavy(C, n_ctrl) )
return;
if( n->outcnt() != 1 ) return; // Multiple bool's from 1 compare?
Node *bol = n->unique_out();
assert( bol->is_Bool(), "expect a bool here" );
if( bol->outcnt() != 1 ) return;// Multiple branches from 1 compare?
Node *iff = bol->unique_out();
if( iff->is_If() ) { // Classic split-if?
if( iff->in(0) != n_ctrl ) return; // Compare must be in same blk as if
} else if (iff->is_CMove()) { // Trying to split-up a CMOVE
if (iff->in(0) != NULL && iff->in(0) != n_ctrl ) return;
if( get_ctrl(iff->in(2)) == n_ctrl ||
get_ctrl(iff->in(3)) == n_ctrl )
return; // Inputs not yet split-up
if ( get_loop(n_ctrl) != get_loop(get_ctrl(iff)) ) {
return; // Loop-invar test gates loop-varying CMOVE
}
} else {
return; // some other kind of node, such as an Allocate
}
for( uint i = 1; i < n_ctrl->req(); i++ )
if( n_ctrl->in(i) == C->top() )
return;
int policy = 0;
IdealLoopTree *n_loop = get_loop(n_ctrl);
for( uint j = 1; j < n_ctrl->req(); j++ )
if( get_loop(n_ctrl->in(j)) != n_loop )
return;
if( !merge_point_safe(n_ctrl) ) {
return;
}
Node *phi = split_thru_phi( n, n_ctrl, policy );
if( !phi ) return;
_igvn.replace_node( n, phi );
Node *bolphi = split_thru_phi( bol, n_ctrl, -1 );
guarantee(bolphi != NULL, "null boolean phi node");
_igvn.replace_node( bol, bolphi );
assert( iff->in(1) == bolphi, "" );
if( bolphi->Value(&_igvn)->singleton() )
return;
if( !iff->is_If() ) {
Node *cmovphi = split_thru_phi( iff, n_ctrl, -1 );
_igvn.replace_node( iff, cmovphi );
return;
}
do_split_if( iff );
return;
}
int n_op = n->Opcode();
if (n_op == Op_If) {
Node *bol = n->in(1);
uint max = bol->outcnt();
if (max > 1 && bol->is_Bool()) {
Node *cutoff = get_ctrl(bol);
Node *prevdom = n;
Node *dom = idom(prevdom);
while (dom != cutoff) {
if (dom->req() > 1 && dom->in(1) == bol && prevdom->in(0) == dom) {
C->set_major_progress();
dominated_by(prevdom, n, false, true);
#ifndef PRODUCT
if( VerifyLoopOptimizations ) verify();
#endif
return;
}
prevdom = dom;
dom = idom(prevdom);
}
}
}
if (has_ctrl(n) && !n->in(0)) {// n not dead and has no control edge (can float about)
Node *n_ctrl = get_ctrl(n);
IdealLoopTree *n_loop = get_loop(n_ctrl);
if( n_loop != _ltree_root ) {
DUIterator_Fast imax, i = n->fast_outs(imax);
for (; i < imax; i++) {
Node* u = n->fast_out(i);
if( !has_ctrl(u) ) break; // Found control user
IdealLoopTree *u_loop = get_loop(get_ctrl(u));
if( u_loop == n_loop ) break; // Found loop-varying use
if( n_loop->is_member( u_loop ) ) break; // Found use in inner loop
if( u->Opcode() == Op_Opaque1 ) break; // Found loop limit, bugfix for 4677003
}
bool did_break = (i < imax); // Did we break out of the previous loop?
if (!did_break && n->outcnt() > 1) { // All uses in outer loops!
Node *late_load_ctrl = NULL;
if (n->is_Load()) {
clear_dom_lca_tags();
late_load_ctrl = get_late_ctrl(n, n_ctrl);
}
if (!n->is_Load() || late_load_ctrl != n_ctrl) {
for (DUIterator_Last jmin, j = n->last_outs(jmin); j >= jmin; ) {
Node *u = n->last_out(j); // Clone private computation per use
_igvn.rehash_node_delayed(u);
Node *x = n->clone(); // Clone computation
Node *x_ctrl = NULL;
if( u->is_Phi() ) {
uint k = 1;
while( u->in(k) != n ) k++;
u->set_req( k, x );
x_ctrl = u->in(0)->in(k);
--j;
} else { // Normal use
for( uint k = 0; k < u->req(); k++ ) {
if( u->in(k) == n ) {
u->set_req( k, x );
--j;
}
}
x_ctrl = get_ctrl(u);
}
x_ctrl = place_near_use(x_ctrl);
if (n->is_Load()) {
x_ctrl = dom_lca(late_load_ctrl, x_ctrl);
x_ctrl = find_non_split_ctrl(x_ctrl);
assert(dom_depth(n_ctrl) <= dom_depth(x_ctrl), "n is later than its clone");
x->set_req(0, x_ctrl);
}
register_new_node(x, x_ctrl);
if (!x->is_Load() && !x->is_DecodeNarrowPtr()) _igvn._worklist.yank(x);
}
_igvn.remove_dead_node(n);
}
}
}
}
if( n_op == Op_Opaque2 &&
n->in(1) != NULL &&
get_loop(get_ctrl(n)) == get_loop(get_ctrl(n->in(1))) ) {
_igvn.replace_node( n, n->in(1) );
}
}
void PhaseIdealLoop::split_if_with_blocks( VectorSet &visited, Node_Stack &nstack ) {
Node *n = C->root();
visited.set(n->_idx); // first, mark node as visited
n = split_if_with_blocks_pre( n );
uint cnt = n->outcnt();
uint i = 0;
while (true) {
if (i < cnt) {
Node* use = n->raw_out(i);
++i;
if (use->outcnt() != 0 && !visited.test_set(use->_idx)) {
use = split_if_with_blocks_pre( use );
nstack.push(n, i); // Save parent and next use's index.
n = use; // Process all children of current use.
cnt = use->outcnt();
i = 0;
}
}
else {
if (cnt != 0 && !n->is_Con()) {
assert(has_node(n), "no dead nodes");
split_if_with_blocks_post( n );
}
if (nstack.is_empty()) {
break;
}
n = nstack.node();
cnt = n->outcnt();
i = nstack.index();
nstack.pop();
}
}
}
BoolNode *PhaseIdealLoop::clone_iff( PhiNode *phi, IdealLoopTree *loop ) {
uint i;
for( i = 1; i < phi->req(); i++ ) {
Node *b = phi->in(i);
if( b->is_Phi() ) {
_igvn.replace_input_of(phi, i, clone_iff( b->as_Phi(), loop ));
} else {
assert( b->is_Bool(), "" );
}
}
Node *sample_bool = phi->in(1);
Node *sample_cmp = sample_bool->in(1);
PhiNode *phi1 = new (C) PhiNode( phi->in(0), Type::TOP );
PhiNode *phi2 = new (C) PhiNode( phi->in(0), Type::TOP );
for( i = 1; i < phi->req(); i++ ) {
Node *n1 = phi->in(i)->in(1)->in(1);
Node *n2 = phi->in(i)->in(1)->in(2);
phi1->set_req( i, n1 );
phi2->set_req( i, n2 );
phi1->set_type( phi1->type()->meet_speculative(n1->bottom_type()));
phi2->set_type( phi2->type()->meet_speculative(n2->bottom_type()));
}
Node *hit1 = _igvn.hash_find_insert(phi1);
if( hit1 ) { // Hit, toss just made Phi
_igvn.remove_dead_node(phi1); // Remove new phi
assert( hit1->is_Phi(), "" );
phi1 = (PhiNode*)hit1; // Use existing phi
} else { // Miss
_igvn.register_new_node_with_optimizer(phi1);
}
Node *hit2 = _igvn.hash_find_insert(phi2);
if( hit2 ) { // Hit, toss just made Phi
_igvn.remove_dead_node(phi2); // Remove new phi
assert( hit2->is_Phi(), "" );
phi2 = (PhiNode*)hit2; // Use existing phi
} else { // Miss
_igvn.register_new_node_with_optimizer(phi2);
}
set_ctrl(phi1, phi->in(0));
set_ctrl(phi2, phi->in(0));
Node *cmp = sample_cmp->clone();
cmp->set_req( 1, phi1 );
cmp->set_req( 2, phi2 );
_igvn.register_new_node_with_optimizer(cmp);
set_ctrl(cmp, phi->in(0));
Node *b = sample_bool->clone();
b->set_req(1,cmp);
_igvn.register_new_node_with_optimizer(b);
set_ctrl(b, phi->in(0));
assert( b->is_Bool(), "" );
return (BoolNode*)b;
}
CmpNode *PhaseIdealLoop::clone_bool( PhiNode *phi, IdealLoopTree *loop ) {
uint i;
for( i = 1; i < phi->req(); i++ ) {
Node *b = phi->in(i);
if( b->is_Phi() ) {
_igvn.replace_input_of(phi, i, clone_bool( b->as_Phi(), loop ));
} else {
assert( b->is_Cmp() || b->is_top(), "inputs are all Cmp or TOP" );
}
}
Node *sample_cmp = phi->in(1);
PhiNode *phi1 = new (C) PhiNode( phi->in(0), Type::TOP );
PhiNode *phi2 = new (C) PhiNode( phi->in(0), Type::TOP );
for( uint j = 1; j < phi->req(); j++ ) {
Node *cmp_top = phi->in(j); // Inputs are all Cmp or TOP
Node *n1, *n2;
if( cmp_top->is_Cmp() ) {
n1 = cmp_top->in(1);
n2 = cmp_top->in(2);
} else {
n1 = n2 = cmp_top;
}
phi1->set_req( j, n1 );
phi2->set_req( j, n2 );
phi1->set_type(phi1->type()->meet_speculative(n1->bottom_type()));
phi2->set_type(phi2->type()->meet_speculative(n2->bottom_type()));
}
Node *hit1 = _igvn.hash_find_insert(phi1);
if( hit1 ) { // Hit, toss just made Phi
_igvn.remove_dead_node(phi1); // Remove new phi
assert( hit1->is_Phi(), "" );
phi1 = (PhiNode*)hit1; // Use existing phi
} else { // Miss
_igvn.register_new_node_with_optimizer(phi1);
}
Node *hit2 = _igvn.hash_find_insert(phi2);
if( hit2 ) { // Hit, toss just made Phi
_igvn.remove_dead_node(phi2); // Remove new phi
assert( hit2->is_Phi(), "" );
phi2 = (PhiNode*)hit2; // Use existing phi
} else { // Miss
_igvn.register_new_node_with_optimizer(phi2);
}
set_ctrl(phi1, phi->in(0));
set_ctrl(phi2, phi->in(0));
Node *cmp = sample_cmp->clone();
cmp->set_req( 1, phi1 );
cmp->set_req( 2, phi2 );
_igvn.register_new_node_with_optimizer(cmp);
set_ctrl(cmp, phi->in(0));
assert( cmp->is_Cmp(), "" );
return (CmpNode*)cmp;
}
void PhaseIdealLoop::sink_use( Node *use, Node *post_loop ) {
if (!use->is_CFG() && get_ctrl(use) == post_loop->in(2)) {
set_ctrl(use, post_loop);
for (DUIterator j = use->outs(); use->has_out(j); j++)
sink_use(use->out(j), post_loop);
}
}
void PhaseIdealLoop::clone_loop( IdealLoopTree *loop, Node_List &old_new, int dd,
Node* side_by_side_idom) {
uint i;
for( i = 0; i < loop->_body.size(); i++ ) {
Node *old = loop->_body.at(i);
Node *nnn = old->clone();
old_new.map( old->_idx, nnn );
_igvn.register_new_node_with_optimizer(nnn);
}
for( i = 0; i < loop->_body.size(); i++ ) {
Node *old = loop->_body.at(i);
Node *nnn = old_new[old->_idx];
if (has_ctrl(old)) {
set_ctrl(nnn, old_new[get_ctrl(old)->_idx]);
} else {
set_loop(nnn, loop->_parent);
if (old->outcnt() > 0) {
set_idom( nnn, old_new[idom(old)->_idx], dd );
}
}
for( uint j = 0; j < nnn->req(); j++ ) {
Node *n = nnn->in(j);
if( n ) {
IdealLoopTree *old_in_loop = get_loop( has_ctrl(n) ? get_ctrl(n) : n );
if( loop->is_member( old_in_loop ) )
nnn->set_req(j, old_new[n->_idx]);
}
}
_igvn.hash_find_insert(nnn);
}
Node *newhead = old_new[loop->_head->_idx];
set_idom(newhead, newhead->in(LoopNode::EntryControl), dd);
ResourceArea *area = Thread::current()->resource_area();
Node_List worklist(area);
uint new_counter = C->unique();
for( i = 0; i < loop->_body.size(); i++ ) {
Node* old = loop->_body.at(i);
if( !old->is_CFG() ) continue;
Node* nnn = old_new[old->_idx];
for (DUIterator_Fast jmax, j = old->fast_outs(jmax); j < jmax; j++)
worklist.push(old->fast_out(j));
while( worklist.size() ) { // Visit all uses
Node *use = worklist.pop();
if (!has_node(use)) continue; // Ignore dead nodes
IdealLoopTree *use_loop = get_loop( has_ctrl(use) ? get_ctrl(use) : use );
if( !loop->is_member( use_loop ) && use->is_CFG() ) {
assert( use->is_Proj(), "" );
Node *newuse = use->clone();
newuse->set_req(0,nnn);
_igvn.register_new_node_with_optimizer(newuse);
set_loop(newuse, use_loop);
set_idom(newuse, nnn, dom_depth(nnn) + 1 );
RegionNode *r = new (C) RegionNode(3);
old_new.map( use->_idx, r );
uint dd_r = MIN2(dom_depth(newuse),dom_depth(use));
assert( dd_r >= dom_depth(dom_lca(newuse,use)), "" );
for (DUIterator_Last lmin, l = use->last_outs(lmin); l >= lmin;) {
Node* useuse = use->last_out(l);
_igvn.rehash_node_delayed(useuse);
uint uses_found = 0;
if( useuse->in(0) == use ) {
useuse->set_req(0, r);
uses_found++;
if( useuse->is_CFG() ) {
assert( dom_depth(useuse) > dd_r, "" );
set_idom(useuse, r, dom_depth(useuse));
}
}
for( uint k = 1; k < useuse->req(); k++ ) {
if( useuse->in(k) == use ) {
useuse->set_req(k, r);
uses_found++;
}
}
l -= uses_found; // we deleted 1 or more copies of this edge
}
r->set_req( 1, newuse );
r->set_req( 2, use );
_igvn.register_new_node_with_optimizer(r);
set_loop(r, use_loop);
set_idom(r, !side_by_side_idom ? newuse->in(0) : side_by_side_idom, dd_r);
} // End of if a loop-exit test
}
}
Node_List *split_if_set = NULL;
Node_List *split_bool_set = NULL;
Node_List *split_cex_set = NULL;
for( i = 0; i < loop->_body.size(); i++ ) {
Node* old = loop->_body.at(i);
Node* nnn = old_new[old->_idx];
for (DUIterator_Fast jmax, j = old->fast_outs(jmax); j < jmax; j++)
worklist.push(old->fast_out(j));
while( worklist.size() ) {
Node *use = worklist.pop();
if (!has_node(use)) continue; // Ignore dead nodes
if (use->in(0) == C->top()) continue;
IdealLoopTree *use_loop = get_loop( has_ctrl(use) ? get_ctrl(use) : use );
if( !loop->is_member( use_loop ) && (!old->is_CFG() || !use->is_CFG())) {
if (use->is_If() || use->is_CMove() || C->is_predicate_opaq(use)) {
if( !split_if_set )
split_if_set = new Node_List(area);
split_if_set->push(use);
}
if( use->is_Bool() ) {
if( !split_bool_set )
split_bool_set = new Node_List(area);
split_bool_set->push(use);
}
if( use->Opcode() == Op_CreateEx ) {
if( !split_cex_set )
split_cex_set = new Node_List(area);
split_cex_set->push(use);
}
uint idx = 0;
while( use->in(idx) != old ) idx++;
Node *prev = use->is_CFG() ? use : get_ctrl(use);
assert( !loop->is_member( get_loop( prev ) ), "" );
Node *cfg = prev->_idx >= new_counter
? prev->in(2)
: idom(prev);
if( use->is_Phi() ) // Phi use is in prior block
cfg = prev->in(idx); // NOT in block of Phi itself
if (cfg->is_top()) { // Use is dead?
_igvn.replace_input_of(use, idx, C->top());
continue;
}
while( !loop->is_member( get_loop( cfg ) ) ) {
prev = cfg;
cfg = cfg->_idx >= new_counter ? cfg->in(2) : idom(cfg);
}
Node *phi;
if( prev->is_Region() ) {
if( idx == 0 ) { // Updating control edge?
phi = prev; // Just use existing control
} else { // Else need a new Phi
phi = PhiNode::make( prev, old );
for( uint i = 1; i < prev->req(); i++ ) {
worklist.push(phi); // Onto worklist once for each 'old' input
}
}
} else {
prev = old_new[prev->_idx];
assert( prev, "just made this in step 7" );
if( idx == 0 ) { // Updating control edge?
phi = prev; // Just use existing control
} else { // Else need a new Phi
phi = PhiNode::make( prev, old );
phi->set_req( 1, nnn );
}
}
if( idx != 0 ) {
Node *hit = _igvn.hash_find_insert(phi);
if( hit == NULL ) {
_igvn.register_new_node_with_optimizer(phi); // Register new phi
} else { // or
_igvn.remove_dead_node(phi);
phi = hit; // Use existing phi
}
set_ctrl(phi, prev);
}
_igvn.replace_input_of(use, idx, phi);
if( use->_idx >= new_counter ) { // If updating new phis
Node *hit = _igvn.hash_find_insert(use);
if( hit ) // Go ahead and re-hash for hits.
_igvn.replace_node( use, hit );
}
sink_use( use, prev );
}
}
}
if( split_if_set ) {
while( split_if_set->size() ) {
Node *iff = split_if_set->pop();
if( iff->in(1)->is_Phi() ) {
BoolNode *b = clone_iff( iff->in(1)->as_Phi(), loop );
_igvn.replace_input_of(iff, 1, b);
}
}
}
if( split_bool_set ) {
while( split_bool_set->size() ) {
Node *b = split_bool_set->pop();
Node *phi = b->in(1);
assert( phi->is_Phi(), "" );
CmpNode *cmp = clone_bool( (PhiNode*)phi, loop );
_igvn.replace_input_of(b, 1, cmp);
}
}
if( split_cex_set ) {
while( split_cex_set->size() ) {
Node *b = split_cex_set->pop();
assert( b->in(0)->is_Region(), "" );
assert( b->in(1)->is_Phi(), "" );
assert( b->in(0)->in(0) == b->in(1)->in(0), "" );
split_up( b, b->in(0), NULL );
}
}
}
int PhaseIdealLoop::stride_of_possible_iv(Node* iff) {
Node* trunc1 = NULL;
Node* trunc2 = NULL;
const TypeInt* ttype = NULL;
if (!iff->is_If() || iff->in(1) == NULL || !iff->in(1)->is_Bool()) {
return 0;
}
BoolNode* bl = iff->in(1)->as_Bool();
Node* cmp = bl->in(1);
if (!cmp || cmp->Opcode() != Op_CmpI && cmp->Opcode() != Op_CmpU) {
return 0;
}
if (is_member(get_loop(iff), get_ctrl(cmp->in(2)))) {
return 0;
}
Node* add2 = NULL;
Node* cmp1 = cmp->in(1);
if (cmp1->is_Phi()) {
Node* phi = cmp1;
for (uint i = 1; i < phi->req(); i++) {
Node* in = phi->in(i);
Node* add = CountedLoopNode::match_incr_with_optional_truncation(in,
&trunc1, &trunc2, &ttype);
if (add && add->in(1) == phi) {
add2 = add->in(2);
break;
}
}
} else {
Node* addtrunc = cmp1;
Node* add = CountedLoopNode::match_incr_with_optional_truncation(addtrunc,
&trunc1, &trunc2, &ttype);
if (add && add->in(1)->is_Phi()) {
Node* phi = add->in(1);
for (uint i = 1; i < phi->req(); i++) {
if (phi->in(i) == addtrunc) {
add2 = add->in(2);
break;
}
}
}
}
if (add2 != NULL) {
const TypeInt* add2t = _igvn.type(add2)->is_int();
if (add2t->is_con()) {
return add2t->get_con();
}
}
return 0;
}
Node* PhaseIdealLoop::stay_in_loop( Node* n, IdealLoopTree *loop) {
Node* unique = NULL;
if (!n) return NULL;
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
Node* use = n->fast_out(i);
if (!has_ctrl(use) && loop->is_member(get_loop(use))) {
if (unique != NULL) {
return NULL;
}
unique = use;
}
}
return unique;
}
void PhaseIdealLoop::register_node(Node* n, IdealLoopTree *loop, Node* pred, int ddepth) {
_igvn.register_new_node_with_optimizer(n);
loop->_body.push(n);
if (n->is_CFG()) {
set_loop(n, loop);
set_idom(n, pred, ddepth);
} else {
set_ctrl(n, pred);
}
}
ProjNode* PhaseIdealLoop::proj_clone(ProjNode* p, IfNode* iff) {
ProjNode* c = p->clone()->as_Proj();
c->set_req(0, iff);
return c;
}
Node* PhaseIdealLoop::short_circuit_if(IfNode* iff, ProjNode* live_proj) {
guarantee(live_proj != NULL, "null projection");
int proj_con = live_proj->_con;
assert(proj_con == 0 || proj_con == 1, "false or true projection");
Node *con = _igvn.intcon(proj_con);
set_ctrl(con, C->root());
if (iff) {
iff->set_req(1, con);
}
return con;
}
ProjNode* PhaseIdealLoop::insert_if_before_proj(Node* left, bool Signed, BoolTest::mask relop, Node* right, ProjNode* proj) {
IfNode* iff = proj->in(0)->as_If();
IdealLoopTree *loop = get_loop(proj);
ProjNode *other_proj = iff->proj_out(!proj->is_IfTrue())->as_Proj();
int ddepth = dom_depth(proj);
_igvn.rehash_node_delayed(iff);
_igvn.rehash_node_delayed(proj);
proj->set_req(0, NULL); // temporary disconnect
ProjNode* proj2 = proj_clone(proj, iff);
register_node(proj2, loop, iff, ddepth);
Node* cmp = Signed ? (Node*) new (C)CmpINode(left, right) : (Node*) new (C)CmpUNode(left, right);
register_node(cmp, loop, proj2, ddepth);
BoolNode* bol = new (C)BoolNode(cmp, relop);
register_node(bol, loop, proj2, ddepth);
IfNode* new_if = new (C)IfNode(proj2, bol, iff->_prob, iff->_fcnt);
register_node(new_if, loop, proj2, ddepth);
proj->set_req(0, new_if); // reattach
set_idom(proj, new_if, ddepth);
ProjNode* new_exit = proj_clone(other_proj, new_if)->as_Proj();
guarantee(new_exit != NULL, "null exit node");
register_node(new_exit, get_loop(other_proj), new_if, ddepth);
return new_exit;
}
RegionNode* PhaseIdealLoop::insert_region_before_proj(ProjNode* proj) {
IfNode* iff = proj->in(0)->as_If();
IdealLoopTree *loop = get_loop(proj);
ProjNode *other_proj = iff->proj_out(!proj->is_IfTrue())->as_Proj();
int ddepth = dom_depth(proj);
_igvn.rehash_node_delayed(iff);
_igvn.rehash_node_delayed(proj);
proj->set_req(0, NULL); // temporary disconnect
ProjNode* proj2 = proj_clone(proj, iff);
register_node(proj2, loop, iff, ddepth);
RegionNode* reg = new (C)RegionNode(2);
reg->set_req(1, proj2);
register_node(reg, loop, iff, ddepth);
IfNode* dum_if = new (C)IfNode(reg, short_circuit_if(NULL, proj), iff->_prob, iff->_fcnt);
register_node(dum_if, loop, reg, ddepth);
proj->set_req(0, dum_if); // reattach
set_idom(proj, dum_if, ddepth);
ProjNode* dum_proj = proj_clone(other_proj, dum_if);
register_node(dum_proj, loop, dum_if, ddepth);
return reg;
}
IfNode* PhaseIdealLoop::insert_cmpi_loop_exit(IfNode* if_cmpu, IdealLoopTree *loop) {
const bool Signed = true;
const bool Unsigned = false;
BoolNode* bol = if_cmpu->in(1)->as_Bool();
if (bol->_test._test != BoolTest::lt) return NULL;
CmpNode* cmpu = bol->in(1)->as_Cmp();
if (cmpu->Opcode() != Op_CmpU) return NULL;
int stride = stride_of_possible_iv(if_cmpu);
if (stride == 0) return NULL;
Node* lp_proj = stay_in_loop(if_cmpu, loop);
guarantee(lp_proj != NULL, "null loop node");
ProjNode* lp_continue = lp_proj->as_Proj();
ProjNode* lp_exit = if_cmpu->proj_out(!lp_continue->is_IfTrue())->as_Proj();
Node* limit = NULL;
if (stride > 0) {
limit = cmpu->in(2);
} else {
limit = _igvn.makecon(TypeInt::ZERO);
set_ctrl(limit, C->root());
}
RegionNode* reg = insert_region_before_proj(lp_exit);
guarantee(reg != NULL, "null region node");
BoolTest::mask rel_i = stride > 0 ? bol->_test._test : BoolTest::ge;
ProjNode* cmpi_exit = insert_if_before_proj(cmpu->in(1), Signed, rel_i, limit, lp_continue);
reg->add_req(cmpi_exit);
BoolTest::mask rel_u = bol->_test._test;
ProjNode* cmpu_exit = insert_if_before_proj(cmpu->in(1), Unsigned, rel_u, cmpu->in(2), lp_continue);
reg->add_req(cmpu_exit);
short_circuit_if(if_cmpu, lp_continue);
return cmpi_exit->in(0)->as_If();
}
void PhaseIdealLoop::remove_cmpi_loop_exit(IfNode* if_cmp, IdealLoopTree *loop) {
Node* lp_proj = stay_in_loop(if_cmp, loop);
assert(if_cmp->in(1)->in(1)->Opcode() == Op_CmpI &&
stay_in_loop(lp_proj, loop)->is_If() &&
stay_in_loop(lp_proj, loop)->in(1)->in(1)->Opcode() == Op_CmpU, "inserted cmpi before cmpu");
Node *con = _igvn.makecon(lp_proj->is_IfTrue() ? TypeInt::ONE : TypeInt::ZERO);
set_ctrl(con, C->root());
if_cmp->set_req(1, con);
}
void PhaseIdealLoop::scheduled_nodelist( IdealLoopTree *loop, VectorSet& member, Node_List &sched ) {
assert(member.test(loop->_head->_idx), "loop head must be in member set");
Arena *a = Thread::current()->resource_area();
VectorSet visited(a);
Node_Stack nstack(a, loop->_body.size());
Node* n = loop->_head; // top of stack is cached in "n"
uint idx = 0;
visited.set(n->_idx);
for(uint i = 0; i < loop->_body.size(); i++ ) {
Node *elt = loop->_body.at(i);
if (member.test(elt->_idx)) {
bool found = false;
for (uint j = 0; j < elt->req(); j++) {
Node* def = elt->in(j);
if (def && member.test(def->_idx) && def != elt) {
found = true;
break;
}
}
if (!found && elt != loop->_head) {
nstack.push(n, idx);
n = elt;
assert(!visited.test(n->_idx), "not seen yet");
visited.set(n->_idx);
}
}
}
while (true) {
if (idx < n->outcnt()) {
Node* use = n->raw_out(idx);
idx++;
if (!visited.test_set(use->_idx)) {
if (member.test(use->_idx)) {
nstack.push(n, idx);
n = use;
idx = 0;
}
}
} else {
sched.push(n);
if (nstack.is_empty()) break;
n = nstack.node();
idx = nstack.index();
nstack.pop();
}
}
}
bool PhaseIdealLoop::has_use_in_set( Node* n, VectorSet& vset ) {
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* use = n->fast_out(j);
if (vset.test(use->_idx)) {
return true;
}
}
return false;
}
bool PhaseIdealLoop::has_use_internal_to_set( Node* n, VectorSet& vset, IdealLoopTree *loop ) {
Node* head = loop->_head;
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* use = n->fast_out(j);
if (vset.test(use->_idx) && !(use->is_Phi() && use->in(0) == head)) {
return true;
}
}
return false;
}
int PhaseIdealLoop::clone_for_use_outside_loop( IdealLoopTree *loop, Node* n, Node_List& worklist ) {
int cloned = 0;
assert(worklist.size() == 0, "should be empty");
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* use = n->fast_out(j);
if( !loop->is_member(get_loop(has_ctrl(use) ? get_ctrl(use) : use)) ) {
worklist.push(use);
}
}
while( worklist.size() ) {
Node *use = worklist.pop();
if (!has_node(use) || use->in(0) == C->top()) continue;
uint j;
for (j = 0; j < use->req(); j++) {
if (use->in(j) == n) break;
}
assert(j < use->req(), "must be there");
Node* n_clone = n->clone();
_igvn.replace_input_of(use, j, n_clone);
cloned++;
Node* use_c;
if (!use->is_Phi()) {
use_c = has_ctrl(use) ? get_ctrl(use) : use->in(0);
} else {
use_c = use->in(0)->in(j);
}
set_ctrl(n_clone, use_c);
assert(!loop->is_member(get_loop(use_c)), "should be outside loop");
get_loop(use_c)->_body.push(n_clone);
_igvn.register_new_node_with_optimizer(n_clone);
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("loop exit cloning old: %d new: %d newbb: %d", n->_idx, n_clone->_idx, get_ctrl(n_clone)->_idx);
}
#endif
}
return cloned;
}
void PhaseIdealLoop::clone_for_special_use_inside_loop( IdealLoopTree *loop, Node* n,
VectorSet& not_peel, Node_List& sink_list, Node_List& worklist ) {
if (n->is_Phi() || n->is_Load()) {
return;
}
assert(worklist.size() == 0, "should be empty");
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* use = n->fast_out(j);
if ( not_peel.test(use->_idx) &&
(use->is_If() || use->is_CMove() || use->is_Bool()) &&
use->in(1) == n) {
worklist.push(use);
}
}
if (worklist.size() > 0) {
Node* n_clone = n->clone();
loop->_body.push(n_clone);
_igvn.register_new_node_with_optimizer(n_clone);
set_ctrl(n_clone, get_ctrl(n));
sink_list.push(n_clone);
not_peel <<= n_clone->_idx; // add n_clone to not_peel set.
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("special not_peeled cloning old: %d new: %d", n->_idx, n_clone->_idx);
}
#endif
while( worklist.size() ) {
Node *use = worklist.pop();
_igvn.rehash_node_delayed(use);
for (uint j = 1; j < use->req(); j++) {
if (use->in(j) == n) {
use->set_req(j, n_clone);
}
}
}
}
}
void PhaseIdealLoop::insert_phi_for_loop( Node* use, uint idx, Node* lp_entry_val, Node* back_edge_val, LoopNode* lp ) {
Node *phi = PhiNode::make(lp, back_edge_val);
phi->set_req(LoopNode::EntryControl, lp_entry_val);
Node *hit = _igvn.hash_find_insert(phi);
if( hit == NULL ) {
_igvn.register_new_node_with_optimizer(phi);
set_ctrl(phi, lp);
} else {
_igvn.remove_dead_node(phi);
phi = hit;
}
_igvn.replace_input_of(use, idx, phi);
}
#ifdef ASSERT
bool PhaseIdealLoop::is_valid_loop_partition( IdealLoopTree *loop, VectorSet& peel, Node_List& peel_list,
VectorSet& not_peel ) {
uint i;
for (i = 0; i < peel_list.size(); i++) {
if (!peel.test(peel_list.at(i)->_idx)) {
return false;
}
}
for (i = 0; i < loop->_body.size(); i++ ) {
Node *def = loop->_body.at(i);
uint di = def->_idx;
if (peel.test(di)) {
if (not_peel.test(di)) {
return false;
}
bool found = false;
for (uint j = 0; j < peel_list.size(); j++) {
if (peel_list.at(j)->_idx == di) {
found = true;
break;
}
}
if (!found) {
return false;
}
} else if (not_peel.test(di)) {
if (peel.test(di)) {
return false;
}
} else {
return false;
}
}
return true;
}
bool PhaseIdealLoop::is_valid_clone_loop_exit_use( IdealLoopTree *loop, Node* use, uint exit_idx) {
Node *use_c = has_ctrl(use) ? get_ctrl(use) : use;
return (use->is_Phi() &&
use_c->is_Region() && use_c->req() == 3 &&
(use_c->in(exit_idx)->Opcode() == Op_IfTrue ||
use_c->in(exit_idx)->Opcode() == Op_IfFalse ||
use_c->in(exit_idx)->Opcode() == Op_JumpProj) &&
loop->is_member( get_loop( use_c->in(exit_idx)->in(0) ) ) );
}
bool PhaseIdealLoop::is_valid_clone_loop_form( IdealLoopTree *loop, Node_List& peel_list,
uint orig_exit_idx, uint clone_exit_idx) {
uint len = peel_list.size();
for (uint i = 0; i < len; i++) {
Node *def = peel_list.at(i);
for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) {
Node *use = def->fast_out(j);
Node *use_c = has_ctrl(use) ? get_ctrl(use) : use;
if (!loop->is_member(get_loop(use_c))) {
if (use->in(0) == def) {
} else if (!is_valid_clone_loop_exit_use(loop, use, orig_exit_idx)) {
return false;
}
}
}
}
return true;
}
#endif
bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) {
assert(!loop->_head->is_CountedLoop(), "Non-counted loop only");
if (!loop->_head->is_Loop()) {
return false; }
LoopNode *head = loop->_head->as_Loop();
if (head->is_partial_peel_loop() || head->partial_peel_has_failed()) {
return false;
}
for(uint ii = 0; ii < loop->_body.size(); ii++ ) {
Node *n = loop->_body.at(ii);
int opc = n->Opcode();
if (n->is_Call() ||
opc == Op_Catch ||
opc == Op_CatchProj ||
opc == Op_Jump ||
opc == Op_JumpProj) {
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("\nExit control too complex: lp: %d", head->_idx);
}
#endif
return false;
}
}
int dd = dom_depth(head);
IfNode *peel_if = NULL;
IfNode *peel_if_cmpu = NULL;
Node *iff = loop->tail();
while( iff != head ) {
if( iff->is_If() ) {
Node *ctrl = get_ctrl(iff->in(1));
if (ctrl->is_top()) return false; // Dead test on live IF.
if( loop->is_member(get_loop(ctrl)) &&
loop->is_loop_exit(iff) &&
is_possible_iv_test(iff)) {
Node* cmp = iff->in(1)->in(1);
if (cmp->Opcode() == Op_CmpI) {
peel_if = iff->as_If();
} else {
assert(cmp->Opcode() == Op_CmpU, "must be CmpI or CmpU");
peel_if_cmpu = iff->as_If();
}
}
}
iff = idom(iff);
}
IfNode* new_peel_if = NULL;
if (peel_if == NULL) {
if (!PartialPeelAtUnsignedTests || peel_if_cmpu == NULL) {
return false; // No peel point found
}
new_peel_if = insert_cmpi_loop_exit(peel_if_cmpu, loop);
if (new_peel_if == NULL) {
return false; // No peel point found
}
peel_if = new_peel_if;
}
Node* last_peel = stay_in_loop(peel_if, loop);
Node* first_not_peeled = stay_in_loop(last_peel, loop);
if (first_not_peeled == NULL || first_not_peeled == head) {
return false;
}
#if !defined(PRODUCT)
if (TraceLoopOpts) {
tty->print("PartialPeel ");
loop->dump_head();
}
if (TracePartialPeeling) {
tty->print_cr("before partial peel one iteration");
Node_List wl;
Node* t = head->in(2);
while (true) {
wl.push(t);
if (t == head) break;
t = idom(t);
}
while (wl.size() > 0) {
Node* tt = wl.pop();
tt->dump();
if (tt == last_peel) tty->print_cr("-- cut --");
}
}
#endif
ResourceArea *area = Thread::current()->resource_area();
VectorSet peel(area);
VectorSet not_peel(area);
Node_List peel_list(area);
Node_List worklist(area);
Node_List sink_list(area);
assert(worklist.size() == 0, "should be empty");
worklist.push(head);
peel.set(head->_idx);
while (worklist.size() > 0) {
Node *n = worklist.pop();
if (n != last_peel) {
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* use = n->fast_out(j);
if (use->is_CFG() &&
loop->is_member(get_loop(use)) &&
!peel.test_set(use->_idx)) {
worklist.push(use);
}
}
}
}
uint i;
for(i = 0; i < loop->_body.size(); i++ ) {
Node *n = loop->_body.at(i);
Node *n_c = has_ctrl(n) ? get_ctrl(n) : n;
if (peel.test(n_c->_idx)) {
peel.set(n->_idx);
} else {
not_peel.set(n->_idx);
}
}
scheduled_nodelist(loop, peel, peel_list );
assert(is_valid_loop_partition(loop, peel, peel_list, not_peel), "bad partition");
uint old_phi_cnt = 0;
for (DUIterator_Fast jmax, j = head->fast_outs(jmax); j < jmax; j++) {
Node* use = head->fast_out(j);
if (use->is_Phi()) old_phi_cnt++;
}
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("\npeeled list");
}
#endif
uint new_phi_cnt = 0;
uint cloned_for_outside_use = 0;
for (i = 0; i < peel_list.size();) {
Node* n = peel_list.at(i);
#if !defined(PRODUCT)
if (TracePartialPeeling) n->dump();
#endif
bool incr = true;
if ( !n->is_CFG() ) {
if ( has_use_in_set(n, not_peel) ) {
if ( !has_use_internal_to_set(n, peel, loop) ) {
if ( n->in(0) == NULL && !n->is_Load() && !n->is_CMove() ) {
cloned_for_outside_use += clone_for_use_outside_loop( loop, n, worklist );
sink_list.push(n);
peel >>= n->_idx; // delete n from peel set.
not_peel <<= n->_idx; // add n to not_peel set.
peel_list.remove(i);
incr = false;
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("sink to not_peeled region: %d newbb: %d",
n->_idx, get_ctrl(n)->_idx);
}
#endif
}
} else {
clone_for_special_use_inside_loop( loop, n, not_peel, sink_list, worklist );
new_phi_cnt++;
}
}
}
if (incr) i++;
}
if (new_phi_cnt > old_phi_cnt + PartialPeelNewPhiDelta) {
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("\nToo many new phis: %d old %d new cmpi: %c",
new_phi_cnt, old_phi_cnt, new_peel_if != NULL?'T':'F');
}
#endif
if (new_peel_if != NULL) {
remove_cmpi_loop_exit(new_peel_if, loop);
}
assert(!head->is_partial_peel_loop(), "not partial peeled");
head->mark_partial_peel_failed();
if (cloned_for_outside_use > 0) {
C->set_major_progress();
return true;
}
return false;
}
LoopNode* new_head = new (C) LoopNode(last_peel, last_peel);
new_head->set_unswitch_count(head->unswitch_count()); // Preserve
_igvn.register_new_node_with_optimizer(new_head);
assert(first_not_peeled->in(0) == last_peel, "last_peel <- first_not_peeled");
first_not_peeled->set_req(0, new_head);
set_loop(new_head, loop);
loop->_body.push(new_head);
not_peel.set(new_head->_idx);
set_idom(new_head, last_peel, dom_depth(first_not_peeled));
set_idom(first_not_peeled, new_head, dom_depth(first_not_peeled));
while (sink_list.size() > 0) {
Node* n = sink_list.pop();
set_ctrl(n, new_head);
}
assert(is_valid_loop_partition(loop, peel, peel_list, not_peel), "bad partition");
clone_loop( loop, old_new, dd );
const uint clone_exit_idx = 1;
const uint orig_exit_idx = 2;
assert(is_valid_clone_loop_form( loop, peel_list, orig_exit_idx, clone_exit_idx ), "bad clone loop");
Node* head_clone = old_new[head->_idx];
LoopNode* new_head_clone = old_new[new_head->_idx]->as_Loop();
Node* orig_tail_clone = head_clone->in(2);
for(i = 0; i < peel_list.size(); i++ ) {
Node *def = peel_list.at(i);
if (!def->is_CFG()) {
for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) {
Node *use = def->fast_out(j);
if (has_node(use) && use->in(0) != C->top() &&
(!peel.test(use->_idx) ||
(use->is_Phi() && use->in(0) == head)) ) {
worklist.push(use);
}
}
while( worklist.size() ) {
Node *use = worklist.pop();
for (uint j = 1; j < use->req(); j++) {
Node* n = use->in(j);
if (n == def) {
Node* use_c = has_ctrl(use) ? get_ctrl(use) : use;
if ( loop->is_member(get_loop( use_c )) ) {
if (old_new[use->_idx] != NULL) { // null for dead code
Node* use_clone = old_new[use->_idx];
_igvn.replace_input_of(use, j, C->top());
insert_phi_for_loop( use_clone, j, old_new[def->_idx], def, new_head_clone );
}
} else {
assert(is_valid_clone_loop_exit_use(loop, use, orig_exit_idx), "clone loop format");
Node* lp_if = use_c->in(orig_exit_idx)->in(0);
if (not_peel.test(lp_if->_idx)) {
assert(j == orig_exit_idx, "use from original loop");
insert_phi_for_loop( use, clone_exit_idx, old_new[def->_idx], def, new_head_clone );
}
}
}
}
}
}
}
for(i = 0; i < loop->_body.size(); i++ ) {
Node *n = loop->_body.at(i);
if (!n->is_CFG() && n->in(0) != NULL &&
not_peel.test(n->_idx) && peel.test(n->in(0)->_idx)) {
Node* n_clone = old_new[n->_idx];
_igvn.replace_input_of(n_clone, 0, new_head_clone);
}
}
_igvn.replace_input_of(new_head_clone, LoopNode::LoopBackControl, last_peel);
_igvn.rehash_node_delayed(new_head); // Multiple edge updates:
new_head->set_req(LoopNode::EntryControl, C->top()); // use rehash_node_delayed / set_req instead of
new_head->set_req(LoopNode::LoopBackControl, C->top()); // multiple replace_input_of calls
_igvn.rehash_node_delayed(head); // Multiple edge updates
head->set_req(LoopNode::EntryControl, head_clone->in(LoopNode::LoopBackControl));
head->set_req(LoopNode::LoopBackControl, C->top());
_igvn.replace_input_of(head_clone, LoopNode::LoopBackControl, C->top());
for (DUIterator_Fast kmax, k = head->fast_outs(kmax); k < kmax; k++) {
Node* use = head->fast_out(k);
if (use->is_Phi() && use->outcnt() > 0) {
Node* use_clone = old_new[use->_idx];
_igvn.rehash_node_delayed(use); // Multiple edge updates
use->set_req(LoopNode::EntryControl, use_clone->in(LoopNode::LoopBackControl));
use->set_req(LoopNode::LoopBackControl, C->top());
_igvn.replace_input_of(use_clone, LoopNode::LoopBackControl, C->top());
}
}
set_idom(head, orig_tail_clone, dd);
recompute_dom_depth();
new_head_clone->set_partial_peel_loop();
C->set_major_progress();
loop->record_for_igvn();
#if !defined(PRODUCT)
if (TracePartialPeeling) {
tty->print_cr("\nafter partial peel one iteration");
Node_List wl(area);
Node* t = last_peel;
while (true) {
wl.push(t);
if (t == head_clone) break;
t = idom(t);
}
while (wl.size() > 0) {
Node* tt = wl.pop();
if (tt == head) tty->print_cr("orig head");
else if (tt == new_head_clone) tty->print_cr("new head");
else if (tt == head_clone) tty->print_cr("clone head");
tt->dump();
}
}
#endif
return true;
}
void PhaseIdealLoop::reorg_offsets(IdealLoopTree *loop) {
if (!loop->_head->is_CountedLoop())
return;
if (!loop->_head->as_Loop()->is_valid_counted_loop())
return;
CountedLoopNode *cl = loop->_head->as_CountedLoop();
CountedLoopEndNode *cle = cl->loopexit();
Node *exit = cle->proj_out(false);
Node *phi = cl->phi();
bool progress = true;
while (progress) {
progress = false;
for (DUIterator_Fast imax, i = phi->fast_outs(imax); i < imax; i++) {
Node* use = phi->fast_out(i); // User of trip-counter
if (!has_ctrl(use)) continue;
Node *u_ctrl = get_ctrl(use);
if (use->is_Phi()) {
u_ctrl = NULL;
for (uint j = 1; j < use->req(); j++)
if (use->in(j) == phi)
u_ctrl = dom_lca(u_ctrl, use->in(0)->in(j));
}
IdealLoopTree *u_loop = get_loop(u_ctrl);
if (u_loop == loop) continue;
if (loop->is_member(u_loop)) continue;
if (dom_lca(exit, u_ctrl) != exit) continue;
Node *opaq = new (C) Opaque2Node( C, cle->incr() );
register_new_node(opaq, exit);
Node *neg_stride = _igvn.intcon(-cle->stride_con());
set_ctrl(neg_stride, C->root());
Node *post = new (C) AddINode( opaq, neg_stride);
register_new_node(post, exit);
_igvn.rehash_node_delayed(use);
for (uint j = 1; j < use->req(); j++) {
if (use->in(j) == phi)
use->set_req(j, post);
}
progress = true;
break;
}
}
}
C:\hotspot-69087d08d473\src\share\vm/opto/loopPredicate.cpp
#include "precompiled.hpp"
#include "opto/loopnode.hpp"
#include "opto/addnode.hpp"
#include "opto/callnode.hpp"
#include "opto/connode.hpp"
#include "opto/loopnode.hpp"
#include "opto/matcher.hpp"
#include "opto/mulnode.hpp"
#include "opto/rootnode.hpp"
#include "opto/subnode.hpp"
void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred) {
assert(n->is_CFG(), "must be control node");
_igvn.register_new_node_with_optimizer(n);
loop->_body.push(n);
set_loop(n, loop);
if (_idom != NULL) {
set_idom(n, pred, dom_depth(pred));
}
}
ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
Deoptimization::DeoptReason reason) {
assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
IfNode* iff = cont_proj->in(0)->as_If();
ProjNode *uncommon_proj = iff->proj_out(1 - cont_proj->_con);
Node *rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
uint proj_index = 1; // region's edge corresponding to uncommon_proj
if (!rgn->is_Region()) { // create a region to guard the call
assert(rgn->is_Call(), "must be call uct");
CallNode* call = rgn->as_Call();
IdealLoopTree* loop = get_loop(call);
rgn = new (C) RegionNode(1);
rgn->add_req(uncommon_proj);
register_control(rgn, loop, uncommon_proj);
_igvn.hash_delete(call);
call->set_req(0, rgn);
if (_idom != NULL) {
set_idom(call, rgn, dom_depth(rgn));
}
} else {
for (; proj_index < rgn->req(); proj_index++)
if (rgn->in(proj_index) == uncommon_proj) break;
assert(proj_index < rgn->req(), "sanity");
}
Node* entry = iff->in(0);
if (new_entry != NULL) {
entry = new_entry;
}
IdealLoopTree* lp = get_loop(entry);
IfNode *new_iff = iff->clone()->as_If();
new_iff->set_req(0, entry);
register_control(new_iff, lp, entry);
Node *if_cont = new (C) IfTrueNode(new_iff);
Node *if_uct = new (C) IfFalseNode(new_iff);
if (cont_proj->is_IfFalse()) {
Node* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
}
register_control(if_cont, lp, new_iff);
register_control(if_uct, get_loop(rgn), new_iff);
_igvn.hash_delete(rgn);
rgn->add_req(if_uct);
if (_idom != NULL) {
Node* ridom = idom(rgn);
Node* nrdom = dom_lca(ridom, new_iff);
set_idom(rgn, nrdom, dom_depth(rgn));
}
assert(rgn->in(rgn->req() -1) == if_uct, "new edge should be last");
bool has_phi = false;
for (DUIterator_Fast imax, i = rgn->fast_outs(imax); i < imax; i++) {
Node* use = rgn->fast_out(i);
if (use->is_Phi() && use->outcnt() > 0) {
assert(use->in(0) == rgn, "");
_igvn.rehash_node_delayed(use);
use->add_req(use->in(proj_index));
has_phi = true;
}
}
assert(!has_phi || rgn->req() > 3, "no phis when region is created");
if (new_entry == NULL) {
_igvn.hash_delete(iff);
iff->set_req(0, if_cont);
if (_idom != NULL) {
set_idom(iff, if_cont, dom_depth(iff));
}
}
return if_cont->as_Proj();
}
ProjNode* PhaseIterGVN::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
Deoptimization::DeoptReason reason) {
assert(new_entry != 0, "only used for clone predicate");
assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
IfNode* iff = cont_proj->in(0)->as_If();
ProjNode *uncommon_proj = iff->proj_out(1 - cont_proj->_con);
Node *rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
uint proj_index = 1; // region's edge corresponding to uncommon_proj
if (!rgn->is_Region()) { // create a region to guard the call
assert(rgn->is_Call(), "must be call uct");
CallNode* call = rgn->as_Call();
rgn = new (C) RegionNode(1);
register_new_node_with_optimizer(rgn);
rgn->add_req(uncommon_proj);
hash_delete(call);
call->set_req(0, rgn);
} else {
for (; proj_index < rgn->req(); proj_index++)
if (rgn->in(proj_index) == uncommon_proj) break;
assert(proj_index < rgn->req(), "sanity");
}
IfNode *new_iff = iff->clone()->as_If();
new_iff->set_req(0, new_entry);
register_new_node_with_optimizer(new_iff);
Node *if_cont = new (C) IfTrueNode(new_iff);
Node *if_uct = new (C) IfFalseNode(new_iff);
if (cont_proj->is_IfFalse()) {
Node* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
}
register_new_node_with_optimizer(if_cont);
register_new_node_with_optimizer(if_uct);
hash_delete(rgn);
rgn->add_req(if_uct);
assert(rgn->in(rgn->req() -1) == if_uct, "new edge should be last");
bool has_phi = false;
for (DUIterator_Fast imax, i = rgn->fast_outs(imax); i < imax; i++) {
Node* use = rgn->fast_out(i);
if (use->is_Phi() && use->outcnt() > 0) {
rehash_node_delayed(use);
use->add_req(use->in(proj_index));
has_phi = true;
}
}
assert(!has_phi || rgn->req() > 3, "no phis when region is created");
return if_cont->as_Proj();
}
ProjNode* PhaseIdealLoop::clone_predicate(ProjNode* predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason,
PhaseIdealLoop* loop_phase,
PhaseIterGVN* igvn) {
ProjNode* new_predicate_proj;
if (loop_phase != NULL) {
new_predicate_proj = loop_phase->create_new_if_for_predicate(predicate_proj, new_entry, reason);
} else {
new_predicate_proj = igvn->create_new_if_for_predicate(predicate_proj, new_entry, reason);
}
IfNode* iff = new_predicate_proj->in(0)->as_If();
Node* ctrl = iff->in(0);
assert(predicate_proj->in(0)->in(1)->in(1)->Opcode()==Op_Opaque1, "must be");
Node* opq = new (igvn->C) Opaque1Node(igvn->C, predicate_proj->in(0)->in(1)->in(1)->in(1));
igvn->C->add_predicate_opaq(opq);
Node* bol = new (igvn->C) Conv2BNode(opq);
if (loop_phase != NULL) {
loop_phase->register_new_node(opq, ctrl);
loop_phase->register_new_node(bol, ctrl);
} else {
igvn->register_new_node_with_optimizer(opq);
igvn->register_new_node_with_optimizer(bol);
}
igvn->hash_delete(iff);
iff->set_req(1, bol);
return new_predicate_proj;
}
Node* PhaseIterGVN::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) {
return PhaseIdealLoop::clone_loop_predicates(old_entry, new_entry, clone_limit_check, NULL, this);
}
Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) {
return clone_loop_predicates(old_entry, new_entry, clone_limit_check, this, &this->_igvn);
}
Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry,
bool clone_limit_check,
PhaseIdealLoop* loop_phase,
PhaseIterGVN* igvn) {
#ifdef ASSERT
if (new_entry == NULL || !(new_entry->is_Proj() || new_entry->is_Region() || new_entry->is_SafePoint())) {
if (new_entry != NULL)
new_entry->dump();
assert(false, "not IfTrue, IfFalse, Region or SafePoint");
}
#endif
Node* entry = old_entry;
ProjNode* limit_check_proj = NULL;
if (LoopLimitCheck) {
limit_check_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (limit_check_proj != NULL) {
entry = entry->in(0)->in(0);
}
}
if (UseLoopPredicate) {
ProjNode* predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (predicate_proj != NULL) { // right pattern that can be used by loop predication
new_entry = clone_predicate(predicate_proj, new_entry,
Deoptimization::Reason_predicate,
loop_phase, igvn);
assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone predicate");
if (TraceLoopPredicate) {
tty->print("Loop Predicate cloned: ");
debug_only( new_entry->in(0)->dump(); )
}
}
}
if (limit_check_proj != NULL && clone_limit_check) {
new_entry = clone_predicate(limit_check_proj, new_entry,
Deoptimization::Reason_loop_limit_check,
loop_phase, igvn);
assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone limit check");
if (TraceLoopLimitCheck) {
tty->print("Loop Limit Check cloned: ");
debug_only( new_entry->in(0)->dump(); )
}
}
return new_entry;
}
Node* PhaseIdealLoop::skip_loop_predicates(Node* entry) {
Node* predicate = NULL;
if (LoopLimitCheck) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate != NULL) {
entry = entry->in(0)->in(0);
}
}
if (UseLoopPredicate) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (predicate != NULL) { // right pattern that can be used by loop predication
IfNode* iff = entry->in(0)->as_If();
ProjNode* uncommon_proj = iff->proj_out(1 - entry->as_Proj()->_con);
Node* rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
entry = entry->in(0)->in(0);
while (entry != NULL && entry->is_Proj() && entry->in(0)->is_If()) {
uncommon_proj = entry->in(0)->as_If()->proj_out(1 - entry->as_Proj()->_con);
if (uncommon_proj->unique_ctrl_out() != rgn)
break;
entry = entry->in(0)->in(0);
}
}
}
return entry;
}
ProjNode* PhaseIdealLoop::find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason) {
if (start_c == NULL || !start_c->is_Proj())
return NULL;
if (start_c->as_Proj()->is_uncommon_trap_if_pattern(reason)) {
return start_c->as_Proj();
}
return NULL;
}
Node* PhaseIdealLoop::find_predicate(Node* entry) {
Node* predicate = NULL;
if (LoopLimitCheck) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate != NULL) { // right pattern that can be used by loop predication
return entry;
}
}
if (UseLoopPredicate) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (predicate != NULL) { // right pattern that can be used by loop predication
return entry;
}
}
return NULL;
}
class Invariance : public StackObj {
VectorSet _visited, _invariant;
Node_Stack _stack;
VectorSet _clone_visited;
Node_List _old_new; // map of old to new (clone)
IdealLoopTree* _lpt;
PhaseIdealLoop* _phase;
void visit(Node* use, Node* n) {
if (_lpt->is_invariant(n)) { // known invariant
_invariant.set(n->_idx);
} else if (!n->is_CFG()) {
Node *n_ctrl = _phase->ctrl_or_self(n);
Node *u_ctrl = _phase->ctrl_or_self(use); // self if use is a CFG
if (_phase->is_dominator(n_ctrl, u_ctrl)) {
_stack.push(n, n->in(0) == NULL ? 1 : 0);
}
}
}
void compute_invariance(Node* n) {
assert(_visited.test(n->_idx), "must be");
visit(n, n);
while (_stack.is_nonempty()) {
Node* n = _stack.node();
uint idx = _stack.index();
if (idx == n->req()) { // all inputs are processed
_stack.pop();
bool all_inputs_invariant = true;
for (uint i = 0; i < n->req(); i++) {
Node* in = n->in(i);
if (in == NULL) continue;
assert(_visited.test(in->_idx), "must have visited input");
if (!_invariant.test(in->_idx)) { // bad guy
all_inputs_invariant = false;
break;
}
}
if (all_inputs_invariant) {
if (n->is_CFG() || n->depends_only_on_test() || n->in(0) == NULL || !_phase->is_member(_lpt, n->in(0))) {
_invariant.set(n->_idx); // I am a invariant too
}
}
} else { // process next input
_stack.set_index(idx + 1);
Node* m = n->in(idx);
if (m != NULL && !_visited.test_set(m->_idx)) {
visit(n, m);
}
}
}
}
void clone_visit(Node* n) {
assert(_invariant.test(n->_idx), "must be invariant");
if (_lpt->is_invariant(n)) { // known invariant
_old_new.map(n->_idx, n);
} else { // to be cloned
assert(!n->is_CFG(), "should not see CFG here");
_stack.push(n, n->in(0) == NULL ? 1 : 0);
}
}
void clone_nodes(Node* n, Node* ctrl) {
clone_visit(n);
while (_stack.is_nonempty()) {
Node* n = _stack.node();
uint idx = _stack.index();
if (idx == n->req()) { // all inputs processed, clone n!
_stack.pop();
Node* n_cl = n->clone();
_old_new.map(n->_idx, n_cl);
_phase->register_new_node(n_cl, ctrl);
for (uint i = 0; i < n->req(); i++) {
Node* in = n_cl->in(i);
if (in == NULL) continue;
n_cl->set_req(i, _old_new[in->_idx]);
}
} else { // process next input
_stack.set_index(idx + 1);
Node* m = n->in(idx);
if (m != NULL && !_clone_visited.test_set(m->_idx)) {
clone_visit(m); // visit the input
}
}
}
}
public:
Invariance(Arena* area, IdealLoopTree* lpt) :
_lpt(lpt), _phase(lpt->_phase),
_visited(area), _invariant(area), _stack(area, 10 /* guess */),
_clone_visited(area), _old_new(area)
{
Node* head = _lpt->_head;
Node* entry = head->in(LoopNode::EntryControl);
if (entry->outcnt() != 1) {
Unique_Node_List wq;
wq.push(entry);
for (uint next = 0; next < wq.size(); ++next) {
Node *n = wq.at(next);
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
Node* u = n->fast_out(i);
if (!u->is_CFG()) {
Node* c = _phase->get_ctrl(u);
if (_lpt->is_member(_phase->get_loop(c)) || _phase->is_dominator(c, head)) {
_visited.set(u->_idx);
wq.push(u);
}
}
}
}
}
}
void map_ctrl(Node* old, Node* n) {
assert(old->is_CFG() && n->is_CFG(), "must be");
_old_new.map(old->_idx, n); // "clone" of old is n
_invariant.set(old->_idx); // old is invariant
_clone_visited.set(old->_idx);
}
bool is_invariant(Node* n) {
if (!_visited.test_set(n->_idx))
compute_invariance(n);
return (_invariant.test(n->_idx) != 0);
}
Node* clone(Node* n, Node* ctrl) {
assert(ctrl->is_CFG(), "must be");
assert(_invariant.test(n->_idx), "must be an invariant");
if (!_clone_visited.test(n->_idx))
clone_nodes(n, ctrl);
return _old_new[n->_idx];
}
};
bool IdealLoopTree::is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar) const {
if (!is_loop_exit(iff)) {
return false;
}
if (!iff->in(1)->is_Bool()) {
return false;
}
const BoolNode *bol = iff->in(1)->as_Bool();
if (bol->_test._test != BoolTest::lt) {
return false;
}
if (!bol->in(1)->is_Cmp()) {
return false;
}
const CmpNode *cmp = bol->in(1)->as_Cmp();
if (cmp->Opcode() != Op_CmpU) {
return false;
}
Node* range = cmp->in(2);
if (range->Opcode() != Op_LoadRange) {
const TypeInt* tint = phase->_igvn.type(range)->isa_int();
if (tint == NULL || tint->empty() || tint->_lo < 0) {
return false;
}
}
if (!invar.is_invariant(range)) {
return false;
}
Node *iv = _head->as_CountedLoop()->phi();
int scale = 0;
Node *offset = NULL;
if (!phase->is_scaled_iv_plus_offset(cmp->in(1), iv, &scale, &offset)) {
return false;
}
if (offset && !invar.is_invariant(offset)) { // offset must be invariant
return false;
}
return true;
}
BoolNode* PhaseIdealLoop::rc_predicate(IdealLoopTree *loop, Node* ctrl,
int scale, Node* offset,
Node* init, Node* limit, jint stride,
Node* range, bool upper, bool &overflow) {
jint con_limit = limit->is_Con() ? limit->get_int() : 0;
jint con_init = init->is_Con() ? init->get_int() : 0;
jint con_offset = offset->is_Con() ? offset->get_int() : 0;
stringStream* predString = NULL;
if (TraceLoopPredicate) {
predString = new stringStream();
predString->print("rc_predicate ");
}
overflow = false;
Node* max_idx_expr = NULL;
const TypeInt* idx_type = TypeInt::INT;
if ((stride > 0) == (scale > 0) == upper) {
if (TraceLoopPredicate) {
if (limit->is_Con()) {
predString->print("(%d ", con_limit);
} else {
predString->print("(limit ");
}
predString->print("- %d) ", stride);
}
const TypeInt* limit_type = _igvn.type(limit)->isa_int();
jint limit_lo = limit_type->_lo;
jint limit_hi = limit_type->_hi;
jint res_lo = limit_lo - stride;
jint res_hi = limit_hi - stride;
if ((stride > 0 && (res_lo < limit_lo)) ||
(stride < 0 && (res_hi > limit_hi))) {
ConINode* con_stride = _igvn.intcon(stride);
set_ctrl(con_stride, C->root());
max_idx_expr = new (C) SubINode(limit, con_stride);
idx_type = TypeInt::make(limit_lo - stride, limit_hi - stride, limit_type->_widen);
} else {
overflow = true;
limit = new (C) ConvI2LNode(limit);
register_new_node(limit, ctrl);
ConLNode* con_stride = _igvn.longcon(stride);
set_ctrl(con_stride, C->root());
max_idx_expr = new (C) SubLNode(limit, con_stride);
}
register_new_node(max_idx_expr, ctrl);
} else {
if (TraceLoopPredicate) {
if (init->is_Con()) {
predString->print("%d ", con_init);
} else {
predString->print("init ");
}
}
idx_type = _igvn.type(init)->isa_int();
max_idx_expr = init;
}
if (scale != 1) {
ConNode* con_scale = _igvn.intcon(scale);
set_ctrl(con_scale, C->root());
if (TraceLoopPredicate) {
predString->print("* %d ", scale);
}
const TypeInt* scale_type = TypeInt::make(scale);
MulINode* mul = new (C) MulINode(max_idx_expr, con_scale);
idx_type = (TypeInt*)mul->mul_ring(idx_type, scale_type);
if (overflow || TypeInt::INT->higher_equal(idx_type)) {
mul->destruct();
if (!overflow) {
max_idx_expr = new (C) ConvI2LNode(max_idx_expr);
register_new_node(max_idx_expr, ctrl);
}
overflow = true;
con_scale = _igvn.longcon(scale);
set_ctrl(con_scale, C->root());
max_idx_expr = new (C) MulLNode(max_idx_expr, con_scale);
} else {
max_idx_expr = mul;
}
register_new_node(max_idx_expr, ctrl);
}
if (offset && (!offset->is_Con() || con_offset != 0)){
if (TraceLoopPredicate) {
if (offset->is_Con()) {
predString->print("+ %d ", con_offset);
} else {
predString->print("+ offset");
}
}
const TypeInt* offset_type = _igvn.type(offset)->isa_int();
jint lo = idx_type->_lo + offset_type->_lo;
jint hi = idx_type->_hi + offset_type->_hi;
if (overflow || (lo > hi) ||
((idx_type->_lo & offset_type->_lo) < 0 && lo >= 0) ||
((~(idx_type->_hi | offset_type->_hi)) < 0 && hi < 0)) {
if (!overflow) {
max_idx_expr = new (C) ConvI2LNode(max_idx_expr);
register_new_node(max_idx_expr, ctrl);
}
overflow = true;
offset = new (C) ConvI2LNode(offset);
register_new_node(offset, ctrl);
max_idx_expr = new (C) AddLNode(max_idx_expr, offset);
} else {
max_idx_expr = new (C) AddINode(max_idx_expr, offset);
}
register_new_node(max_idx_expr, ctrl);
}
CmpNode* cmp = NULL;
if (overflow) {
range = new (C) ConvI2LNode(range);
register_new_node(range, ctrl);
if (!Matcher::has_match_rule(Op_CmpUL)) {
ConINode* sign_pos = _igvn.intcon(BitsPerLong - 1);
set_ctrl(sign_pos, C->root());
Node* sign_bit_mask = new (C) RShiftLNode(max_idx_expr, sign_pos);
register_new_node(sign_bit_mask, ctrl);
max_idx_expr = new (C) OrLNode(max_idx_expr, sign_bit_mask);
register_new_node(max_idx_expr, ctrl);
ConLNode* remove_sign_mask = _igvn.longcon(max_jlong);
set_ctrl(remove_sign_mask, C->root());
max_idx_expr = new (C) AndLNode(max_idx_expr, remove_sign_mask);
register_new_node(max_idx_expr, ctrl);
cmp = new (C) CmpLNode(max_idx_expr, range);
} else {
cmp = new (C) CmpULNode(max_idx_expr, range);
}
} else {
cmp = new (C) CmpUNode(max_idx_expr, range);
}
register_new_node(cmp, ctrl);
BoolNode* bol = new (C) BoolNode(cmp, BoolTest::lt);
register_new_node(bol, ctrl);
if (TraceLoopPredicate) {
predString->print_cr("<u range");
tty->print("%s", predString->as_string());
}
return bol;
}
bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
if (!UseLoopPredicate) return false;
if (!loop->_head->is_Loop()) {
return false;
}
LoopNode* head = loop->_head->as_Loop();
if (head->unique_ctrl_out()->Opcode() == Op_NeverBranch) {
return false;
}
CountedLoopNode *cl = NULL;
if (head->is_valid_counted_loop()) {
cl = head->as_CountedLoop();
if (!cl->is_normal_loop()) return false;
BoolTest::mask bt = cl->loopexit()->test_trip();
if (bt != BoolTest::lt && bt != BoolTest::gt)
cl = NULL;
}
Node* entry = head->in(LoopNode::EntryControl);
ProjNode *predicate_proj = NULL;
if (LoopLimitCheck) {
predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate_proj != NULL)
entry = predicate_proj->in(0)->in(0);
}
predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (!predicate_proj) {
#ifndef PRODUCT
if (TraceLoopPredicate) {
tty->print("missing predicate:");
loop->dump_head();
head->dump(1);
}
#endif
return false;
}
ConNode* zero = _igvn.intcon(0);
set_ctrl(zero, C->root());
ResourceArea *area = Thread::current()->resource_area();
Invariance invar(area, loop);
Node_List if_proj_list(area);
Node *current_proj = loop->tail(); //start from tail
while (current_proj != head) {
if (loop == get_loop(current_proj) && // still in the loop ?
current_proj->is_Proj() && // is a projection ?
current_proj->in(0)->Opcode() == Op_If) { // is a if projection ?
if_proj_list.push(current_proj);
}
current_proj = idom(current_proj);
}
bool hoisted = false; // true if at least one proj is promoted
while (if_proj_list.size() > 0) {
ProjNode* new_predicate_proj = NULL;
ProjNode* proj = if_proj_list.pop()->as_Proj();
IfNode* iff = proj->in(0)->as_If();
if (!proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) {
if (loop->is_loop_exit(iff)) {
break;
} else {
continue;
}
}
Node* test = iff->in(1);
if (!test->is_Bool()){ //Conv2B, ...
continue;
}
BoolNode* bol = test->as_Bool();
if (invar.is_invariant(bol)) {
new_predicate_proj = create_new_if_for_predicate(predicate_proj, NULL,
Deoptimization::Reason_predicate);
Node* ctrl = new_predicate_proj->in(0)->as_If()->in(0);
BoolNode* new_predicate_bol = invar.clone(bol, ctrl)->as_Bool();
bool negated = false;
if (proj->_con != predicate_proj->_con) {
new_predicate_bol = new (C) BoolNode(new_predicate_bol->in(1), new_predicate_bol->_test.negate());
register_new_node(new_predicate_bol, ctrl);
negated = true;
}
IfNode* new_predicate_iff = new_predicate_proj->in(0)->as_If();
_igvn.hash_delete(new_predicate_iff);
new_predicate_iff->set_req(1, new_predicate_bol);
#ifndef PRODUCT
if (TraceLoopPredicate) {
tty->print("Predicate invariant if%s: %d ", negated ? " negated" : "", new_predicate_iff->_idx);
loop->dump_head();
} else if (TraceLoopOpts) {
tty->print("Predicate IC ");
loop->dump_head();
}
#endif
} else if ((cl != NULL) && (proj->_con == predicate_proj->_con) &&
loop->is_range_check_if(iff, this, invar)) {
const Node* cmp = bol->in(1)->as_Cmp();
Node* idx = cmp->in(1);
assert(!invar.is_invariant(idx), "index is variant");
Node* rng = cmp->in(2);
assert(rng->Opcode() == Op_LoadRange || _igvn.type(rng)->is_int()->_lo >= 0, "must be");
assert(invar.is_invariant(rng), "range must be invariant");
int scale = 1;
Node* offset = zero;
bool ok = is_scaled_iv_plus_offset(idx, cl->phi(), &scale, &offset);
assert(ok, "must be index expression");
Node* init = cl->init_trip();
Node* limit = exact_limit(loop);
int stride = cl->stride()->get_int();
ProjNode* lower_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate);
ProjNode* upper_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate);
assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate");
Node *ctrl = lower_bound_proj->in(0)->as_If()->in(0);
rng = invar.clone(rng, ctrl);
if (offset && offset != zero) {
assert(invar.is_invariant(offset), "offset must be loop invariant");
offset = invar.clone(offset, ctrl);
}
bool overflow = false;
Node* lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false, overflow);
IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If();
_igvn.hash_delete(lower_bound_iff);
lower_bound_iff->set_req(1, lower_bound_bol);
if (TraceLoopPredicate) tty->print_cr("lower bound check if: %d", lower_bound_iff->_idx);
Node* upper_bound_bol = rc_predicate(loop, lower_bound_proj, scale, offset, init, limit, stride, rng, true, overflow);
IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If();
_igvn.hash_delete(upper_bound_iff);
upper_bound_iff->set_req(1, upper_bound_bol);
if (TraceLoopPredicate) tty->print_cr("upper bound check if: %d", lower_bound_iff->_idx);
new_predicate_proj = upper_bound_proj;
#ifndef PRODUCT
if (TraceLoopOpts && !TraceLoopPredicate) {
tty->print("Predicate RC ");
loop->dump_head();
}
#endif
} else {
continue;
}
assert(new_predicate_proj != NULL, "sanity");
invar.map_ctrl(proj, new_predicate_proj); // so that invariance test can be appropriate
dominated_by( new_predicate_proj, iff, proj->_con != new_predicate_proj->_con );
hoisted = true;
C->set_major_progress();
} // end while
#ifndef PRODUCT
if (TraceLoopPredicate && hoisted) {
tty->print("Loop Predication Performed:");
loop->dump_head();
}
#endif
return hoisted;
}
bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) {
bool hoisted = false;
if (_child) {
hoisted = _child->loop_predication( phase);
}
if (!_irreducible && !tail()->is_top()) {
hoisted |= phase->loop_predication_impl(this);
}
if (_next) { //sibling
hoisted |= _next->loop_predication( phase);
}
return hoisted;
}
C:\hotspot-69087d08d473\src\share\vm/opto/loopTransform.cpp
#include "precompiled.hpp"
#include "compiler/compileLog.hpp"
#include "memory/allocation.inline.hpp"
#include "opto/addnode.hpp"
#include "opto/callnode.hpp"
#include "opto/connode.hpp"
#include "opto/divnode.hpp"
#include "opto/loopnode.hpp"
#include "opto/mulnode.hpp"
#include "opto/rootnode.hpp"
#include "opto/runtime.hpp"
#include "opto/subnode.hpp"
Node *IdealLoopTree::is_loop_exit(Node *iff) const {
if( iff->outcnt() != 2 ) return NULL; // Ignore partially dead tests
PhaseIdealLoop *phase = _phase;
if( !is_member(phase->get_loop( iff->raw_out(0) )) )
return iff->raw_out(0);
if( !is_member(phase->get_loop( iff->raw_out(1) )) )
return iff->raw_out(1);
return NULL;
}
void IdealLoopTree::record_for_igvn() {
for( uint i = 0; i < _body.size(); i++ ) {
Node *n = _body.at(i);
_phase->_igvn._worklist.push(n);
}
}
void IdealLoopTree::compute_exact_trip_count( PhaseIdealLoop *phase ) {
if (!_head->as_Loop()->is_valid_counted_loop()) {
return;
}
CountedLoopNode* cl = _head->as_CountedLoop();
cl->set_nonexact_trip_count();
if (!phase->is_member(this, phase->get_ctrl(cl->loopexit()->in(CountedLoopEndNode::TestValue))))
return; // Infinite loop
#ifdef ASSERT
BoolTest::mask bt = cl->loopexit()->test_trip();
assert(bt == BoolTest::lt || bt == BoolTest::gt ||
bt == BoolTest::ne, "canonical test is expected");
#endif
Node* init_n = cl->init_trip();
Node* limit_n = cl->limit();
if (init_n != NULL && init_n->is_Con() &&
limit_n != NULL && limit_n->is_Con()) {
int stride_con = cl->stride_con();
jlong init_con = cl->init_trip()->get_int();
jlong limit_con = cl->limit()->get_int();
int stride_m = stride_con - (stride_con > 0 ? 1 : -1);
jlong trip_count = (limit_con - init_con + stride_m)/stride_con;
if (trip_count > 0 && (julong)trip_count < (julong)max_juint) {
cl->set_exact_trip_count((uint)trip_count);
}
}
}
void IdealLoopTree::compute_profile_trip_cnt( PhaseIdealLoop *phase ) {
if (!_head->is_CountedLoop()) {
return;
}
CountedLoopNode* head = _head->as_CountedLoop();
if (head->profile_trip_cnt() != COUNT_UNKNOWN) {
return; // Already computed
}
float trip_cnt = (float)max_jint; // default is big
Node* back = head->in(LoopNode::LoopBackControl);
while (back != head) {
if ((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
back->in(0) &&
back->in(0)->is_If() &&
back->in(0)->as_If()->_fcnt != COUNT_UNKNOWN &&
back->in(0)->as_If()->_prob != PROB_UNKNOWN) {
break;
}
back = phase->idom(back);
}
if (back != head) {
assert((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
back->in(0), "if-projection exists");
IfNode* back_if = back->in(0)->as_If();
float loop_back_cnt = back_if->_fcnt * back_if->_prob;
float loop_exit_cnt = 0.0f;
for( uint i = 0; i < _body.size(); i++ ) {
Node *n = _body[i];
if( n->is_If() ) {
IfNode *iff = n->as_If();
if( iff->_fcnt != COUNT_UNKNOWN && iff->_prob != PROB_UNKNOWN ) {
Node *exit = is_loop_exit(iff);
if( exit ) {
float exit_prob = iff->_prob;
if (exit->Opcode() == Op_IfFalse) exit_prob = 1.0 - exit_prob;
if (exit_prob > PROB_MIN) {
float exit_cnt = iff->_fcnt * exit_prob;
loop_exit_cnt += exit_cnt;
}
}
}
}
}
if (loop_exit_cnt > 0.0f) {
trip_cnt = (loop_back_cnt + loop_exit_cnt) / loop_exit_cnt;
} else {
trip_cnt = loop_back_cnt;
}
}
#ifndef PRODUCT
if (TraceProfileTripCount) {
tty->print_cr("compute_profile_trip_cnt lp: %d cnt: %f\n", head->_idx, trip_cnt);
}
#endif
head->set_profile_trip_cnt(trip_cnt);
}
int IdealLoopTree::is_invariant_addition(Node* n, PhaseIdealLoop *phase) {
int op = n->Opcode();
if (op == Op_AddI || op == Op_SubI) {
bool in1_invar = this->is_invariant(n->in(1));
bool in2_invar = this->is_invariant(n->in(2));
if (in1_invar && !in2_invar) return 1;
if (!in1_invar && in2_invar) return 2;
}
return 0;
}
Node* IdealLoopTree::reassociate_add_sub(Node* n1, PhaseIdealLoop *phase) {
if (!n1->is_Add() && !n1->is_Sub() || n1->outcnt() == 0) return NULL;
if (is_invariant(n1)) return NULL;
int inv1_idx = is_invariant_addition(n1, phase);
if (!inv1_idx) return NULL;
if (n1->is_Add() && n1->in(2)->is_Con()) return NULL;
Node* inv1 = n1->in(inv1_idx);
Node* n2 = n1->in(3 - inv1_idx);
int inv2_idx = is_invariant_addition(n2, phase);
if (!inv2_idx) return NULL;
Node* x = n2->in(3 - inv2_idx);
Node* inv2 = n2->in(inv2_idx);
bool neg_x = n2->is_Sub() && inv2_idx == 1;
bool neg_inv2 = n2->is_Sub() && inv2_idx == 2;
bool neg_inv1 = n1->is_Sub() && inv1_idx == 2;
if (n1->is_Sub() && inv1_idx == 1) {
neg_x = !neg_x;
neg_inv2 = !neg_inv2;
}
Node* inv1_c = phase->get_ctrl(inv1);
Node* inv2_c = phase->get_ctrl(inv2);
Node* n_inv1;
if (neg_inv1) {
Node *zero = phase->_igvn.intcon(0);
phase->set_ctrl(zero, phase->C->root());
n_inv1 = new (phase->C) SubINode(zero, inv1);
phase->register_new_node(n_inv1, inv1_c);
} else {
n_inv1 = inv1;
}
Node* inv;
if (neg_inv2) {
inv = new (phase->C) SubINode(n_inv1, inv2);
} else {
inv = new (phase->C) AddINode(n_inv1, inv2);
}
phase->register_new_node(inv, phase->get_early_ctrl(inv));
Node* addx;
if (neg_x) {
addx = new (phase->C) SubINode(inv, x);
} else {
addx = new (phase->C) AddINode(x, inv);
}
phase->register_new_node(addx, phase->get_ctrl(x));
phase->_igvn.replace_node(n1, addx);
assert(phase->get_loop(phase->get_ctrl(n1)) == this, "");
_body.yank(n1);
return addx;
}
void IdealLoopTree::reassociate_invariants(PhaseIdealLoop *phase) {
for (int i = _body.size() - 1; i >= 0; i--) {
Node *n = _body.at(i);
for (int j = 0; j < 5; j++) {
Node* nn = reassociate_add_sub(n, phase);
if (nn == NULL) break;
n = nn; // again
};
}
}
bool IdealLoopTree::policy_peeling( PhaseIdealLoop *phase ) const {
Node *test = ((IdealLoopTree*)this)->tail();
int body_size = ((IdealLoopTree*)this)->_body.size();
if( body_size > 255 /* Prevent overflow for large body_size */
|| (body_size * body_size + phase->C->live_nodes()) > phase->C->max_node_limit() ) {
return false; // too large to safely clone
}
while( test != _head ) { // Scan till run off top of loop
if( test->is_If() ) { // Test?
Node *ctrl = phase->get_ctrl(test->in(1));
if (ctrl->is_top())
return false; // Found dead test on live IF? No peeling!
assert( test->Opcode() == Op_If || test->Opcode() == Op_CountedLoopEnd, "Check this code when new subtype is added");
if( !is_member(phase->get_loop(ctrl)) &&
is_loop_exit(test) )
return true; // Found reason to peel!
}
test = phase->idom(test);
}
return false;
}
void PhaseIdealLoop::peeled_dom_test_elim( IdealLoopTree *loop, Node_List &old_new ) {
bool progress = true;
while( progress ) {
progress = false; // Reset for next iteration
Node *prev = loop->_head->in(LoopNode::LoopBackControl);//loop->tail();
Node *test = prev->in(0);
while( test != loop->_head ) { // Scan till run off top of loop
int p_op = prev->Opcode();
if( (p_op == Op_IfFalse || p_op == Op_IfTrue) &&
test->is_If() && // Test?
!test->in(1)->is_Con() && // And not already obvious?
!loop->is_member(get_loop(get_ctrl(test->in(1))))){
for( uint i = 0; i < loop->_body.size(); i++ ) {
Node *n = loop->_body.at(i);
if( n->is_If() && n->in(1) == test->in(1) /*&& n != loop->tail()->in(0)*/ ) {
progress = true;
dominated_by( old_new[prev->_idx], n );
}
}
}
prev = test;
test = idom(test);
} // End of scan tests in loop
} // End of while( progress )
}
void PhaseIdealLoop::do_peeling( IdealLoopTree *loop, Node_List &old_new ) {
C->set_major_progress();
#ifndef PRODUCT
if (TraceLoopOpts) {
tty->print("Peel ");
loop->dump_head();
}
#endif
Node* head = loop->_head;
bool counted_loop = head->is_CountedLoop();
if (counted_loop) {
CountedLoopNode *cl = head->as_CountedLoop();
assert(cl->trip_count() > 0, "peeling a fully unrolled loop");
cl->set_trip_count(cl->trip_count() - 1);
if (cl->is_main_loop()) {
cl->set_normal_loop();
#ifndef PRODUCT
if (PrintOpto && VerifyLoopOptimizations) {
tty->print("Peeling a 'main' loop; resetting to 'normal' ");
loop->dump_head();
}
#endif
}
}
Node* entry = head->in(LoopNode::EntryControl);
clone_loop( loop, old_new, dom_depth(head) );
Node* new_entry = old_new[head->in(LoopNode::LoopBackControl)->_idx];
_igvn.hash_delete(head);
head->set_req(LoopNode::EntryControl, new_entry);
for (DUIterator_Fast jmax, j = head->fast_outs(jmax); j < jmax; j++) {
Node* old = head->fast_out(j);
if (old->in(0) == loop->_head && old->req() == 3 && old->is_Phi()) {
Node* new_exit_value = old_new[old->in(LoopNode::LoopBackControl)->_idx];
if (!new_exit_value ) // Backedge value is ALSO loop invariant?
new_exit_value = old->in(LoopNode::LoopBackControl);
_igvn.hash_delete(old);
old->set_req(LoopNode::EntryControl, new_exit_value);
}
}
Node* new_head = old_new[head->_idx];
_igvn.hash_delete(new_head);
new_head->set_req(LoopNode::LoopBackControl, C->top());
for (DUIterator_Fast j2max, j2 = new_head->fast_outs(j2max); j2 < j2max; j2++) {
Node* use = new_head->fast_out(j2);
if (use->in(0) == new_head && use->req() == 3 && use->is_Phi()) {
_igvn.hash_delete(use);
use->set_req(LoopNode::LoopBackControl, C->top());
}
}
int dd = dom_depth(head);
set_idom(head, head->in(1), dd);
for (uint j3 = 0; j3 < loop->_body.size(); j3++) {
Node *old = loop->_body.at(j3);
Node *nnn = old_new[old->_idx];
if (!has_ctrl(nnn))
set_idom(nnn, idom(nnn), dd-1);
}
peeled_dom_test_elim(loop,old_new);
loop->record_for_igvn();
}
#define EMPTY_LOOP_SIZE 7 // number of nodes in an empty loop
bool IdealLoopTree::policy_maximally_unroll( PhaseIdealLoop *phase ) const {
CountedLoopNode *cl = _head->as_CountedLoop();
assert(cl->is_normal_loop(), "");
if (!cl->is_valid_counted_loop())
return false; // Malformed counted loop
if (!cl->has_exact_trip_count()) {
return false;
}
uint trip_count = cl->trip_count();
assert(trip_count > 1, "one iteration loop should be optimized out already");
assert(trip_count < max_juint, "exact trip_count should be less than max_uint.");
uint body_size = _body.size();
uint unroll_limit = (uint)LoopUnrollLimit * 4;
assert( (intx)unroll_limit == LoopUnrollLimit * 4, "LoopUnrollLimit must fit in 32bits");
if (trip_count > unroll_limit || body_size > unroll_limit) {
return false;
}
if (trip_count <= 3)
return true;
uint new_body_size = EMPTY_LOOP_SIZE + (body_size - EMPTY_LOOP_SIZE) * trip_count;
uint tst_body_size = (new_body_size - EMPTY_LOOP_SIZE) / trip_count + EMPTY_LOOP_SIZE;
if (body_size != tst_body_size) // Check for int overflow
return false;
if (new_body_size > unroll_limit ||
new_body_size >= phase->C->max_node_limit() - phase->C->live_nodes()) {
return false;
}
for (uint k = 0; k < _body.size(); k++) {
Node* n = _body.at(k);
switch (n->Opcode()) {
case Op_StrComp:
case Op_StrEquals:
case Op_StrIndexOf:
case Op_EncodeISOArray:
case Op_AryEq: {
return false;
}
#if INCLUDE_RTM_OPT
case Op_FastLock:
case Op_FastUnlock: {
if (UseRTMLocking) {
return false;
}
}
#endif
} // switch
}
return true; // Do maximally unroll
}
bool IdealLoopTree::policy_unroll( PhaseIdealLoop *phase ) const {
CountedLoopNode *cl = _head->as_CountedLoop();
assert(cl->is_normal_loop() || cl->is_main_loop(), "");
if (!cl->is_valid_counted_loop())
return false; // Malformed counted loop
if (cl->trip_count() <= (uint)(cl->is_normal_loop() ? 2 : 1)) return false;
int future_unroll_ct = cl->unrolled_count() * 2;
if (future_unroll_ct > LoopMaxUnroll) return false;
if (abs(cl->stride_con()) > (1<<2)*future_unroll_ct) return false;
if (UnrollLimitForProfileCheck > 0 &&
cl->profile_trip_cnt() != COUNT_UNKNOWN &&
future_unroll_ct > UnrollLimitForProfileCheck &&
(float)future_unroll_ct > cl->profile_trip_cnt() - 1.0) {
return false;
}
if (UseSuperWord && cl->node_count_before_unroll() > 0 &&
future_unroll_ct > LoopUnrollMin &&
(future_unroll_ct - 1) * 10.0 > cl->profile_trip_cnt() &&
1.2 * cl->node_count_before_unroll() < (double)_body.size()) {
return false;
}
Node *init_n = cl->init_trip();
Node *limit_n = cl->limit();
int stride_con = cl->stride_con();
if (init_n == NULL || !init_n->is_Con() ||
limit_n == NULL || !limit_n->is_Con()) {
Node* phi = cl->phi();
if (phi != NULL) {
assert(phi->is_Phi() && phi->in(0) == _head, "Counted loop should have iv phi.");
const TypeInt* iv_type = phase->_igvn.type(phi)->is_int();
int next_stride = stride_con * 2; // stride after this unroll
if (next_stride > 0) {
if (iv_type->_lo + next_stride <= iv_type->_lo || // overflow
iv_type->_lo + next_stride > iv_type->_hi) {
return false; // over-unrolling
}
} else if (next_stride < 0) {
if (iv_type->_hi + next_stride >= iv_type->_hi || // overflow
iv_type->_hi + next_stride < iv_type->_lo) {
return false; // over-unrolling
}
}
}
}
const TypeInt* limit_type = phase->_igvn.type(limit_n)->is_int();
if (stride_con > 0 && ((limit_type->_hi - stride_con) >= limit_type->_hi) ||
stride_con < 0 && ((limit_type->_lo - stride_con) <= limit_type->_lo))
return false; // overflow
uint body_size = _body.size();
int xors_in_loop = 0;
for (uint k = 0; k < _body.size(); k++) {
Node* n = _body.at(k);
switch (n->Opcode()) {
case Op_XorI: xors_in_loop++; break; // CRC32 java code
case Op_ModL: body_size += 30; break;
case Op_DivL: body_size += 30; break;
case Op_MulL: body_size += 10; break;
case Op_StrComp:
case Op_StrEquals:
case Op_StrIndexOf:
case Op_EncodeISOArray:
case Op_AryEq: {
return false;
}
#if INCLUDE_RTM_OPT
case Op_FastLock:
case Op_FastUnlock: {
if (UseRTMLocking) {
return false;
}
}
#endif
} // switch
}
if (body_size > (uint)LoopUnrollLimit) {
if (xors_in_loop >= 4 && body_size < (uint)LoopUnrollLimit*4) return true;
return false;
}
return true;
}
bool IdealLoopTree::policy_align( PhaseIdealLoop *phase ) const {
return false;
}
bool IdealLoopTree::policy_range_check( PhaseIdealLoop *phase ) const {
if (!RangeCheckElimination) return false;
CountedLoopNode *cl = _head->as_CountedLoop();
if (cl->is_main_no_pre_loop()) return false; // Disallowed for now.
Node *trip_counter = cl->phi();
for (uint i = 0; i < _body.size(); i++) {
Node *iff = _body[i];
if (iff->Opcode() == Op_If) { // Test?
Node *bol = iff->in(1);
if (bol->req() != 2) continue; // dead constant test
if (!bol->is_Bool()) {
assert(UseLoopPredicate && bol->Opcode() == Op_Conv2B, "predicate check only");
continue;
}
if (bol->as_Bool()->_test._test == BoolTest::ne)
continue; // not RC
Node *cmp = bol->in(1);
Node *rc_exp = cmp->in(1);
Node *limit = cmp->in(2);
Node *limit_c = phase->get_ctrl(limit);
if( limit_c == phase->C->top() )
return false; // Found dead test on live IF? No RCE!
if( is_member(phase->get_loop(limit_c) ) ) {
rc_exp = cmp->in(2);
limit = cmp->in(1);
limit_c = phase->get_ctrl(limit);
if( is_member(phase->get_loop(limit_c) ) )
continue; // Both inputs are loop varying; cannot RCE
}
if (!phase->is_scaled_iv_plus_offset(rc_exp, trip_counter, NULL, NULL)) {
continue;
}
if( is_loop_exit(iff) )
return true; // Found reason to split iterations
} // End of is IF
}
return false;
}
bool IdealLoopTree::policy_peel_only( PhaseIdealLoop *phase ) const {
for( uint i = 0; i < _body.size(); i++ )
if( _body[i]->is_Mem() )
return false;
return true;
}
Node *PhaseIdealLoop::clone_up_backedge_goo( Node *back_ctrl, Node *preheader_ctrl, Node *n, VectorSet &visited, Node_Stack &clones ) {
if( get_ctrl(n) != back_ctrl ) return n;
if (visited.test_set(n->_idx)) {
Node *x = clones.find(n->_idx);
if (x != NULL)
return x;
return n;
}
Node *x = NULL; // If required, a clone of 'n'
if( n->in(0) && n->in(0) == back_ctrl ) {
assert(clones.find(n->_idx) == NULL, "dead loop");
x = n->clone(); // Clone a copy of 'n' to preheader
clones.push(x, n->_idx);
x->set_req( 0, preheader_ctrl ); // Fix x's control input to preheader
}
for( uint i = 1; i < n->req(); i++ ) {
Node *g = clone_up_backedge_goo( back_ctrl, preheader_ctrl, n->in(i), visited, clones );
if( g != n->in(i) ) {
if( !x ) {
assert(clones.find(n->_idx) == NULL, "dead loop");
x = n->clone();
clones.push(x, n->_idx);
}
x->set_req(i, g);
}
}
if( x ) { // x can legally float to pre-header location
register_new_node( x, preheader_ctrl );
return x;
} else { // raise n to cover LCA of uses
set_ctrl( n, find_non_split_ctrl(back_ctrl->in(0)) );
}
return n;
}
bool PhaseIdealLoop::cast_incr_before_loop(Node* incr, Node* ctrl, Node* loop) {
Node* castii = new (C) CastIINode(incr, TypeInt::INT, true);
castii->set_req(0, ctrl);
register_new_node(castii, ctrl);
for (DUIterator_Fast imax, i = incr->fast_outs(imax); i < imax; i++) {
Node* n = incr->fast_out(i);
if (n->is_Phi() && n->in(0) == loop) {
int nrep = n->replace_edge(incr, castii);
return true;
}
}
return false;
}
void PhaseIdealLoop::insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_new, bool peel_only ) {
#ifndef PRODUCT
if (TraceLoopOpts) {
if (peel_only)
tty->print("PeelMainPost ");
else
tty->print("PreMainPost ");
loop->dump_head();
}
#endif
C->set_major_progress();
CountedLoopNode *main_head = loop->_head->as_CountedLoop();
assert( main_head->is_normal_loop(), "" );
CountedLoopEndNode *main_end = main_head->loopexit();
guarantee(main_end != NULL, "no loop exit node");
assert( main_end->outcnt() == 2, "1 true, 1 false path only" );
uint dd_main_head = dom_depth(main_head);
uint max = main_head->outcnt();
Node *pre_header= main_head->in(LoopNode::EntryControl);
Node *init = main_head->init_trip();
Node *incr = main_end ->incr();
Node *limit = main_end ->limit();
Node *stride = main_end ->stride();
Node *cmp = main_end ->cmp_node();
BoolTest::mask b_test = main_end->test_trip();
Node *bol = main_end->in(CountedLoopEndNode::TestValue);
if( bol->outcnt() != 1 ) {
bol = bol->clone();
register_new_node(bol,main_end->in(CountedLoopEndNode::TestControl));
_igvn.hash_delete(main_end);
main_end->set_req(CountedLoopEndNode::TestValue, bol);
}
if( cmp->outcnt() != 1 ) {
cmp = cmp->clone();
register_new_node(cmp,main_end->in(CountedLoopEndNode::TestControl));
_igvn.hash_delete(bol);
bol->set_req(1, cmp);
}
Node* main_exit = main_end->proj_out(false);
assert( main_exit->Opcode() == Op_IfFalse, "" );
int dd_main_exit = dom_depth(main_exit);
clone_loop( loop, old_new, dd_main_exit );
assert( old_new[main_end ->_idx]->Opcode() == Op_CountedLoopEnd, "" );
CountedLoopNode *post_head = old_new[main_head->_idx]->as_CountedLoop();
post_head->set_post_loop(main_head);
CountedLoopEndNode* post_end = old_new[main_end ->_idx]->as_CountedLoopEnd();
post_end->_prob = PROB_FAIR;
IfFalseNode *new_main_exit = new (C) IfFalseNode(main_end);
_igvn.register_new_node_with_optimizer( new_main_exit );
set_idom(new_main_exit, main_end, dd_main_exit );
set_loop(new_main_exit, loop->_parent);
Node *zer_opaq = new (C) Opaque1Node(C, incr);
Node *zer_cmp = new (C) CmpINode( zer_opaq, limit );
Node *zer_bol = new (C) BoolNode( zer_cmp, b_test );
register_new_node( zer_opaq, new_main_exit );
register_new_node( zer_cmp , new_main_exit );
register_new_node( zer_bol , new_main_exit );
IfNode *zer_iff = new (C) IfNode( new_main_exit, zer_bol, PROB_FAIR, COUNT_UNKNOWN );
_igvn.register_new_node_with_optimizer( zer_iff );
set_idom(zer_iff, new_main_exit, dd_main_exit);
set_loop(zer_iff, loop->_parent);
_igvn.replace_input_of(main_exit, 0, zer_iff);
set_idom(main_exit, zer_iff, dd_main_exit);
set_idom(main_exit->unique_out(), zer_iff, dd_main_exit);
Node *zer_taken = new (C) IfTrueNode( zer_iff );
_igvn.register_new_node_with_optimizer( zer_taken );
set_idom(zer_taken, zer_iff, dd_main_exit);
set_loop(zer_taken, loop->_parent);
_igvn.hash_delete( post_head );
post_head->set_req(LoopNode::EntryControl, zer_taken);
set_idom(post_head, zer_taken, dd_main_exit);
Arena *a = Thread::current()->resource_area();
VectorSet visited(a);
Node_Stack clones(a, main_head->back_control()->outcnt());
for (DUIterator_Fast imax, i = main_head->fast_outs(imax); i < imax; i++) {
Node* main_phi = main_head->fast_out(i);
if( main_phi->is_Phi() && main_phi->in(0) == main_head && main_phi->outcnt() >0 ) {
Node *post_phi = old_new[main_phi->_idx];
Node *fallmain = clone_up_backedge_goo(main_head->back_control(),
post_head->init_control(),
main_phi->in(LoopNode::LoopBackControl),
visited, clones);
_igvn.hash_delete(post_phi);
post_phi->set_req( LoopNode::EntryControl, fallmain );
}
}
main_exit = new_main_exit;
clone_loop( loop, old_new, dd_main_head );
CountedLoopNode* pre_head = old_new[main_head->_idx]->as_CountedLoop();
CountedLoopEndNode* pre_end = old_new[main_end ->_idx]->as_CountedLoopEnd();
pre_head->set_pre_loop(main_head);
Node *pre_incr = old_new[incr->_idx];
pre_end->_prob = PROB_FAIR;
Node* pre_exit = pre_end->proj_out(false);
assert( pre_exit->Opcode() == Op_IfFalse, "" );
IfFalseNode *new_pre_exit = new (C) IfFalseNode(pre_end);
_igvn.register_new_node_with_optimizer( new_pre_exit );
set_idom(new_pre_exit, pre_end, dd_main_head);
set_loop(new_pre_exit, loop->_parent);
Node *min_opaq = new (C) Opaque1Node(C, limit);
Node *min_cmp = new (C) CmpINode( pre_incr, min_opaq );
Node *min_bol = new (C) BoolNode( min_cmp, b_test );
register_new_node( min_opaq, new_pre_exit );
register_new_node( min_cmp , new_pre_exit );
register_new_node( min_bol , new_pre_exit );
IfNode *min_iff = new (C) IfNode( new_pre_exit, min_bol, PROB_ALWAYS, COUNT_UNKNOWN );
_igvn.register_new_node_with_optimizer( min_iff );
set_idom(min_iff, new_pre_exit, dd_main_head);
set_loop(min_iff, loop->_parent);
_igvn.hash_delete( pre_exit );
pre_exit->set_req(0, min_iff);
set_idom(pre_exit, min_iff, dd_main_head);
set_idom(pre_exit->unique_out(), min_iff, dd_main_head);
Node *min_taken = new (C) IfTrueNode( min_iff );
_igvn.register_new_node_with_optimizer( min_taken );
set_idom(min_taken, min_iff, dd_main_head);
set_loop(min_taken, loop->_parent);
_igvn.hash_delete( main_head );
main_head->set_req(LoopNode::EntryControl, min_taken);
set_idom(main_head, min_taken, dd_main_head);
visited.Clear();
clones.clear();
for (DUIterator_Fast i2max, i2 = main_head->fast_outs(i2max); i2 < i2max; i2++) {
Node* main_phi = main_head->fast_out(i2);
if( main_phi->is_Phi() && main_phi->in(0) == main_head && main_phi->outcnt() > 0 ) {
Node *pre_phi = old_new[main_phi->_idx];
Node *fallpre = clone_up_backedge_goo(pre_head->back_control(),
main_head->init_control(),
pre_phi->in(LoopNode::LoopBackControl),
visited, clones);
_igvn.hash_delete(main_phi);
main_phi->set_req( LoopNode::EntryControl, fallpre );
}
}
bool inserted = cast_incr_before_loop(zer_opaq->in(1), zer_taken, post_head);
assert(inserted, "no castII inserted");
inserted = cast_incr_before_loop(pre_incr, min_taken, main_head);
assert(inserted, "no castII inserted");
Node *cmp_end = pre_end->cmp_node();
assert( cmp_end->in(2) == limit, "" );
Node *pre_limit = new (C) AddINode( init, stride );
Node *pre_opaq = new (C) Opaque1Node(C, pre_limit, limit);
register_new_node( pre_limit, pre_head->in(0) );
register_new_node( pre_opaq , pre_head->in(0) );
assert( cmp_end->outcnt() == 1, "no other users" );
_igvn.hash_delete(cmp_end);
cmp_end->set_req(2, peel_only ? pre_limit : pre_opaq);
if (pre_end->in(CountedLoopEndNode::TestValue)->as_Bool()->_test._test == BoolTest::ne) {
BoolTest::mask new_test = (main_end->stride_con() > 0) ? BoolTest::lt : BoolTest::gt;
Node* pre_bol = pre_end->in(CountedLoopEndNode::TestValue)->as_Bool();
BoolNode* new_bol0 = new (C) BoolNode(pre_bol->in(1), new_test);
register_new_node( new_bol0, pre_head->in(0) );
_igvn.hash_delete(pre_end);
pre_end->set_req(CountedLoopEndNode::TestValue, new_bol0);
assert(min_iff->in(CountedLoopEndNode::TestValue) == min_bol, "guard okay");
BoolNode* new_bol1 = new (C) BoolNode(min_bol->in(1), new_test);
register_new_node( new_bol1, new_pre_exit );
_igvn.hash_delete(min_iff);
min_iff->set_req(CountedLoopEndNode::TestValue, new_bol1);
BoolNode* main_bol = main_end->in(CountedLoopEndNode::TestValue)->as_Bool();
BoolNode* new_bol2 = new (C) BoolNode(main_bol->in(1), new_test);
register_new_node( new_bol2, main_end->in(CountedLoopEndNode::TestControl) );
_igvn.hash_delete(main_end);
main_end->set_req(CountedLoopEndNode::TestValue, new_bol2);
}
main_head->set_main_loop();
if( peel_only ) main_head->set_main_no_pre_loop();
main_head->set_trip_count(main_head->trip_count() - 1);
post_head->set_profile_trip_cnt(4.0);
pre_head->set_profile_trip_cnt(4.0);
peeled_dom_test_elim(loop,old_new);
loop->record_for_igvn();
}
bool IdealLoopTree::is_invariant(Node* n) const {
Node *n_c = _phase->has_ctrl(n) ? _phase->get_ctrl(n) : n;
if (n_c->is_top()) return false;
return !is_member(_phase->get_loop(n_c));
}
void PhaseIdealLoop::do_unroll( IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip ) {
assert(LoopUnrollLimit, "");
CountedLoopNode *loop_head = loop->_head->as_CountedLoop();
CountedLoopEndNode *loop_end = loop_head->loopexit();
assert(loop_end, "");
#ifndef PRODUCT
if (PrintOpto && VerifyLoopOptimizations) {
tty->print("Unrolling ");
loop->dump_head();
} else if (TraceLoopOpts) {
if (loop_head->trip_count() < (uint)LoopUnrollLimit) {
tty->print("Unroll %d(%2d) ", loop_head->unrolled_count()*2, loop_head->trip_count());
} else {
tty->print("Unroll %d ", loop_head->unrolled_count()*2);
}
loop->dump_head();
}
#endif
loop_head->set_node_count_before_unroll(loop->_body.size());
Node *ctrl = loop_head->in(LoopNode::EntryControl);
Node *limit = loop_head->limit();
Node *init = loop_head->init_trip();
Node *stride = loop_head->stride();
Node *opaq = NULL;
if (adjust_min_trip) { // If not maximally unrolling, need adjustment
if (!is_canonical_main_loop_entry(loop_head)) {
return;
}
opaq = ctrl->in(0)->in(1)->in(1)->in(2);
assert(opaq->outcnt() == 1 && opaq->in(1) == limit, "");
}
C->set_major_progress();
Node* new_limit = NULL;
if (UnrollLimitCheck) {
int stride_con = stride->get_int();
int stride_p = (stride_con > 0) ? stride_con : -stride_con;
uint old_trip_count = loop_head->trip_count();
assert(old_trip_count > 1 &&
(!adjust_min_trip || stride_p <= (1<<3)*loop_head->unrolled_count()), "sanity");
if (!adjust_min_trip) {
assert(old_trip_count > 1 && (old_trip_count & 1) == 0,
"odd trip count for maximally unroll");
} else if (loop_head->has_exact_trip_count() && init->is_Con()) {
jlong init_con = init->get_int();
jlong limit_con = limit->get_int();
int new_stride_con = stride_con * 2;
int stride_m = new_stride_con - (stride_con > 0 ? 1 : -1);
jlong trip_count = (limit_con - init_con + stride_m)/new_stride_con;
assert(trip_count > 0 && (julong)trip_count < (julong)max_juint/2, "sanity");
uint new_trip_count = (uint)trip_count;
adjust_min_trip = (old_trip_count != new_trip_count*2);
}
if (adjust_min_trip) {
Node* cmp = loop_end->cmp_node();
assert(cmp->in(2) == limit, "sanity");
assert(opaq != NULL && opaq->in(1) == limit, "sanity");
const TypeInt* limit_type = _igvn.type(limit)->is_int();
assert(stride_con > 0 && ((limit_type->_hi - stride_con) < limit_type->_hi) ||
stride_con < 0 && ((limit_type->_lo - stride_con) > limit_type->_lo), "sanity");
if (limit->is_Con()) {
new_limit = _igvn.intcon(limit->get_int() - stride_con);
set_ctrl(new_limit, C->root());
} else {
if (loop_head->unrolled_count() == 1) { // only for first unroll
assert(has_ctrl(opaq), "should have it");
Node* opaq_ctrl = get_ctrl(opaq);
limit = new (C) Opaque2Node( C, limit );
register_new_node( limit, opaq_ctrl );
}
if (stride_con > 0 && (java_subtract(limit_type->_lo, stride_con) < limit_type->_lo) ||
stride_con < 0 && (java_subtract(limit_type->_hi, stride_con) > limit_type->_hi)) {
new_limit = new (C) SubINode(limit, stride);
} else {
BoolTest::mask bt = loop_end->test_trip();
assert(bt == BoolTest::lt || bt == BoolTest::gt, "canonical test is expected");
Node* adj_max = _igvn.intcon((stride_con > 0) ? min_jint : max_jint);
set_ctrl(adj_max, C->root());
Node* old_limit = NULL;
Node* adj_limit = NULL;
Node* bol = limit->is_CMove() ? limit->in(CMoveNode::Condition) : NULL;
if (loop_head->unrolled_count() > 1 &&
limit->is_CMove() && limit->Opcode() == Op_CMoveI &&
limit->in(CMoveNode::IfTrue) == adj_max &&
bol->as_Bool()->_test._test == bt &&
bol->in(1)->Opcode() == Op_CmpI &&
bol->in(1)->in(2) == limit->in(CMoveNode::IfFalse)) {
old_limit = bol->in(1)->in(1);
adj_limit = limit->in(CMoveNode::IfFalse);
adj_limit = new (C) SubINode(adj_limit, stride);
} else {
old_limit = limit;
adj_limit = new (C) SubINode(limit, stride);
}
assert(old_limit != NULL && adj_limit != NULL, "");
register_new_node( adj_limit, ctrl ); // adjust amount
Node* adj_cmp = new (C) CmpINode(old_limit, adj_limit);
register_new_node( adj_cmp, ctrl );
Node* adj_bool = new (C) BoolNode(adj_cmp, bt);
register_new_node( adj_bool, ctrl );
new_limit = new (C) CMoveINode(adj_bool, adj_limit, adj_max, TypeInt::INT);
}
register_new_node(new_limit, ctrl);
}
assert(new_limit != NULL, "");
assert(loop_end->in(1)->in(1) == cmp, "sanity");
if (cmp->outcnt() == 1 && loop_end->in(1)->outcnt() == 1) {
_igvn.hash_delete(cmp);
cmp->set_req(2, new_limit);
} else {
Node* ctrl2 = loop_end->in(0);
Node* cmp2 = cmp->clone();
cmp2->set_req(2, new_limit);
register_new_node(cmp2, ctrl2);
Node* bol2 = loop_end->in(1)->clone();
bol2->set_req(1, cmp2);
register_new_node(bol2, ctrl2);
_igvn.hash_delete(loop_end);
loop_end->set_req(1, bol2);
}
assert(opaq->outcnt() == 1, "");
_igvn.hash_delete(opaq);
opaq->set_req(1, new_limit);
}
loop_head->set_trip_count(old_trip_count / 2);
loop_head->double_unrolled_count();
} else { // LoopLimitCheck
loop_head->set_trip_count(loop_head->trip_count() / 2);
loop_head->double_unrolled_count();
Node *span = new (C) SubINode( limit, init );
register_new_node( span, ctrl );
Node *trip = new (C) DivINode( 0, span, stride );
register_new_node( trip, ctrl );
Node *mtwo = _igvn.intcon(-2);
set_ctrl(mtwo, C->root());
Node *rond = new (C) AndINode( trip, mtwo );
register_new_node( rond, ctrl );
Node *spn2 = new (C) MulINode( rond, stride );
register_new_node( spn2, ctrl );
new_limit = new (C) AddINode( spn2, init );
register_new_node( new_limit, ctrl );
Node *ctrl2 = loop_end->in(0);
Node *cmp2 = new (C) CmpINode( loop_head->incr(), new_limit );
register_new_node( cmp2, ctrl2 );
Node *bol2 = new (C) BoolNode( cmp2, loop_end->test_trip() );
register_new_node( bol2, ctrl2 );
_igvn.hash_delete(loop_end);
loop_end->set_req(CountedLoopEndNode::TestValue, bol2);
if( adjust_min_trip ) {
assert( new_limit != NULL, "" );
assert( opaq->outcnt() == 1, "" );
_igvn.hash_delete(opaq);
opaq->set_req(1, new_limit);
}
} // LoopLimitCheck
uint dd = dom_depth(loop_head);
clone_loop( loop, old_new, dd );
for (DUIterator_Fast jmax, j = loop_head->fast_outs(jmax); j < jmax; j++) {
Node* phi = loop_head->fast_out(j);
if( phi->is_Phi() && phi->in(0) == loop_head && phi->outcnt() > 0 ) {
Node *newphi = old_new[phi->_idx];
_igvn.hash_delete( phi );
_igvn.hash_delete( newphi );
phi ->set_req(LoopNode:: EntryControl, newphi->in(LoopNode::LoopBackControl));
newphi->set_req(LoopNode::LoopBackControl, phi ->in(LoopNode::LoopBackControl));
phi ->set_req(LoopNode::LoopBackControl, C->top());
}
}
Node *clone_head = old_new[loop_head->_idx];
_igvn.hash_delete( clone_head );
loop_head ->set_req(LoopNode:: EntryControl, clone_head->in(LoopNode::LoopBackControl));
clone_head->set_req(LoopNode::LoopBackControl, loop_head ->in(LoopNode::LoopBackControl));
loop_head ->set_req(LoopNode::LoopBackControl, C->top());
loop->_head = clone_head; // New loop header
set_idom(loop_head, loop_head ->in(LoopNode::EntryControl), dd);
set_idom(clone_head, clone_head->in(LoopNode::EntryControl), dd);
Node *newcle = old_new[loop_end->_idx];
_igvn.hash_delete( newcle );
Node *one = _igvn.intcon(1);
set_ctrl(one, C->root());
newcle->set_req(1, one);
uint max = loop->_body.size();
for( uint k = 0; k < max; k++ ) {
Node *old = loop->_body.at(k);
Node *nnn = old_new[old->_idx];
loop->_body.push(nnn);
if (!has_ctrl(old))
set_loop(nnn, loop);
}
loop->record_for_igvn();
}
void PhaseIdealLoop::do_maximally_unroll( IdealLoopTree *loop, Node_List &old_new ) {
CountedLoopNode *cl = loop->_head->as_CountedLoop();
assert(cl->has_exact_trip_count(), "trip count is not exact");
assert(cl->trip_count() > 0, "");
#ifndef PRODUCT
if (TraceLoopOpts) {
tty->print("MaxUnroll %d ", cl->trip_count());
loop->dump_head();
}
#endif
if ((cl->trip_count() & 1) == 1) {
do_peeling(loop, old_new);
}
if (cl->trip_count() > 0) {
assert((cl->trip_count() & 1) == 0, "missed peeling");
do_unroll(loop, old_new, false);
}
}
bool IdealLoopTree::dominates_backedge(Node* ctrl) {
assert(ctrl->is_CFG(), "must be control");
Node* backedge = _head->as_Loop()->in(LoopNode::LoopBackControl);
return _phase->dom_lca_internal(ctrl, backedge) == ctrl;
}
Node* PhaseIdealLoop::adjust_limit(bool is_positive_stride, Node* scale, Node* offset, Node* rc_limit, Node* old_limit, Node* pre_ctrl, bool round) {
Node* sub = new (C) SubLNode(rc_limit, offset);
register_new_node(sub, pre_ctrl);
Node* limit = new (C) DivLNode(NULL, sub, scale);
register_new_node(limit, pre_ctrl);
if (round) {
limit = new (C) AddLNode(limit, _igvn.longcon(is_positive_stride ? -1 : 1));
register_new_node(limit, pre_ctrl);
}
Node* cmp = new (C) CmpLNode(limit, _igvn.longcon(is_positive_stride ? min_jint : max_jint));
register_new_node(cmp, pre_ctrl);
Node* bol = new (C) BoolNode(cmp, is_positive_stride ? BoolTest::lt : BoolTest::gt);
register_new_node(bol, pre_ctrl);
limit = new (C) ConvL2INode(limit);
register_new_node(limit, pre_ctrl);
limit = new (C) CMoveINode(bol, limit, _igvn.intcon(is_positive_stride ? min_jint : max_jint), TypeInt::INT);
register_new_node(limit, pre_ctrl);
limit = is_positive_stride ? (Node*)(new (C) MinINode(old_limit, limit))
: (Node*)(new (C) MaxINode(old_limit, limit));
register_new_node(limit, pre_ctrl);
return limit;
}
void PhaseIdealLoop::add_constraint(jlong stride_con, jlong scale_con, Node* offset, Node* low_limit, Node* upper_limit, Node* pre_ctrl, Node** pre_limit, Node** main_limit) {
assert(_igvn.type(offset)->isa_long() != NULL && _igvn.type(low_limit)->isa_long() != NULL &&
_igvn.type(upper_limit)->isa_long() != NULL, "arguments should be long values");
bool is_positive_stride = (stride_con > 0);
bool round = ABS(scale_con) > 1;
Node* scale = _igvn.longcon(scale_con);
set_ctrl(scale, C->root());
if ((stride_con^scale_con) >= 0) { // Use XOR to avoid overflow
} else {
Node* one = _igvn.longcon(1);
set_ctrl(one, C->root());
Node* plus_one = new (C) AddLNode(offset, one);
register_new_node( plus_one, pre_ctrl );
}
}
bool PhaseIdealLoop::is_scaled_iv(Node* exp, Node* iv, int* p_scale) {
if (exp == iv) {
if (p_scale != NULL) {
}
return true;
}
int opc = exp->Opcode();
if (opc == Op_MulI) {
if (exp->in(1) == iv && exp->in(2)->is_Con()) {
if (p_scale != NULL) {
}
return true;
}
if (exp->in(2) == iv && exp->in(1)->is_Con()) {
if (p_scale != NULL) {
}
return true;
}
} else if (opc == Op_LShiftI) {
if (exp->in(1) == iv && exp->in(2)->is_Con()) {
if (p_scale != NULL) {
}
return true;
}
}
return false;
}
bool PhaseIdealLoop::is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset, int depth) {
if (is_scaled_iv(exp, iv, p_scale)) {
if (p_offset != NULL) {
Node *zero = _igvn.intcon(0);
set_ctrl(zero, C->root());
}
return true;
}
int opc = exp->Opcode();
if (opc == Op_AddI) {
if (is_scaled_iv(exp->in(1), iv, p_scale)) {
if (p_offset != NULL) {
}
return true;
}
if (is_scaled_iv(exp->in(2), iv, p_scale)) {
if (p_offset != NULL) {
}
return true;
}
if (exp->in(2)->is_Con()) {
Node* offset2 = NULL;
if (depth < 2 &&
is_scaled_iv_plus_offset(exp->in(1), iv, p_scale,
p_offset != NULL ? &offset2 : NULL, depth+1)) {
if (p_offset != NULL) {
Node *ctrl_off2 = get_ctrl(offset2);
Node* offset = new (C) AddINode(offset2, exp->in(2));
register_new_node(offset, ctrl_off2);
}
return true;
}
}
} else if (opc == Op_SubI) {
if (is_scaled_iv(exp->in(1), iv, p_scale)) {
if (p_offset != NULL) {
Node *zero = _igvn.intcon(0);
set_ctrl(zero, C->root());
Node *ctrl_off = get_ctrl(exp->in(2));
Node* offset = new (C) SubINode(zero, exp->in(2));
register_new_node(offset, ctrl_off);
}
return true;
}
if (is_scaled_iv(exp->in(2), iv, p_scale)) {
if (p_offset != NULL) {
}
return true;
}
}
return false;
}
void PhaseIdealLoop::do_range_check( IdealLoopTree *loop, Node_List &old_new ) {
#ifndef PRODUCT
if (PrintOpto && VerifyLoopOptimizations) {
tty->print("Range Check Elimination ");
loop->dump_head();
} else if (TraceLoopOpts) {
tty->print("RangeCheck ");
loop->dump_head();
}
#endif
assert(RangeCheckElimination, "");
CountedLoopNode *cl = loop->_head->as_CountedLoop();
if (!cl->stride_is_con())
return;
Node *trip_counter = cl->phi();
Node *main_limit = cl->limit();
if (!is_canonical_main_loop_entry(cl)) {
return;
}
Node *ctrl = cl->in(LoopNode::EntryControl);
Node *iffm = ctrl->in(0);
Node *opqzm = iffm->in(1)->in(1)->in(2);
assert(opqzm->in(1) == main_limit, "do not understand situation");
Node *p_f = iffm->in(0);
if (p_f->Opcode() != Op_IfFalse) {
return;
}
CountedLoopEndNode *pre_end = p_f->in(0)->as_CountedLoopEnd();
assert(pre_end->loopnode()->is_pre_loop(), "");
Node *pre_opaq1 = pre_end->limit();
if (pre_opaq1->Opcode() != Op_Opaque1)
return;
Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
Node *pre_limit = pre_opaq->in(1);
Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl);
Node *orig_limit = pre_opaq->original_loop_limit();
if (orig_limit == NULL || _igvn.type(orig_limit) == Type::TOP)
return;
int stride_con = cl->stride_con();
Node* zero = _igvn.longcon(0);
Node* one = _igvn.longcon(1);
Node* mini = _igvn.longcon(-max_jint);
set_ctrl(zero, C->root());
set_ctrl(one, C->root());
set_ctrl(mini, C->root());
for( uint i = 0; i < loop->_body.size(); i++ ) {
Node *iff = loop->_body[i];
if( iff->Opcode() == Op_If ) { // Test?
Node *exit = loop->is_loop_exit(iff);
if( !exit ) continue;
int flip = (exit->Opcode() == Op_IfTrue) ? 1 : 0;
Node *i1 = iff->in(1);
if( !i1->is_Bool() ) continue;
BoolNode *bol = i1->as_Bool();
BoolTest b_test = bol->_test;
if( flip )
b_test = b_test.negate();
Node *cmp = bol->in(1);
Node *rc_exp = cmp->in(1);
Node *limit = cmp->in(2);
jint scale_con= 1; // Assume trip counter not scaled
Node *limit_c = get_ctrl(limit);
if( loop->is_member(get_loop(limit_c) ) ) {
b_test = b_test.commute();
rc_exp = cmp->in(2);
limit = cmp->in(1);
limit_c = get_ctrl(limit);
if( loop->is_member(get_loop(limit_c) ) )
continue; // Both inputs are loop varying; cannot RCE
}
if( limit_c == ctrl ) {
continue; // Don't rce this check but continue looking for other candidates.
}
Node *offset = NULL;
if (!is_scaled_iv_plus_offset(rc_exp, trip_counter, &scale_con, &offset)) {
continue;
}
Node *offset_c = get_ctrl(offset);
if( loop->is_member( get_loop(offset_c) ) )
continue; // Offset is not really loop invariant
if( offset_c == ctrl ) {
continue; // Don't rce this check but continue looking for other candidates.
}
#ifdef ASSERT
if (TraceRangeLimitCheck) {
tty->print_cr("RC bool node%s", flip ? " flipped:" : ":");
bol->dump(2);
}
#endif
jlong lscale_con = scale_con;
Node* int_offset = offset;
offset = new (C) ConvI2LNode(offset);
register_new_node(offset, pre_ctrl);
Node* int_limit = limit;
limit = new (C) ConvI2LNode(limit);
register_new_node(limit, pre_ctrl);
if( cmp->Opcode() == Op_CmpU ) {// Unsigned compare is really 2 tests
if( b_test._test == BoolTest::lt ) { // Range checks always use lt
add_constraint(stride_con, lscale_con, offset, zero, limit, pre_ctrl, &pre_limit, &main_limit);
} else {
#ifndef PRODUCT
if( PrintOpto )
tty->print_cr("missed RCE opportunity");
#endif
continue; // In release mode, ignore it
}
} else { // Otherwise work on normal compares
switch( b_test._test ) {
case BoolTest::gt:
case BoolTest::ge:
lscale_con = -lscale_con;
offset = new (C) SubLNode(zero, offset);
register_new_node( offset, pre_ctrl );
limit = new (C) SubLNode(zero, limit);
register_new_node( limit, pre_ctrl );
case BoolTest::le:
if (b_test._test != BoolTest::gt) {
limit = new (C) AddLNode(limit, one);
register_new_node( limit, pre_ctrl );
}
case BoolTest::lt:
add_constraint(stride_con, lscale_con, offset, mini, limit, pre_ctrl, &pre_limit, &main_limit);
break;
default:
#ifndef PRODUCT
if( PrintOpto )
tty->print_cr("missed RCE opportunity");
#endif
continue; // Unhandled case
}
}
C->set_major_progress();
Node *kill_con = _igvn.intcon( 1-flip );
set_ctrl(kill_con, C->root());
_igvn.replace_input_of(iff, 1, kill_con);
assert(iff->is_If(), "");
ProjNode* dp = ((IfNode*)iff)->proj_out(1-flip);
for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) {
Node* cd = dp->fast_out(i); // Control-dependent node
if (cd->is_Load() && cd->depends_only_on_test()) { // Loads can now float around in the loop
_igvn.replace_input_of(cd, 0, ctrl); // ctrl, not NULL
--i;
--imax;
}
}
} // End of is IF
}
if (pre_limit != orig_limit) {
pre_limit = (stride_con > 0) ? (Node*)new (C) MinINode(pre_limit, orig_limit)
: (Node*)new (C) MaxINode(pre_limit, orig_limit);
register_new_node(pre_limit, pre_ctrl);
}
_igvn.hash_delete(pre_opaq);
pre_opaq->set_req(1, pre_limit);
cl->set_nonexact_trip_count();
if (!LoopLimitCheck && stride_con != 1 && stride_con != -1) { // Cutout for common case
Node *ctrl = get_ctrl(main_limit);
Node *stride = cl->stride();
Node *init = cl->init_trip();
Node *span = new (C) SubINode(main_limit,init);
register_new_node(span,ctrl);
Node *rndup = _igvn.intcon(stride_con + ((stride_con>0)?-1:1));
Node *add = new (C) AddINode(span,rndup);
register_new_node(add,ctrl);
Node *div = new (C) DivINode(0,add,stride);
register_new_node(div,ctrl);
Node *mul = new (C) MulINode(div,stride);
register_new_node(mul,ctrl);
Node *newlim = new (C) AddINode(mul,init);
register_new_node(newlim,ctrl);
main_limit = newlim;
}
Node *main_cle = cl->loopexit();
Node *main_bol = main_cle->in(1);
if( main_bol->outcnt() > 1 ) {// BoolNode shared?
_igvn.hash_delete(main_cle);
main_bol = main_bol->clone();// Clone a private BoolNode
register_new_node( main_bol, main_cle->in(0) );
main_cle->set_req(1,main_bol);
}
Node *main_cmp = main_bol->in(1);
if( main_cmp->outcnt() > 1 ) { // CmpNode shared?
_igvn.hash_delete(main_bol);
main_cmp = main_cmp->clone();// Clone a private CmpNode
register_new_node( main_cmp, main_cle->in(0) );
main_bol->set_req(1,main_cmp);
}
_igvn.replace_input_of(main_cmp, 2, main_limit);
assert( opqzm->outcnt() == 1, "cannot hack shared node" );
_igvn.replace_input_of(opqzm, 1, main_limit);
}
void IdealLoopTree::DCE_loop_body() {
for( uint i = 0; i < _body.size(); i++ )
if( _body.at(i)->outcnt() == 0 )
_body.map( i--, _body.pop() );
}
void IdealLoopTree::adjust_loop_exit_prob( PhaseIdealLoop *phase ) {
Node *test = tail();
while( test != _head ) {
uint top = test->Opcode();
if( top == Op_IfTrue || top == Op_IfFalse ) {
int test_con = ((ProjNode*)test)->_con;
assert(top == (uint)(test_con? Op_IfTrue: Op_IfFalse), "sanity");
IfNode *iff = test->in(0)->as_If();
if( iff->outcnt() == 2 ) { // Ignore dead tests
Node *bol = iff->in(1);
if( bol && bol->req() > 1 && bol->in(1) &&
((bol->in(1)->Opcode() == Op_StorePConditional ) ||
(bol->in(1)->Opcode() == Op_StoreIConditional ) ||
(bol->in(1)->Opcode() == Op_StoreLConditional ) ||
(bol->in(1)->Opcode() == Op_CompareAndSwapI ) ||
(bol->in(1)->Opcode() == Op_CompareAndSwapL ) ||
(bol->in(1)->Opcode() == Op_CompareAndSwapP ) ||
(bol->in(1)->Opcode() == Op_CompareAndSwapN )))
return; // Allocation loops RARELY take backedge
Node* ex = iff->proj_out(1-test_con);
float p = iff->_prob;
if( !phase->is_member( this, ex ) && iff->_fcnt == COUNT_UNKNOWN ) {
if( top == Op_IfTrue ) {
if( p < (PROB_FAIR + PROB_UNLIKELY_MAG(3))) {
iff->_prob = PROB_STATIC_FREQUENT;
}
} else {
if( p > (PROB_FAIR - PROB_UNLIKELY_MAG(3))) {
iff->_prob = PROB_STATIC_INFREQUENT;
}
}
}
}
}
test = phase->idom(test);
}
}
bool IdealLoopTree::policy_do_remove_empty_loop( PhaseIdealLoop *phase ) {
if (_body.size() > EMPTY_LOOP_SIZE)
return false;
if (!_head->is_CountedLoop())
return false; // Dead loop
CountedLoopNode *cl = _head->as_CountedLoop();
if (!cl->is_valid_counted_loop())
return false; // Malformed loop
if (!phase->is_member(this, phase->get_ctrl(cl->loopexit()->in(CountedLoopEndNode::TestValue))))
return false; // Infinite loop
#ifdef ASSERT
Node* iv = NULL;
for (DUIterator_Fast imax, i = cl->fast_outs(imax); i < imax; i++) {
Node* n = cl->fast_out(i);
if (n->Opcode() == Op_Phi) {
assert(iv == NULL, "Too many phis" );
iv = n;
}
}
assert(iv == cl->phi(), "Wrong phi" );
#endif
bool needs_guard = !cl->is_main_loop() && !cl->is_post_loop();
if (needs_guard) {
const TypeInt* init_t = phase->_igvn.type(cl->init_trip())->is_int();
const TypeInt* limit_t = phase->_igvn.type(cl->limit())->is_int();
int stride_con = cl->stride_con();
if (stride_con > 0) {
needs_guard = (init_t->_hi >= limit_t->_lo);
} else {
needs_guard = (init_t->_lo <= limit_t->_hi);
}
}
if (needs_guard) {
Node* inctrl = PhaseIdealLoop::skip_loop_predicates(cl->in(LoopNode::EntryControl));
if (inctrl->Opcode() == Op_IfTrue) {
Node* iff = inctrl->in(0);
if (iff->is_If()) {
Node* bol = iff->in(1);
if (bol->is_Bool() && bol->as_Bool()->_test._test == cl->loopexit()->test_trip()) {
Node* cmp = bol->in(1);
if (cmp->is_Cmp() && cmp->in(1) == cl->init_trip() && cmp->in(2) == cl->limit()) {
needs_guard = false;
}
}
}
}
}
#ifndef PRODUCT
if (PrintOpto) {
tty->print("Removing empty loop with%s zero trip guard", needs_guard ? "out" : "");
this->dump_head();
} else if (TraceLoopOpts) {
tty->print("Empty with%s zero trip guard ", needs_guard ? "out" : "");
this->dump_head();
}
#endif
if (needs_guard) {
Node_List old_new;
phase->do_peeling(this, old_new);
}
Node *phi = cl->phi();
Node *exact_limit = phase->exact_limit(this);
if (exact_limit != cl->limit()) {
Node* cmp = cl->loopexit()->cmp_node();
assert(cl->limit() == cmp->in(2), "sanity");
if (cmp->outcnt() > 1) {
cmp = cmp->clone();
cmp = phase->_igvn.register_new_node_with_optimizer(cmp);
BoolNode *bol = cl->loopexit()->in(CountedLoopEndNode::TestValue)->as_Bool();
phase->_igvn.replace_input_of(bol, 1, cmp); // put bol on worklist
}
phase->_igvn._worklist.push(cmp->in(2)); // put limit on worklist
phase->_igvn.replace_input_of(cmp, 2, exact_limit); // put cmp on worklist
}
Node *final = new (phase->C) SubINode( exact_limit, cl->stride() );
phase->register_new_node(final,cl->in(LoopNode::EntryControl));
phase->_igvn.replace_node(phi,final);
phase->C->set_major_progress();
return true;
}
bool IdealLoopTree::policy_do_one_iteration_loop( PhaseIdealLoop *phase ) {
if (!_head->as_Loop()->is_valid_counted_loop())
return false; // Only for counted loop
CountedLoopNode *cl = _head->as_CountedLoop();
if (!cl->has_exact_trip_count() || cl->trip_count() != 1) {
return false;
}
#ifndef PRODUCT
if(TraceLoopOpts) {
tty->print("OneIteration ");
this->dump_head();
}
#endif
Node *init_n = cl->init_trip();
#ifdef ASSERT
assert(init_n->get_int() + cl->stride_con() >= cl->limit()->get_int(), "should be one iteration");
#endif
phase->_igvn.replace_node(cl->phi(), cl->init_trip());
phase->C->set_major_progress();
return true;
}
bool IdealLoopTree::iteration_split_impl( PhaseIdealLoop *phase, Node_List &old_new ) {
compute_exact_trip_count(phase);
if (policy_do_one_iteration_loop(phase))
return true;
if (policy_do_remove_empty_loop(phase))
return true; // Here we removed an empty loop
bool should_peel = policy_peeling(phase); // Should we peel?
bool should_unswitch = policy_unswitching(phase);
if (!_head->is_CountedLoop()) { // Non-counted loop
if (PartialPeelLoop && phase->partial_peel(this, old_new)) {
return false;
}
if (should_peel) { // Should we peel?
#ifndef PRODUCT
if (PrintOpto) tty->print_cr("should_peel");
#endif
phase->do_peeling(this,old_new);
} else if (should_unswitch) {
phase->do_unswitching(this, old_new);
}
return true;
}
CountedLoopNode *cl = _head->as_CountedLoop();
if (!cl->is_valid_counted_loop()) return true; // Ignore various kinds of broken loops
if (cl->is_pre_loop() || cl->is_post_loop()) return true;
compute_profile_trip_cnt(phase);
if (cl->is_normal_loop()) {
if (should_unswitch) {
phase->do_unswitching(this, old_new);
return true;
}
bool should_maximally_unroll = policy_maximally_unroll(phase);
if (should_maximally_unroll) {
phase->do_maximally_unroll(this,old_new);
return true;
}
}
int nodes_left = phase->C->max_node_limit() - phase->C->live_nodes();
if ((int)(2 * _body.size()) > nodes_left) {
return true;
}
bool should_unroll = policy_unroll(phase);
bool should_rce = policy_range_check(phase);
bool should_align = policy_align(phase);
bool may_rce_align = !policy_peel_only(phase) || should_rce || should_align;
if (should_rce || should_align || should_unroll) {
if (cl->is_normal_loop()) // Convert to 'pre/main/post' loops
phase->insert_pre_post_loops(this,old_new, !may_rce_align);
if (should_rce)
phase->do_range_check(this,old_new);
if (should_unroll && !should_peel)
phase->do_unroll(this,old_new, true);
if (should_align)
Unimplemented();
} else { // Else we have an unchanged counted loop
if (should_peel) // Might want to peel but do nothing else
phase->do_peeling(this,old_new);
}
return true;
}
bool IdealLoopTree::iteration_split( PhaseIdealLoop *phase, Node_List &old_new ) {
if (_child && !_child->iteration_split(phase, old_new))
return false;
DCE_loop_body();
if (_parent /*not the root loop*/ &&
!_irreducible &&
!tail()->is_top()) {
adjust_loop_exit_prob(phase);
}
if (!_child && // If not an inner loop, do not split
!_irreducible &&
_allow_optimizations &&
!tail()->is_top()) { // Also ignore the occasional dead backedge
if (!_has_call) {
if (!iteration_split_impl(phase, old_new)) {
return false;
}
} else if (policy_unswitching(phase)) {
phase->do_unswitching(this, old_new);
}
}
phase->reorg_offsets(this);
if (_next && !_next->iteration_split(phase, old_new))
return false;
return true;
}
bool PhaseIdealLoop::do_intrinsify_fill() {
bool changed = false;
for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
IdealLoopTree* lpt = iter.current();
changed |= intrinsify_fill(lpt);
}
return changed;
}
bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& store_value,
Node*& shift, Node*& con) {
const char* msg = NULL;
Node* msg_node = NULL;
store_value = NULL;
con = NULL;
shift = NULL;
CountedLoopNode* head = lpt->_head->as_CountedLoop();
for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
Node* n = lpt->_body.at(i);
if (n->outcnt() == 0) continue; // Ignore dead
if (n->is_Store()) {
if (store != NULL) {
msg = "multiple stores";
break;
}
int opc = n->Opcode();
if (opc == Op_StoreP || opc == Op_StoreN || opc == Op_StoreNKlass || opc == Op_StoreCM) {
msg = "oop fills not handled";
break;
}
Node* value = n->in(MemNode::ValueIn);
if (!lpt->is_invariant(value)) {
msg = "variant store value";
} else if (!_igvn.type(n->in(MemNode::Address))->isa_aryptr()) {
msg = "not array address";
}
store = n;
store_value = value;
} else if (n->is_If() && n != head->loopexit()) {
msg = "extra control flow";
msg_node = n;
}
}
if (store == NULL) {
return false;
}
if (msg == NULL && head->stride_con() != 1) {
if (head->stride_con() < 0) {
msg = "negative stride";
} else {
msg = "non-unit stride";
}
}
if (msg == NULL && !store->in(MemNode::Address)->is_AddP()) {
msg = "can't handle store address";
msg_node = store->in(MemNode::Address);
}
if (msg == NULL &&
(!store->in(MemNode::Memory)->is_Phi() ||
store->in(MemNode::Memory)->in(LoopNode::LoopBackControl) != store)) {
msg = "store memory isn't proper phi";
msg_node = store->in(MemNode::Memory);
}
BasicType t = store->as_Mem()->memory_type();
const char* fill_name;
if (msg == NULL &&
StubRoutines::select_fill_function(t, false, fill_name) == NULL) {
msg = "unsupported store";
msg_node = store;
}
if (msg != NULL) {
#ifndef PRODUCT
if (TraceOptimizeFill) {
tty->print_cr("not fill intrinsic candidate: %s", msg);
if (msg_node != NULL) msg_node->dump();
}
#endif
return false;
}
Node* elements[4];
Node* cast = NULL;
Node* conv = NULL;
bool found_index = false;
int count = store->in(MemNode::Address)->as_AddP()->unpack_offsets(elements, ARRAY_SIZE(elements));
for (int e = 0; e < count; e++) {
Node* n = elements[e];
if (n->is_Con() && con == NULL) {
con = n;
} else if (n->Opcode() == Op_LShiftX && shift == NULL) {
Node* value = n->in(1);
#ifdef _LP64
if (value->Opcode() == Op_ConvI2L) {
conv = value;
value = value->in(1);
}
if (value->Opcode() == Op_CastII &&
value->as_CastII()->has_range_check()) {
cast = value;
value = value->in(1);
}
#endif
if (value != head->phi()) {
msg = "unhandled shift in address";
} else {
if (type2aelembytes(store->as_Mem()->memory_type(), true) != (1 << n->in(2)->get_int())) {
msg = "scale doesn't match";
} else {
found_index = true;
shift = n;
}
}
} else if (n->Opcode() == Op_ConvI2L && conv == NULL) {
conv = n;
n = n->in(1);
if (n->Opcode() == Op_CastII &&
n->as_CastII()->has_range_check()) {
cast = n;
n = n->in(1);
}
if (n == head->phi()) {
found_index = true;
} else {
msg = "unhandled input to ConvI2L";
}
} else if (n == head->phi()) {
found_index = true;
} else {
msg = "unhandled node in address";
msg_node = n;
}
}
if (count == -1) {
msg = "malformed address expression";
msg_node = store;
}
if (!found_index) {
msg = "missing use of index";
}
if (msg == NULL && shift == NULL && t != T_BYTE && t != T_BOOLEAN) {
msg = "can't find shift";
msg_node = store;
}
if (msg != NULL) {
#ifndef PRODUCT
if (TraceOptimizeFill) {
tty->print_cr("not fill intrinsic: %s", msg);
if (msg_node != NULL) msg_node->dump();
}
#endif
return false;
}
VectorSet ok(Thread::current()->resource_area());
ok.set(store->_idx);
ok.set(store->in(MemNode::Memory)->_idx);
CountedLoopEndNode* loop_exit = head->loopexit();
guarantee(loop_exit != NULL, "no loop exit node");
ok.set(head->_idx);
ok.set(loop_exit->_idx);
ok.set(head->phi()->_idx);
ok.set(head->incr()->_idx);
ok.set(loop_exit->cmp_node()->_idx);
ok.set(loop_exit->in(1)->_idx);
if (con) ok.set(con->_idx);
if (shift) ok.set(shift->_idx);
if (cast) ok.set(cast->_idx);
if (conv) ok.set(conv->_idx);
for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
Node* n = lpt->_body.at(i);
if (n->outcnt() == 0) continue; // Ignore dead
if (ok.test(n->_idx)) continue;
if (n->is_IfTrue() && n->in(0) == loop_exit) continue;
if (!n->is_AddP()) {
msg = "unhandled node";
msg_node = n;
break;
}
}
for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
Node* n = lpt->_body.at(i);
if (n == store || n == loop_exit || n == head->incr() || n == store->in(MemNode::Memory)) continue;
for (SimpleDUIterator iter(n); iter.has_next(); iter.next()) {
Node* use = iter.get();
if (!lpt->_body.contains(use)) {
msg = "node is used outside loop";
msg_node = n;
break;
}
}
}
#ifdef ASSERT
if (TraceOptimizeFill) {
if (msg != NULL) {
tty->print_cr("no fill intrinsic: %s", msg);
if (msg_node != NULL) msg_node->dump();
} else {
tty->print_cr("fill intrinsic for:");
}
store->dump();
if (Verbose) {
lpt->_body.dump();
}
}
#endif
return msg == NULL;
}
bool PhaseIdealLoop::intrinsify_fill(IdealLoopTree* lpt) {
if (!lpt->is_counted() || !lpt->is_inner()) {
return false;
}
CountedLoopNode* head = lpt->_head->as_CountedLoop();
if (!head->is_valid_counted_loop() || !head->is_normal_loop()) {
return false;
}
Node* store = NULL;
Node* store_value = NULL;
Node* shift = NULL;
Node* offset = NULL;
if (!match_fill_loop(lpt, store, store_value, shift, offset)) {
return false;
}
Node* exit = head->loopexit()->proj_out(0);
if (exit == NULL) {
return false;
}
#ifndef PRODUCT
if (TraceLoopOpts) {
tty->print("ArrayFill ");
lpt->dump_head();
}
#endif
Node* base = store->in(MemNode::Address)->as_AddP()->in(AddPNode::Base);
Node* index = head->init_trip();
#ifdef _LP64
index = new (C) ConvI2LNode(index);
_igvn.register_new_node_with_optimizer(index);
#endif
if (shift != NULL) {
index = new (C) LShiftXNode(index, shift->in(2));
_igvn.register_new_node_with_optimizer(index);
}
index = new (C) AddPNode(base, base, index);
_igvn.register_new_node_with_optimizer(index);
Node* from = new (C) AddPNode(base, index, offset);
_igvn.register_new_node_with_optimizer(from);
Node* len = new (C) SubINode(head->limit(), head->init_trip());
_igvn.register_new_node_with_optimizer(len);
BasicType t = store->as_Mem()->memory_type();
bool aligned = false;
if (offset != NULL && head->init_trip()->is_Con()) {
int element_size = type2aelembytes(t);
aligned = (offset->find_intptr_t_type()->get_con() + head->init_trip()->get_int() * element_size) % HeapWordSize == 0;
}
const char* fill_name;
address fill = StubRoutines::select_fill_function(t, aligned, fill_name);
assert(fill != NULL, "what?");
if (t == T_FLOAT) {
store_value = new (C) MoveF2INode(store_value);
_igvn.register_new_node_with_optimizer(store_value);
} else if (t == T_DOUBLE) {
store_value = new (C) MoveD2LNode(store_value);
_igvn.register_new_node_with_optimizer(store_value);
}
if (CCallingConventionRequiresIntsAsLongs &&
(t == T_FLOAT || t == T_INT || is_subword_type(t))) {
store_value = new (C) ConvI2LNode(store_value);
_igvn.register_new_node_with_optimizer(store_value);
}
Node* mem_phi = store->in(MemNode::Memory);
Node* result_ctrl;
Node* result_mem;
const TypeFunc* call_type = OptoRuntime::array_fill_Type();
CallLeafNode *call = new (C) CallLeafNoFPNode(call_type, fill,
fill_name, TypeAryPtr::get_array_body_type(t));
uint cnt = 0;
call->init_req(TypeFunc::Parms + cnt++, from);
call->init_req(TypeFunc::Parms + cnt++, store_value);
if (CCallingConventionRequiresIntsAsLongs) {
call->init_req(TypeFunc::Parms + cnt++, C->top());
}
#ifdef _LP64
len = new (C) ConvI2LNode(len);
_igvn.register_new_node_with_optimizer(len);
#endif
call->init_req(TypeFunc::Parms + cnt++, len);
#ifdef _LP64
call->init_req(TypeFunc::Parms + cnt++, C->top());
#endif
call->init_req(TypeFunc::Control, head->init_control());
call->init_req(TypeFunc::I_O, C->top()); // Does no I/O.
call->init_req(TypeFunc::Memory, mem_phi->in(LoopNode::EntryControl));
call->init_req(TypeFunc::ReturnAdr, C->start()->proj_out(TypeFunc::ReturnAdr));
call->init_req(TypeFunc::FramePtr, C->start()->proj_out(TypeFunc::FramePtr));
_igvn.register_new_node_with_optimizer(call);
result_ctrl = new (C) ProjNode(call,TypeFunc::Control);
_igvn.register_new_node_with_optimizer(result_ctrl);
result_mem = new (C) ProjNode(call,TypeFunc::Memory);
_igvn.register_new_node_with_optimizer(result_mem);
AllocateNode* alloc = AllocateNode::Ideal_allocation(base, this);
if (alloc != NULL && alloc->is_AllocateArray()) {
Node* length = alloc->as_AllocateArray()->Ideal_length();
if (head->limit() == length &&
head->init_trip() == _igvn.intcon(0)) {
if (TraceOptimizeFill) {
tty->print_cr("Eliminated zeroing in allocation");
}
alloc->maybe_set_complete(&_igvn);
} else {
#ifdef ASSERT
if (TraceOptimizeFill) {
tty->print_cr("filling array but bounds don't match");
alloc->dump();
head->init_trip()->dump();
head->limit()->dump();
length->dump();
}
#endif
}
}
_igvn.replace_node(store->in(MemNode::Memory), result_mem);
lazy_replace(exit, result_ctrl);
_igvn.replace_node(store, result_mem);
_igvn.replace_node(head->incr(), head->limit());
for (uint i = 0; i < lpt->_body.size(); i++) {
Node* n = lpt->_body.at(i);
_igvn.replace_node(n, C->top());
}
return true;
}
C:\hotspot-69087d08d473\src\share\vm/opto/loopUnswitch.cpp
#include "precompiled.hpp"
#include "memory/allocation.inline.hpp"
#include "opto/connode.hpp"
#include "opto/loopnode.hpp"
#include "opto/rootnode.hpp"
bool IdealLoopTree::policy_unswitching( PhaseIdealLoop *phase ) const {
if( !LoopUnswitching ) {
return false;
}
if (!_head->is_Loop()) {
return false;
}
int nodes_left = phase->C->max_node_limit() - phase->C->live_nodes();
if ((int)(2 * _body.size()) > nodes_left) {
return false; // Too speculative if running low on nodes.
}
LoopNode* head = _head->as_Loop();
if (head->unswitch_count() + 1 > head->unswitch_max()) {
return false;
}
return phase->find_unswitching_candidate(this) != NULL;
}
IfNode* PhaseIdealLoop::find_unswitching_candidate(const IdealLoopTree *loop) const {
LoopNode *head = loop->_head->as_Loop();
IfNode* unswitch_iff = NULL;
Node* n = head->in(LoopNode::LoopBackControl);
while (n != head) {
Node* n_dom = idom(n);
if (n->is_Region()) {
if (n_dom->is_If()) {
IfNode* iff = n_dom->as_If();
if (iff->in(1)->is_Bool()) {
BoolNode* bol = iff->in(1)->as_Bool();
if (bol->in(1)->is_Cmp()) {
if (loop->is_invariant(bol) && !loop->is_loop_exit(iff)) {
unswitch_iff = iff;
}
}
}
}
}
n = n_dom;
}
return unswitch_iff;
}
void PhaseIdealLoop::do_unswitching (IdealLoopTree *loop, Node_List &old_new) {
LoopNode *head = loop->_head->as_Loop();
IfNode* unswitch_iff = find_unswitching_candidate((const IdealLoopTree *)loop);
assert(unswitch_iff != NULL, "should be at least one");
#ifndef PRODUCT
if (TraceLoopOpts) {
tty->print("Unswitch %d ", head->unswitch_count()+1);
loop->dump_head();
}
#endif
if (head->is_CountedLoop() && !head->as_CountedLoop()->is_normal_loop()) {
head->as_CountedLoop()->set_normal_loop();
}
ProjNode* proj_true = create_slow_version_of_loop(loop, old_new);
#ifdef ASSERT
Node* uniqc = proj_true->unique_ctrl_out();
Node* entry = head->in(LoopNode::EntryControl);
Node* predicate = find_predicate(entry);
if (predicate != NULL && LoopLimitCheck && UseLoopPredicate) {
entry = find_predicate(entry->in(0)->in(0));
if (entry != NULL) predicate = entry;
}
if (predicate != NULL) predicate = predicate->in(0);
assert(proj_true->is_IfTrue() &&
(predicate == NULL && uniqc == head ||
predicate != NULL && uniqc == predicate), "by construction");
#endif
LoopNode* head_clone = old_new[head->_idx]->as_Loop();
int nct = head->unswitch_count() + 1;
head->set_unswitch_count(nct);
head_clone->set_unswitch_count(nct);
IfNode* invar_iff = proj_true->in(0)->as_If();
Node* invar_iff_c = invar_iff->in(0);
BoolNode* bol = unswitch_iff->in(1)->as_Bool();
invar_iff->set_req(1, bol);
invar_iff->_prob = unswitch_iff->_prob;
ProjNode* proj_false = invar_iff->proj_out(0)->as_Proj();
Node_List worklist;
for (DUIterator_Fast imax, i = unswitch_iff->fast_outs(imax); i < imax; i++) {
ProjNode* proj= unswitch_iff->fast_out(i)->as_Proj();
for (DUIterator_Fast jmax, j = proj->fast_outs(jmax); j < jmax; j++) {
Node* use = proj->fast_out(j);
if (use->Opcode() == Op_CheckCastPP && loop->is_invariant(use->in(1))) {
worklist.push(use);
}
}
ProjNode* invar_proj = invar_iff->proj_out(proj->_con)->as_Proj();
while (worklist.size() > 0) {
Node* use = worklist.pop();
Node* nuse = use->clone();
nuse->set_req(0, invar_proj);
_igvn.replace_input_of(use, 1, nuse);
register_new_node(nuse, invar_proj);
Node* use_clone = old_new[use->_idx];
_igvn.replace_input_of(use_clone, 1, nuse);
}
}
_igvn.rehash_node_delayed(unswitch_iff);
short_circuit_if(unswitch_iff, proj_true);
IfNode* unswitch_iff_clone = old_new[unswitch_iff->_idx]->as_If();
_igvn.rehash_node_delayed(unswitch_iff_clone);
short_circuit_if(unswitch_iff_clone, proj_false);
loop->record_for_igvn();
for(int i = loop->_body.size() - 1; i >= 0 ; i--) {
Node *n = loop->_body[i];
Node *n_clone = old_new[n->_idx];
_igvn._worklist.push(n_clone);
}
#ifndef PRODUCT
if (TraceLoopUnswitching) {
tty->print_cr("Loop unswitching orig: %d @ %d new: %d @ %d",
head->_idx, unswitch_iff->_idx,
old_new[head->_idx]->_idx, unswitch_iff_clone->_idx);
}
#endif
C->set_major_progress();
}
ProjNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop,
Node_List &old_new) {
LoopNode* head = loop->_head->as_Loop();
bool counted_loop = head->is_CountedLoop();
Node* entry = head->in(LoopNode::EntryControl);
_igvn.rehash_node_delayed(entry);
IdealLoopTree* outer_loop = loop->_parent;
Node *cont = _igvn.intcon(1);
set_ctrl(cont, C->root());
Node* opq = new (C) Opaque1Node(C, cont);
register_node(opq, outer_loop, entry, dom_depth(entry));
Node *bol = new (C) Conv2BNode(opq);
register_node(bol, outer_loop, entry, dom_depth(entry));
IfNode* iff = new (C) IfNode(entry, bol, PROB_MAX, COUNT_UNKNOWN);
register_node(iff, outer_loop, entry, dom_depth(entry));
ProjNode* iffast = new (C) IfTrueNode(iff);
register_node(iffast, outer_loop, iff, dom_depth(iff));
ProjNode* ifslow = new (C) IfFalseNode(iff);
register_node(ifslow, outer_loop, iff, dom_depth(iff));
clone_loop(loop, old_new, dom_depth(head), iff);
assert(old_new[head->_idx]->is_Loop(), "" );
Node* iffast_pred = clone_loop_predicates(entry, iffast, !counted_loop);
_igvn.replace_input_of(head, LoopNode::EntryControl, iffast_pred);
set_idom(head, iffast_pred, dom_depth(head));
Node* ifslow_pred = clone_loop_predicates(entry, ifslow, !counted_loop);
LoopNode* slow_head = old_new[head->_idx]->as_Loop();
_igvn.replace_input_of(slow_head, LoopNode::EntryControl, ifslow_pred);
set_idom(slow_head, ifslow_pred, dom_depth(slow_head));
recompute_dom_depth();
return iffast;
}
C:\hotspot-69087d08d473\src\share\vm/opto/machnode.cpp
#include "precompiled.hpp"
#include "gc_interface/collectedHeap.hpp"
#include "opto/machnode.hpp"
#include "opto/regalloc.hpp"
int MachOper::reg(PhaseRegAlloc *ra_, const Node *node) const {
return (int)ra_->get_encode(node);
}
int MachOper::reg(PhaseRegAlloc *ra_, const Node *node, int idx) const {
return (int)(ra_->get_encode(node->in(idx)));
}
intptr_t MachOper::constant() const { return 0x00; }
relocInfo::relocType MachOper::constant_reloc() const { return relocInfo::none; }
jdouble MachOper::constantD() const { ShouldNotReachHere(); return 0.0; }
jfloat MachOper::constantF() const { ShouldNotReachHere(); return 0.0; }
jlong MachOper::constantL() const { ShouldNotReachHere(); return CONST64(0) ; }
TypeOopPtr *MachOper::oop() const { return NULL; }
int MachOper::ccode() const { return 0x00; }
int MachOper::base (PhaseRegAlloc *ra_, const Node *node, int idx) const { return 0x00; }
int MachOper::index(PhaseRegAlloc *ra_, const Node *node, int idx) const { return 0x00; }
int MachOper::scale() const { return 0x00; }
int MachOper::disp (PhaseRegAlloc *ra_, const Node *node, int idx) const { return 0x00; }
int MachOper::constant_disp() const { return 0; }
int MachOper::base_position() const { return -1; } // no base input
int MachOper::index_position() const { return -1; } // no index input
relocInfo::relocType MachOper::disp_reloc() const { return relocInfo::none; }
Label* MachOper::label() const { ShouldNotReachHere(); return 0; }
intptr_t MachOper::method() const { ShouldNotReachHere(); return 0; }
void MachOper::negate() {
ShouldNotCallThis();
}
const Type *MachOper::type() const {
return Type::BOTTOM;
}
const RegMask *MachOper::in_RegMask(int index) const {
ShouldNotReachHere();
return NULL;
}
#ifndef PRODUCT
void MachOper::dump_spec(outputStream *st) const { }
#endif
uint MachOper::hash() const {
ShouldNotCallThis();
return 5;
}
uint MachOper::cmp( const MachOper &oper ) const {
ShouldNotCallThis();
return opcode() == oper.opcode();
}
uint labelOper::hash() const {
return _block_num;
}
uint labelOper::cmp( const MachOper &oper ) const {
return (opcode() == oper.opcode()) && (_label == oper.label());
}
uint methodOper::hash() const {
return (uint)_method;
}
uint methodOper::cmp( const MachOper &oper ) const {
return (opcode() == oper.opcode()) && (_method == oper.method());
}
void MachNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
#ifdef ASSERT
tty->print("missing MachNode emit function: ");
dump();
#endif
ShouldNotCallThis();
}
void MachNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {}
uint MachNode::size(PhaseRegAlloc *ra_) const {
return MachNode::emit_size(ra_);
}
uint MachNode::emit_size(PhaseRegAlloc *ra_) const {
assert(ra_ == ra_->C->regalloc(), "sanity");
return ra_->C->scratch_emit_size(this);
}
uint MachNode::hash() const {
uint no = num_opnds();
uint sum = rule();
for( uint i=0; i<no; i++ )
sum += _opnds[i]->hash();
return sum+Node::hash();
}
uint MachNode::cmp( const Node &node ) const {
MachNode& n = *((Node&)node).as_Mach();
uint no = num_opnds();
if( no != n.num_opnds() ) return 0;
if( rule() != n.rule() ) return 0;
for( uint i=0; i<no; i++ ) // All operands must match
if( !_opnds[i]->cmp( *n._opnds[i] ) )
return 0; // mis-matched operands
return 1; // match
}
MachNode *MachNode::cisc_version(int offset, Compile* C) {
ShouldNotCallThis();
return NULL;
}
void MachNode::use_cisc_RegMask() {
ShouldNotReachHere();
}
const RegMask &MachNode::in_RegMask( uint idx ) const {
uint numopnds = num_opnds(); // Virtual call for number of operands
uint skipped = oper_input_base(); // Sum of leaves skipped so far
if( idx < skipped ) {
assert( ideal_Opcode() == Op_AddP, "expected base ptr here" );
assert( idx == 1, "expected base ptr here" );
return *Compile::current()->matcher()->idealreg2spillmask[Op_RegP];
}
uint opcnt = 1; // First operand
uint num_edges = _opnds[1]->num_edges(); // leaves for first operand
while( idx >= skipped+num_edges ) {
skipped += num_edges;
opcnt++; // Bump operand count
assert( opcnt < numopnds, "Accessing non-existent operand" );
num_edges = _opnds[opcnt]->num_edges(); // leaves for next operand
}
const RegMask *rm = cisc_RegMask();
if( rm == NULL || (int)opcnt != cisc_operand() ) {
rm = _opnds[opcnt]->in_RegMask(idx-skipped);
}
return *rm;
}
扫一扫
10-06
381
381
10-08
841
841
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
