443 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
444 "sha512_implCompress",
445 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
446 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
447 "sha1_implCompressMB",
448 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
449 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
450 "sha256_implCompressMB",
451 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
452 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
453 "sha512_implCompressMB",
454 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
455 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
456 "encodeBlock",
457 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
458 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
459 };
460
461 if (call->is_call_to_arraycopystub()) {
462 Node* dest = NULL;
463 const TypeTuple* args = n->as_Call()->_tf->domain();
464 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
465 if (args->field_at(i)->isa_ptr()) {
466 j++;
467 if (j == 2) {
468 dest = n->in(i);
469 break;
470 }
471 }
472 }
473 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
474 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
475 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
476 }
477 } else if (strlen(call->_name) > 5 &&
478 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
479 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
480 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
481 }
482 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
483 // skip
562 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } },
563 Op_StrEquals,
564 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
565 Op_EncodeISOArray,
566 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
567 Op_HasNegatives,
568 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
569 Op_CastP2X,
570 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
571 Op_StrIndexOfChar,
572 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
573 };
574
575 const int others_len = sizeof(others) / sizeof(others[0]);
576 int i = 0;
577 for (; i < others_len; i++) {
578 if (others[i].opcode == n->Opcode()) {
579 break;
580 }
581 }
582 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
583 if (i != others_len) {
584 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
585 for (uint j = 0; j < inputs_len; j++) {
586 int pos = others[i].inputs[j].pos;
587 if (pos == -1) {
588 break;
589 }
590 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
591 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
592 }
593 }
594 for (uint j = 1; j < stop; j++) {
595 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
596 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
597 uint k = 0;
598 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
599 if (k == inputs_len) {
600 fatal("arg %d for node %s not covered", j, n->Name());
601 }
602 }
782 mem_ctrl = phase->ctrl_or_self(mem);
783 }
784 return mem;
785 }
786
787 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
788 Node* mem = NULL;
789 Node* c = ctrl;
790 do {
791 if (c->is_Region()) {
792 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) {
793 Node* u = c->fast_out(i);
794 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
795 if (u->adr_type() == TypePtr::BOTTOM) {
796 mem = u;
797 }
798 }
799 }
800 } else {
801 if (c->is_Call() && c->as_Call()->adr_type() != NULL) {
802 CallProjections projs;
803 c->as_Call()->extract_projections(&projs, true, false);
804 if (projs.fallthrough_memproj != NULL) {
805 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
806 if (projs.catchall_memproj == NULL) {
807 mem = projs.fallthrough_memproj;
808 } else {
809 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
810 mem = projs.fallthrough_memproj;
811 } else {
812 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
813 mem = projs.catchall_memproj;
814 }
815 }
816 }
817 } else {
818 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
819 if (proj != NULL &&
820 proj->adr_type() == TypePtr::BOTTOM) {
821 mem = proj;
822 }
823 }
824 } else {
825 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
826 Node* u = c->fast_out(i);
827 if (u->is_Proj() &&
828 u->bottom_type() == Type::MEMORY &&
829 u->adr_type() == TypePtr::BOTTOM) {
830 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
831 assert(mem == NULL, "only one proj");
832 mem = u;
833 }
1035 if (c != ctrl ||
1036 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1037 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1038 phase->igvn().rehash_node_delayed(u);
1039 int nb = u->replace_edge(ctrl, region);
1040 if (u->is_CFG()) {
1041 if (phase->idom(u) == ctrl) {
1042 phase->set_idom(u, region, phase->dom_depth(region));
1043 }
1044 } else if (phase->get_ctrl(u) == ctrl) {
1045 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1046 uses.push(u);
1047 }
1048 assert(nb == 1, "more than 1 ctrl input?");
1049 --i, imax -= nb;
1050 }
1051 }
1052 }
1053 }
1054
1055 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
1056 Node* region = NULL;
1057 while (c != ctrl) {
1058 if (c->is_Region()) {
1059 region = c;
1060 }
1061 c = phase->idom(c);
1062 }
1063 assert(region != NULL, "");
1064 Node* phi = new PhiNode(region, n->bottom_type());
1065 for (uint j = 1; j < region->req(); j++) {
1066 Node* in = region->in(j);
1067 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1068 phi->init_req(j, n);
1069 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1070 phi->init_req(j, n_clone);
1071 } else {
1072 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1073 }
1074 }
1075 phase->register_new_node(phi, region);
1076 return phi;
1077 }
1078
1079 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1080 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1081
1082 Unique_Node_List uses;
1083 for (int i = 0; i < state->enqueue_barriers_count(); i++) {
1084 Node* barrier = state->enqueue_barrier(i);
1085 Node* ctrl = phase->get_ctrl(barrier);
1086 IdealLoopTree* loop = phase->get_loop(ctrl);
1087 Node* head = loop->head();
1088 if (head->is_OuterStripMinedLoop()) {
1089 // Expanding a barrier here will break loop strip mining
1165 if (phase->has_ctrl(in)) {
1166 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1167 #ifdef ASSERT
1168 for (uint i = 0; i < stack.size(); i++) {
1169 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1170 }
1171 #endif
1172 stack.push(in, 0);
1173 }
1174 } else {
1175 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1176 }
1177 }
1178 } else {
1179 phase->set_ctrl(n, call->in(0));
1180 stack.pop();
1181 }
1182 } while(stack.size() > 0);
1183 continue;
1184 }
1185 CallProjections projs;
1186 call->extract_projections(&projs, false, false);
1187
1188 #ifdef ASSERT
1189 VectorSet cloned;
1190 #endif
1191 Node* lrb_clone = lrb->clone();
1192 phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1193 phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1194
1195 stack.push(lrb, 0);
1196 clones.push(lrb_clone);
1197
1198 do {
1199 assert(stack.size() == clones.size(), "");
1200 Node* n = stack.node();
1201 #ifdef ASSERT
1202 if (n->is_Load()) {
1203 Node* mem = n->in(MemNode::Memory);
1204 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1205 Node* u = mem->fast_out(j);
1206 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1207 }
1208 }
1209 #endif
1210 uint idx = stack.index();
1211 Node* n_clone = clones.at(clones.size()-1);
1212 if (idx < n->outcnt()) {
1213 Node* u = n->raw_out(idx);
1214 Node* c = phase->ctrl_or_self(u);
1215 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
1216 stack.set_index(idx+1);
1217 assert(!u->is_CFG(), "");
1218 stack.push(u, 0);
1219 assert(!cloned.test_set(u->_idx), "only one clone");
1220 Node* u_clone = u->clone();
1221 int nb = u_clone->replace_edge(n, n_clone);
1222 assert(nb > 0, "should have replaced some uses");
1223 phase->register_new_node(u_clone, projs.catchall_catchproj);
1224 clones.push(u_clone);
1225 phase->set_ctrl(u, projs.fallthrough_catchproj);
1226 } else {
1227 bool replaced = false;
1228 if (u->is_Phi()) {
1229 for (uint k = 1; k < u->req(); k++) {
1230 if (u->in(k) == n) {
1231 if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1232 phase->igvn().replace_input_of(u, k, n_clone);
1233 replaced = true;
1234 } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1235 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1236 replaced = true;
1237 }
1238 }
1239 }
1240 } else {
1241 if (phase->is_dominator(projs.catchall_catchproj, c)) {
1242 phase->igvn().rehash_node_delayed(u);
1243 int nb = u->replace_edge(n, n_clone);
1244 assert(nb > 0, "should have replaced some uses");
1245 replaced = true;
1246 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1247 if (u->is_If()) {
1248 // Can't break If/Bool/Cmp chain
1249 assert(n->is_Bool(), "unexpected If shape");
1250 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1251 assert(n_clone->is_Bool(), "unexpected clone");
1252 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1253 Node* bol_clone = n->clone();
1254 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1255 bol_clone->set_req(1, cmp_clone);
1256
1257 Node* nn = stack.node_at(stack.size()-3);
1258 Node* nn_clone = clones.at(clones.size()-3);
1259 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1260
1261 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase));
1262 assert(nb > 0, "should have replaced some uses");
1263
1264 phase->register_new_node(bol_clone, u->in(0));
1265 phase->register_new_node(cmp_clone, u->in(0));
1266
2375 Node* u = c->fast_out(i);
2376 if (u->is_Phi() && u->bottom_type() == Type::MEMORY &&
2377 u != n) {
2378 if (u->adr_type() == TypePtr::BOTTOM) {
2379 fix_memory_uses(u, n, n, c);
2380 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
2381 _phase->lazy_replace(u, n);
2382 --i; --imax;
2383 }
2384 }
2385 }
2386 }
2387 }
2388 }
2389
2390 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2391 Node* c = _phase->get_ctrl(n);
2392 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Call()) {
2393 assert(c == n->in(0), "");
2394 CallNode* call = c->as_Call();
2395 CallProjections projs;
2396 call->extract_projections(&projs, true, false);
2397 if (projs.catchall_memproj != NULL) {
2398 if (projs.fallthrough_memproj == n) {
2399 c = projs.fallthrough_catchproj;
2400 } else {
2401 assert(projs.catchall_memproj == n, "");
2402 c = projs.catchall_catchproj;
2403 }
2404 }
2405 }
2406 return c;
2407 }
2408
2409 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2410 if (_phase->has_ctrl(n))
2411 return get_ctrl(n);
2412 else {
2413 assert (n->is_CFG(), "must be a CFG node");
2414 return n;
2415 }
2416 }
2417
2418 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2419 return m != NULL && get_ctrl(m) == c;
2420 }
2421
2422 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|
443 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
444 "sha512_implCompress",
445 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
446 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
447 "sha1_implCompressMB",
448 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
449 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
450 "sha256_implCompressMB",
451 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
452 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
453 "sha512_implCompressMB",
454 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
455 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
456 "encodeBlock",
457 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
458 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
459 };
460
461 if (call->is_call_to_arraycopystub()) {
462 Node* dest = NULL;
463 const TypeTuple* args = n->as_Call()->_tf->domain_sig();
464 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
465 if (args->field_at(i)->isa_ptr()) {
466 j++;
467 if (j == 2) {
468 dest = n->in(i);
469 break;
470 }
471 }
472 }
473 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
474 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
475 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
476 }
477 } else if (strlen(call->_name) > 5 &&
478 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
479 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
480 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
481 }
482 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
483 // skip
562 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } },
563 Op_StrEquals,
564 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
565 Op_EncodeISOArray,
566 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
567 Op_HasNegatives,
568 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
569 Op_CastP2X,
570 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
571 Op_StrIndexOfChar,
572 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
573 };
574
575 const int others_len = sizeof(others) / sizeof(others[0]);
576 int i = 0;
577 for (; i < others_len; i++) {
578 if (others[i].opcode == n->Opcode()) {
579 break;
580 }
581 }
582 uint stop = n->is_Call() ? n->as_Call()->tf()->domain_sig()->cnt() : n->req();
583 if (i != others_len) {
584 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
585 for (uint j = 0; j < inputs_len; j++) {
586 int pos = others[i].inputs[j].pos;
587 if (pos == -1) {
588 break;
589 }
590 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
591 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
592 }
593 }
594 for (uint j = 1; j < stop; j++) {
595 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
596 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
597 uint k = 0;
598 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
599 if (k == inputs_len) {
600 fatal("arg %d for node %s not covered", j, n->Name());
601 }
602 }
782 mem_ctrl = phase->ctrl_or_self(mem);
783 }
784 return mem;
785 }
786
787 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
788 Node* mem = NULL;
789 Node* c = ctrl;
790 do {
791 if (c->is_Region()) {
792 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) {
793 Node* u = c->fast_out(i);
794 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
795 if (u->adr_type() == TypePtr::BOTTOM) {
796 mem = u;
797 }
798 }
799 }
800 } else {
801 if (c->is_Call() && c->as_Call()->adr_type() != NULL) {
802 CallProjections* projs = c->as_Call()->extract_projections(true, false);
803 if (projs->fallthrough_memproj != NULL) {
804 if (projs->fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
805 if (projs->catchall_memproj == NULL) {
806 mem = projs->fallthrough_memproj;
807 } else {
808 if (phase->is_dominator(projs->fallthrough_catchproj, ctrl)) {
809 mem = projs->fallthrough_memproj;
810 } else {
811 assert(phase->is_dominator(projs->catchall_catchproj, ctrl), "one proj must dominate barrier");
812 mem = projs->catchall_memproj;
813 }
814 }
815 }
816 } else {
817 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
818 if (proj != NULL &&
819 proj->adr_type() == TypePtr::BOTTOM) {
820 mem = proj;
821 }
822 }
823 } else {
824 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
825 Node* u = c->fast_out(i);
826 if (u->is_Proj() &&
827 u->bottom_type() == Type::MEMORY &&
828 u->adr_type() == TypePtr::BOTTOM) {
829 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
830 assert(mem == NULL, "only one proj");
831 mem = u;
832 }
1034 if (c != ctrl ||
1035 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1036 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1037 phase->igvn().rehash_node_delayed(u);
1038 int nb = u->replace_edge(ctrl, region);
1039 if (u->is_CFG()) {
1040 if (phase->idom(u) == ctrl) {
1041 phase->set_idom(u, region, phase->dom_depth(region));
1042 }
1043 } else if (phase->get_ctrl(u) == ctrl) {
1044 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1045 uses.push(u);
1046 }
1047 assert(nb == 1, "more than 1 ctrl input?");
1048 --i, imax -= nb;
1049 }
1050 }
1051 }
1052 }
1053
1054 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections* projs, PhaseIdealLoop* phase) {
1055 Node* region = NULL;
1056 while (c != ctrl) {
1057 if (c->is_Region()) {
1058 region = c;
1059 }
1060 c = phase->idom(c);
1061 }
1062 assert(region != NULL, "");
1063 Node* phi = new PhiNode(region, n->bottom_type());
1064 for (uint j = 1; j < region->req(); j++) {
1065 Node* in = region->in(j);
1066 if (phase->is_dominator(projs->fallthrough_catchproj, in)) {
1067 phi->init_req(j, n);
1068 } else if (phase->is_dominator(projs->catchall_catchproj, in)) {
1069 phi->init_req(j, n_clone);
1070 } else {
1071 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1072 }
1073 }
1074 phase->register_new_node(phi, region);
1075 return phi;
1076 }
1077
1078 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1079 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1080
1081 Unique_Node_List uses;
1082 for (int i = 0; i < state->enqueue_barriers_count(); i++) {
1083 Node* barrier = state->enqueue_barrier(i);
1084 Node* ctrl = phase->get_ctrl(barrier);
1085 IdealLoopTree* loop = phase->get_loop(ctrl);
1086 Node* head = loop->head();
1087 if (head->is_OuterStripMinedLoop()) {
1088 // Expanding a barrier here will break loop strip mining
1164 if (phase->has_ctrl(in)) {
1165 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1166 #ifdef ASSERT
1167 for (uint i = 0; i < stack.size(); i++) {
1168 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1169 }
1170 #endif
1171 stack.push(in, 0);
1172 }
1173 } else {
1174 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1175 }
1176 }
1177 } else {
1178 phase->set_ctrl(n, call->in(0));
1179 stack.pop();
1180 }
1181 } while(stack.size() > 0);
1182 continue;
1183 }
1184 CallProjections* projs = call->extract_projections(false, false);
1185 #ifdef ASSERT
1186 VectorSet cloned;
1187 #endif
1188 Node* lrb_clone = lrb->clone();
1189 phase->register_new_node(lrb_clone, projs->catchall_catchproj);
1190 phase->set_ctrl(lrb, projs->fallthrough_catchproj);
1191
1192 stack.push(lrb, 0);
1193 clones.push(lrb_clone);
1194
1195 do {
1196 assert(stack.size() == clones.size(), "");
1197 Node* n = stack.node();
1198 #ifdef ASSERT
1199 if (n->is_Load()) {
1200 Node* mem = n->in(MemNode::Memory);
1201 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1202 Node* u = mem->fast_out(j);
1203 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1204 }
1205 }
1206 #endif
1207 uint idx = stack.index();
1208 Node* n_clone = clones.at(clones.size()-1);
1209 if (idx < n->outcnt()) {
1210 Node* u = n->raw_out(idx);
1211 Node* c = phase->ctrl_or_self(u);
1212 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs->fallthrough_proj)) {
1213 stack.set_index(idx+1);
1214 assert(!u->is_CFG(), "");
1215 stack.push(u, 0);
1216 assert(!cloned.test_set(u->_idx), "only one clone");
1217 Node* u_clone = u->clone();
1218 int nb = u_clone->replace_edge(n, n_clone);
1219 assert(nb > 0, "should have replaced some uses");
1220 phase->register_new_node(u_clone, projs->catchall_catchproj);
1221 clones.push(u_clone);
1222 phase->set_ctrl(u, projs->fallthrough_catchproj);
1223 } else {
1224 bool replaced = false;
1225 if (u->is_Phi()) {
1226 for (uint k = 1; k < u->req(); k++) {
1227 if (u->in(k) == n) {
1228 if (phase->is_dominator(projs->catchall_catchproj, u->in(0)->in(k))) {
1229 phase->igvn().replace_input_of(u, k, n_clone);
1230 replaced = true;
1231 } else if (!phase->is_dominator(projs->fallthrough_catchproj, u->in(0)->in(k))) {
1232 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1233 replaced = true;
1234 }
1235 }
1236 }
1237 } else {
1238 if (phase->is_dominator(projs->catchall_catchproj, c)) {
1239 phase->igvn().rehash_node_delayed(u);
1240 int nb = u->replace_edge(n, n_clone);
1241 assert(nb > 0, "should have replaced some uses");
1242 replaced = true;
1243 } else if (!phase->is_dominator(projs->fallthrough_catchproj, c)) {
1244 if (u->is_If()) {
1245 // Can't break If/Bool/Cmp chain
1246 assert(n->is_Bool(), "unexpected If shape");
1247 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1248 assert(n_clone->is_Bool(), "unexpected clone");
1249 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1250 Node* bol_clone = n->clone();
1251 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1252 bol_clone->set_req(1, cmp_clone);
1253
1254 Node* nn = stack.node_at(stack.size()-3);
1255 Node* nn_clone = clones.at(clones.size()-3);
1256 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1257
1258 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase));
1259 assert(nb > 0, "should have replaced some uses");
1260
1261 phase->register_new_node(bol_clone, u->in(0));
1262 phase->register_new_node(cmp_clone, u->in(0));
1263
2372 Node* u = c->fast_out(i);
2373 if (u->is_Phi() && u->bottom_type() == Type::MEMORY &&
2374 u != n) {
2375 if (u->adr_type() == TypePtr::BOTTOM) {
2376 fix_memory_uses(u, n, n, c);
2377 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
2378 _phase->lazy_replace(u, n);
2379 --i; --imax;
2380 }
2381 }
2382 }
2383 }
2384 }
2385 }
2386
2387 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2388 Node* c = _phase->get_ctrl(n);
2389 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Call()) {
2390 assert(c == n->in(0), "");
2391 CallNode* call = c->as_Call();
2392 CallProjections* projs = call->extract_projections(true, false);
2393 if (projs->catchall_memproj != NULL) {
2394 if (projs->fallthrough_memproj == n) {
2395 c = projs->fallthrough_catchproj;
2396 } else {
2397 assert(projs->catchall_memproj == n, "");
2398 c = projs->catchall_catchproj;
2399 }
2400 }
2401 }
2402 return c;
2403 }
2404
2405 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2406 if (_phase->has_ctrl(n))
2407 return get_ctrl(n);
2408 else {
2409 assert (n->is_CFG(), "must be a CFG node");
2410 return n;
2411 }
2412 }
2413
2414 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2415 return m != NULL && get_ctrl(m) == c;
2416 }
2417
2418 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|