2010-02-04 Tom de Vries gcc/ stmt.c (set_jump_prob): Fix assert condition. 2010-01-27 Tom de Vries gcc/ stmt.c (rtx_seq_cost): Use insn_rtx_cost instead of rtx_cost. 2010-01-26 Tom de Vries gcc/ * stmt.c (struct case_bit_test): Add rev_hi and rev_lo field. * stmt.c (emit_case_bit_test_jump): New function. * stmt.c (rtx_seq_cost): New function. * stmt.c (choose_case_bit_test_expand_method): New function. * stmt.c (set_bit): New function. * stmt.c (emit_case_bit_test): Adjust comment. * stmt.c (emit_case_bit_test): Set and update rev_hi and rev_lo fields. * stmt.c (emit_case_bit_test): Use set_bit. * stmt.c (emit_case_bit_test): Use choose_case_bit_test_expand_method. * stmt.c (emit_case_bit_test): Use emit_case_bit_test_jump. * testsuite/gcc.dg/switch-bittest.c: New test. 2010-01-25 Tom de Vries gcc/ * stmt.c (emit_case_bit_tests): Change prototype. * stmt.c (struct case_bit_test): Add prob field. * stmt.c (get_label_prob): New function. * stmt.c (set_jump_prob): New function. * stmt.c (emit_case_bit_tests): Use get_label_prob. * stmt.c (emit_case_bit_tests): Set prob field. * stmt.c (emit_case_bit_tests): Use set_jump_prob. * stmt.c (expand_case): Add new args to emit_case_bit_tests invocation. * testsuite/gcc.dg/switch-prob.c: Add test. === modified file 'gcc/stmt.c' Index: gcc-4_5-branch/gcc/stmt.c =================================================================== --- gcc-4_5-branch.orig/gcc/stmt.c +++ gcc-4_5-branch/gcc/stmt.c @@ -117,7 +117,8 @@ static void expand_value_return (rtx); static int estimate_case_costs (case_node_ptr); static bool lshift_cheap_p (void); static int case_bit_test_cmp (const void *, const void *); -static void emit_case_bit_tests (tree, tree, tree, tree, case_node_ptr, rtx); +static void emit_case_bit_tests (tree, tree, tree, tree, case_node_ptr, tree, + rtx, basic_block); static void balance_case_nodes (case_node_ptr *, case_node_ptr); static int node_has_low_bound (case_node_ptr, tree); static int node_has_high_bound (case_node_ptr, tree); @@ -2107,8 +2108,11 @@ struct case_bit_test { HOST_WIDE_INT hi; HOST_WIDE_INT lo; + HOST_WIDE_INT rev_hi; + HOST_WIDE_INT rev_lo; rtx label; int bits; + int prob; }; /* Determine whether "1 << x" is relatively cheap in word_mode. */ @@ -2148,10 +2152,193 @@ case_bit_test_cmp (const void *p1, const return CODE_LABEL_NUMBER (d2->label) - CODE_LABEL_NUMBER (d1->label); } +/* Emit a bit test and a conditional jump. */ + +static void +emit_case_bit_test_jump (unsigned int count, rtx index, rtx label, + unsigned int method, HOST_WIDE_INT hi, + HOST_WIDE_INT lo, HOST_WIDE_INT rev_hi, + HOST_WIDE_INT rev_lo) +{ + rtx expr; + + if (method == 1) + { + /* (1 << index). */ + if (count == 0) + index = expand_binop (word_mode, ashl_optab, const1_rtx, + index, NULL_RTX, 1, OPTAB_WIDEN); + /* CST. */ + expr = immed_double_const (lo, hi, word_mode); + /* ((1 << index) & CST). */ + expr = expand_binop (word_mode, and_optab, index, expr, + NULL_RTX, 1, OPTAB_WIDEN); + /* if (((1 << index) & CST)). */ + emit_cmp_and_jump_insns (expr, const0_rtx, NE, NULL_RTX, + word_mode, 1, label); + } + else if (method == 2) + { + /* (bit_reverse (CST)) */ + expr = immed_double_const (rev_lo, rev_hi, word_mode); + /* ((bit_reverse (CST)) << index) */ + expr = expand_binop (word_mode, ashl_optab, expr, + index, NULL_RTX, 1, OPTAB_WIDEN); + /* if (((bit_reverse (CST)) << index) < 0). */ + emit_cmp_and_jump_insns (expr, const0_rtx, LT, NULL_RTX, + word_mode, 0, label); + } + else + gcc_unreachable (); +} + +/* Return the cost of rtx sequence SEQ. The sequence is supposed to contain one + jump, which has no effect in the cost. */ + +static unsigned int +rtx_seq_cost (rtx seq) +{ + rtx one; + unsigned int nr_branches = 0; + unsigned int sum = 0, cost; + + for (one = seq; one != NULL_RTX; one = NEXT_INSN (one)) + if (JUMP_P (one)) + nr_branches++; + else + { + cost = insn_rtx_cost (PATTERN (one), optimize_insn_for_speed_p ()); + if (dump_file) + { + print_rtl_single (dump_file, one); + fprintf (dump_file, "cost: %u\n", cost); + } + sum += cost; + } + + gcc_assert (nr_branches == 1); + + if (dump_file) + fprintf (dump_file, "total cost: %u\n", sum); + return sum; +} + +/* Generate the rtx sequences for 2 bit test expansion methods, measure the cost + and choose the cheapest. */ + +static unsigned int +choose_case_bit_test_expand_method (rtx label) +{ + rtx seq, index; + unsigned int cost[2]; + static bool method_known = false; + static unsigned int method; + + /* If already known, return the method. */ + if (method_known) + return method; + + index = gen_rtx_REG (word_mode, 10000); + + for (method = 1; method <= 2; ++method) + { + start_sequence (); + emit_case_bit_test_jump (0, index, label, method, 0, 0x0f0f0f0f, 0, + 0x0f0f0f0f); + seq = get_insns (); + end_sequence (); + cost[method - 1] = rtx_seq_cost (seq); + } + + /* Determine method based on heuristic. */ + method = ((cost[1] < cost[0]) ? 1 : 0) + 1; + + /* Save and return method. */ + method_known = true; + return method; +} + +/* Get the edge probability of the edge from SRC to LABEL_DECL. */ + +static int +get_label_prob (basic_block src, tree label_decl) +{ + basic_block dest; + int prob = 0, nr_prob = 0; + unsigned int i; + edge e; + + if (label_decl == NULL_TREE) + return 0; + + dest = VEC_index (basic_block, label_to_block_map, + LABEL_DECL_UID (label_decl)); + + for (i = 0; i < EDGE_COUNT (src->succs); ++i) + { + e = EDGE_SUCC (src, i); + + if (e->dest != dest) + continue; + + prob += e->probability; + nr_prob++; + } + + gcc_assert (nr_prob == 1); + + return prob; +} + +/* Add probability note with scaled PROB to JUMP and update INV_SCALE. This + function is intended to be used with a series of conditional jumps to L[i] + where the probabilities p[i] to get to L[i] are known, and the jump + probabilities j[i] need to be computed. + + The algorithm to calculate the probabilities is + + scale = REG_BR_PROB_BASE; + for (i = 0; i < n; ++i) + { + j[i] = p[i] * scale / REG_BR_PROB_BASE; + f[i] = REG_BR_PROB_BASE - j[i]; + scale = scale / (f[i] / REG_BR_PROB_BASE); + } + + The implementation uses inv_scale (REG_BR_PROB_BASE / scale) instead of + scale, because scale tends to grow bigger than REG_BR_PROB_BASE. */ + +static void +set_jump_prob (rtx jump, int prob, int *inv_scale) +{ + /* j[i] = p[i] * scale / REG_BR_PROB_BASE. */ + int jump_prob = prob * REG_BR_PROB_BASE / *inv_scale; + /* f[i] = REG_BR_PROB_BASE - j[i]. */ + int fallthrough_prob = REG_BR_PROB_BASE - jump_prob; + + gcc_assert (jump_prob <= REG_BR_PROB_BASE); + add_reg_note (jump, REG_BR_PROB, GEN_INT (jump_prob)); + + /* scale = scale / (f[i] / REG_BR_PROB_BASE). */ + *inv_scale = *inv_scale * fallthrough_prob / REG_BR_PROB_BASE; +} + +/* Set bit in hwi hi/lo pair. */ + +static void +set_bit (HOST_WIDE_INT *hi, HOST_WIDE_INT *lo, unsigned int j) +{ + if (j >= HOST_BITS_PER_WIDE_INT) + *hi |= (HOST_WIDE_INT) 1 << (j - HOST_BITS_PER_INT); + else + *lo |= (HOST_WIDE_INT) 1 << j; +} + /* Expand a switch statement by a short sequence of bit-wise comparisons. "switch(x)" is effectively converted into - "if ((1 << (x-MINVAL)) & CST)" where CST and MINVAL are - integer constants. + "if ((1 << (x-MINVAL)) & CST)" or + "if (((bit_reverse (CST)) << (x-MINVAL)) < 0)", where CST + and MINVAL are integer constants. INDEX_EXPR is the value being switched on, which is of type INDEX_TYPE. MINVAL is the lowest case value of in @@ -2165,14 +2352,17 @@ case_bit_test_cmp (const void *p1, const static void emit_case_bit_tests (tree index_type, tree index_expr, tree minval, - tree range, case_node_ptr nodes, rtx default_label) + tree range, case_node_ptr nodes, tree default_label_decl, + rtx default_label, basic_block bb) { struct case_bit_test test[MAX_CASE_BIT_TESTS]; enum machine_mode mode; rtx expr, index, label; unsigned int i,j,lo,hi; struct case_node *n; - unsigned int count; + unsigned int count, method; + int inv_scale = REG_BR_PROB_BASE; + int default_prob = get_label_prob (bb, default_label_decl); count = 0; for (n = nodes; n; n = n->right) @@ -2187,8 +2377,11 @@ emit_case_bit_tests (tree index_type, tr gcc_assert (count < MAX_CASE_BIT_TESTS); test[i].hi = 0; test[i].lo = 0; + test[i].rev_hi = 0; + test[i].rev_lo = 0; test[i].label = label; test[i].bits = 1; + test[i].prob = get_label_prob (bb, n->code_label); count++; } else @@ -2199,10 +2392,11 @@ emit_case_bit_tests (tree index_type, tr hi = tree_low_cst (fold_build2 (MINUS_EXPR, index_type, n->high, minval), 1); for (j = lo; j <= hi; j++) - if (j >= HOST_BITS_PER_WIDE_INT) - test[i].hi |= (HOST_WIDE_INT) 1 << (j - HOST_BITS_PER_INT); - else - test[i].lo |= (HOST_WIDE_INT) 1 << j; + { + set_bit (&test[i].hi, &test[i].lo, j); + set_bit (&test[i].rev_hi, &test[i].rev_lo, + GET_MODE_BITSIZE (word_mode) - j - 1); + } } qsort (test, count, sizeof(*test), case_bit_test_cmp); @@ -2216,20 +2410,20 @@ emit_case_bit_tests (tree index_type, tr mode = TYPE_MODE (index_type); expr = expand_normal (range); if (default_label) - emit_cmp_and_jump_insns (index, expr, GTU, NULL_RTX, mode, 1, - default_label); + { + emit_cmp_and_jump_insns (index, expr, GTU, NULL_RTX, mode, 1, + default_label); + set_jump_prob (get_last_insn (), default_prob / 2, &inv_scale); + } index = convert_to_mode (word_mode, index, 0); - index = expand_binop (word_mode, ashl_optab, const1_rtx, - index, NULL_RTX, 1, OPTAB_WIDEN); + method = choose_case_bit_test_expand_method (test[0].label); for (i = 0; i < count; i++) { - expr = immed_double_const (test[i].lo, test[i].hi, word_mode); - expr = expand_binop (word_mode, and_optab, index, expr, - NULL_RTX, 1, OPTAB_WIDEN); - emit_cmp_and_jump_insns (expr, const0_rtx, NE, NULL_RTX, - word_mode, 1, test[i].label); + emit_case_bit_test_jump (i, index, test[i].label, method, test[i].hi, + test[i].lo, test[i].rev_hi, test[i].rev_lo); + set_jump_prob (get_last_insn (), test[i].prob, &inv_scale); } if (default_label) @@ -2400,7 +2594,8 @@ expand_case (gimple stmt) range = maxval; } emit_case_bit_tests (index_type, index_expr, minval, range, - case_list, default_label); + case_list, default_label_decl, default_label, + gimple_bb (stmt)); } /* If range of values is much bigger than number of values, Index: gcc-4_5-branch/gcc/testsuite/gcc.dg/switch-bittest.c =================================================================== --- /dev/null +++ gcc-4_5-branch/gcc/testsuite/gcc.dg/switch-bittest.c @@ -0,0 +1,25 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -fdump-rtl-expand" } */ + +const char * +f (const char *p) +{ + while (1) + { + switch (*p) + { + case 9: + case 10: + case 13: + case 32: + break; + default: + return p; + } + } +} + +/* { dg-final { scan-rtl-dump-times "jump_insn" 4 "expand" { target mips*-*-* } } } */ +/* { dg-final { scan-rtl-dump-times "REG_BR_PROB" 2 "expand" { target mips*-*-* } } } */ +/* { dg-final { scan-rtl-dump-times "lt " 1 "expand" { target mips*-*-* } } } */ +/* { dg-final { cleanup-rtl-dump "expand" } } */ Index: gcc-4_5-branch/gcc/testsuite/gcc.dg/switch-prob.c =================================================================== --- /dev/null +++ gcc-4_5-branch/gcc/testsuite/gcc.dg/switch-prob.c @@ -0,0 +1,25 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -fdump-rtl-expand" } */ + +const char * +f (const char *p) +{ + while (1) + { + switch (*p) + { + case 9: + case 10: + case 13: + case 32: + break; + default: + return p; + } + } +} + +/* { dg-final { scan-rtl-dump-times "jump_insn" 4 "expand" { target mips*-*-* } } } */ +/* { dg-final { scan-rtl-dump-times "REG_BR_PROB" 2 "expand" { target mips*-*-* } } } */ +/* { dg-final { scan-rtl-dump-times "heuristics" 0 "expand" { target mips*-*-* } } } */ +/* { dg-final { cleanup-rtl-dump "expand" } } */