Expose loop-invariant inputs to GVN and LICM

docs/vir.md

@@ -256,17 +256,20 @@ elimination (DCE) removes unused work.
 
 1. Remove unreachable blocks.
 2. SCCP and CFG simplification.
-3. Local CSE.
-4. GVN.
-5. Reassociation, LICM and pointer-induction strength reduction.
-6. Local CSE and DCE, including ordinary overwritten-store elimination.
-7. CFG simplification, then a final local CSE and DCE over the simplified graph.
-8. Full graph verification before dumps and machine lowering.
+3. Block-parameter simplification: a parameter that receives one value on
+   every edge, or itself on a back edge, is replaced by that value, so a
+   loop-invariant input does not look loop-defined to CSE, GVN and LICM.
+4. Local CSE.
+5. GVN.
+6. Reassociation, LICM and pointer-induction strength reduction.
+7. Local CSE and DCE, including ordinary overwritten-store elimination.
+8. CFG simplification, then a final local CSE and DCE over the simplified graph.
+9. Full graph verification before dumps and machine lowering.
 
 | Level | Scheduled transformations |
 | --- | --- |
 | O0 | Construction folds, unreachable-block removal, and mandatory verification; optional optimization passes return without transforming the graph. |
-| O2 (default) | SCCP, CFG simplification, local CSE, GVN, reassociation, LICM, strength reduction, and DCE. |
+| O2 (default) | SCCP, CFG simplification, block-parameter simplification, local CSE, GVN, reassociation, LICM, strength reduction, and DCE. |
 
 Pass-local tables are temporary. O0 does not allocate the optional CSE, GVN,
 loop, or optimization worklists. Mandatory verification still checks dominance,

src/vir-frontend.c

@@ -1190,6 +1190,7 @@ void vir_frontend_optimize(vir_function_t *graph)
         fatal("Invalid VIR reachability");
     check_native_pass("sccp", vir_sccp(graph, level));
     check_native_pass("cfg", vir_simplify_cfg(graph, level));
+    check_native_pass("params", vir_simplify_params(graph, level));
     check_native_pass("cse", vir_local_cse(graph, level));
     check_native_pass("gvn", vir_gvn_with_stats(graph, level, NULL));
     check_native_pass("reassociate", vir_reassociate(graph, level));

src/vir.c

@@ -4073,32 +4073,46 @@ static bool vir_dominance_init(const vir_function_t *func,
     return true;
 }
 
-/* Drop loop-invariant and forwarding parameters without source SSA bindings. */
+/* The one value @param, the @index-th parameter of @block, receives on every
+ * incoming edge apart from itself, or NULL when the edges disagree.
+ */
+static vir_value_t *vir_param_trivial_value(const vir_block_t *block,
+                                            const vir_value_t *param,
+                                            int index)
+{
+    vir_value_t *candidate = NULL;
+
+    for (vir_edge_t *edge = block->incoming; edge; edge = edge->next_incoming) {
+        vir_value_t *value = edge->args[index];
+        if (value == param)
+            continue;
+        if (candidate && candidate != value)
+            return NULL;
+        candidate = value;
+    }
+    return candidate;
+}
+
+/* Drop loop-invariant and forwarding parameters without source SSA bindings.
+ * The entry block's parameters are the function's arguments, whose caller
+ * supplies a value no edge shows, so they always stay.
+ */
 static int vir_simplify_params_with_context(vir_function_t *func,
                                             const vir_dominance_t *dominance)
 {
     int changed = 0, previous;
     do {
         previous = changed;
-        for (vir_block_t *block = func->blocks; block; block = block->next) {
+        for (vir_block_t *block = func->blocks->next; block;
+             block = block->next) {
             int index = 0;
             for (vir_value_t *param = block->params, *next; param;
                  param = next) {
-                vir_value_t *candidate = NULL;
-                bool trivial = true;
+                vir_value_t *candidate =
+                    vir_param_trivial_value(block, param, index);
                 next = param->param_next;
-                for (vir_edge_t *edge = block->incoming; edge;
-                     edge = edge->next_incoming) {
-                    vir_value_t *value = edge->args[index];
-                    if (value == param)
-                        continue;
-                    if (candidate && candidate != value)
-                        trivial = false;
-                    candidate = value;
-                }
-                if (!trivial || !candidate ||
-                    !vir_replacement_dominates_with_context(func, dominance,
-                                                            param, candidate)) {
+                if (!candidate || !vir_replacement_dominates_with_context(
+                                      func, dominance, param, candidate)) {
                     index++;
                     continue;
                 }
@@ -4117,6 +4131,32 @@ static int vir_simplify_params_with_context(vir_function_t *func,
     return changed;
 }
 
+/* Dominance is built only once some parameter has a single incoming value. */
+int vir_simplify_params(vir_function_t *func, vir_opt_level_t opt_level)
+{
+    vir_dominance_t dominance;
+    int changed;
+
+    if (!func || !vir_opt_level_valid(opt_level))
+        return -1;
+    if (opt_level == VIR_OPT_O0 || !func->blocks)
+        return 0;
+    for (vir_block_t *block = func->blocks->next; block; block = block->next) {
+        int index = 0;
+        for (vir_value_t *param = block->params; param;
+             param = param->param_next, index++)
+            if (vir_param_trivial_value(block, param, index))
+                goto simplify;
+    }
+    return 0;
+simplify:
+    if (!vir_dominance_init(func, &dominance))
+        return -1;
+    changed = vir_simplify_params_with_context(func, &dominance);
+    vir_dominance_release(&dominance);
+    return changed;
+}
+
 /* An affine offset is safe to recur only while every narrower intermediate
  * stays nonnegative and fits its signed type. Pointer-width arithmetic itself
  * is modular, so ILP32 does not need the LP64 narrowing proof.
@@ -5036,6 +5076,22 @@ static bool vir_fold_branch_condition(vir_block_t *block)
     return true;
 }
 
+/* A branch whose two edges reach the same block with the same arguments goes
+ * there either way, so it is a jump, and DCE can drop its condition.
+ */
+static int vir_fold_same_target_branch(vir_function_t *func, vir_block_t *block)
+{
+    vir_edge_t *taken = block->true_edge, *other = block->false_edge;
+
+    if (block->terminator != VIR_TERM_BRANCH || taken->to != other->to ||
+        taken->arg_count != other->arg_count)
+        return 0;
+    for (int i = 0; i < taken->arg_count; i++)
+        if (taken->args[i] != other->args[i])
+            return 0;
+    return vir_edge_delete(func, other) ? 1 : -1;
+}
+
 /* An incoming constant can select a branch without materializing its boolean
  * parameter. Keep the bypassed computations local and successors argument-free.
  */
@@ -5117,6 +5173,14 @@ int vir_simplify_cfg_with_pruning(vir_function_t *func,
             vir_block_t *target;
             int inlined;
 
+            /* Folding only drops an edge, so the scan can go on in place. */
+            inlined = vir_fold_same_target_branch(func, block);
+            if (inlined < 0)
+                return -1;
+            if (inlined) {
+                changes++;
+                changed = true;
+            }
             inlined = vir_thread_constant_edge(func, block);
             if (!inlined)
                 inlined = vir_inline_return_param(func, block);

src/vir.h

@@ -406,16 +406,25 @@ int vir_remove_unreachable(vir_function_t *func);
 int vir_function_block_count(const vir_function_t *func);
 
 /* At O1/O2, bypass empty jump blocks and local branches selected by constant
- * incoming arguments. The regular helper also drops newly unreachable blocks;
- * the explicit helper can defer that deletion for a caller that must run
- * another pass on the rewritten topology. Branch threading excludes effects and
- * successor arguments. A deferred caller must prune before lowering (and before
- * any final reachability-sensitive verification).
+ * incoming arguments, and turn a branch whose arms reach one block with the
+ * same arguments into a jump. The regular helper also drops newly unreachable
+ * blocks; the explicit helper can defer that deletion for a caller that must
+ * run another pass on the rewritten topology. Branch threading excludes effects
+ * and successor arguments. A deferred caller must prune before lowering (and
+ * before any final reachability-sensitive verification).
  */
 int vir_simplify_cfg(vir_function_t *func, vir_opt_level_t opt_level);
 int vir_simplify_cfg_with_pruning(vir_function_t *func,
                                   vir_opt_level_t opt_level,
                                   bool prune_unreachable);
+
+/* At O1/O2, replace each block parameter that receives one value on every edge,
+ * or itself on a back edge, by that value. The entry block's parameters are the
+ * function's arguments and stay. A loop-invariant input otherwise reaches CSE,
+ * GVN and LICM as a value defined inside the loop. The CFG is unchanged.
+ */
+int vir_simplify_params(vir_function_t *func, vir_opt_level_t opt_level);
+
 int vir_block_set_branch(vir_function_t *func,
                          vir_block_t *block,
                          vir_value_t *condition,

tests/vir-core.c

@@ -2273,6 +2273,52 @@ int main(void)
     assert(vir_verify(&func, &error));
     vir_function_release(&func);
 
+    /* A header parameter that only carries an input around its loop hides a
+     * product of that input from GVN. Parameter simplification, skipped at O0,
+     * replaces it by the input; GVN then reuses the product computed before the
+     * loop inside it.
+     */
+    vir_function_init(&func, 64);
+    assert(vir_function_set_pointer_bits(&func, 64));
+    entry = vir_block_create(&func);
+    merge = vir_block_create(&func);
+    otherwise = vir_block_create(&func);
+    dead = vir_block_create(&func);
+    entry_param = vir_block_add_param(&func, entry, VIR_TYPE_I1);
+    one = vir_block_add_param(&func, entry, VIR_TYPE_I32);
+    two = vir_block_add_param(&func, entry, VIR_TYPE_I32);
+    stack_duplicate = vir_block_add_param(&func, entry, VIR_TYPE_PTR);
+    sum = vir_binary(&func, entry, VIR_OP_MUL, VIR_TYPE_I32, one, two);
+    param = vir_block_add_param(&func, merge, VIR_TYPE_I32);
+    product =
+        vir_binary(&func, otherwise, VIR_OP_MUL, VIR_TYPE_I32, param, two);
+    {
+        vir_value_t *entry_args[] = {one};
+        vir_value_t *backedge_args[] = {param};
+        vir_edge_args_t loop_true = {otherwise, NULL, 0};
+        vir_edge_args_t loop_false = {dead, NULL, 0};
+        vir_cse_stats_t gvn_stats;
+
+        assert(entry_param && one && two && sum && param && product &&
+               stack_duplicate &&
+               vir_store(&func, otherwise, stack_duplicate, product) &&
+               vir_edge_create(&func, entry, merge, entry_args, 1) &&
+               vir_edge_create(&func, otherwise, merge, backedge_args, 1) &&
+               vir_block_set_branch(&func, merge, entry_param, &loop_true,
+                                    &loop_false) &&
+               vir_block_set_return(&func, dead, sum) &&
+               vir_verify(&func, &error));
+        assert(vir_gvn_with_stats(&func, VIR_OPT_O2, &gvn_stats) == 0 &&
+               product->uses && merge->param_count == 1);
+        assert(vir_simplify_params(&func, VIR_OPT_O0) == 0 &&
+               merge->param_count == 1);
+        assert(vir_simplify_params(&func, VIR_OPT_O2) == 1 &&
+               merge->param_count == 0 && product->op0 == one);
+        assert(vir_gvn_with_stats(&func, VIR_OPT_O2, &gvn_stats) == 1 &&
+               !product->uses && sum->uses && vir_verify(&func, &error));
+    }
+    vir_function_release(&func);
+
     /* LICM derives a natural loop only at O2 and hoists an effect-free value
      * with operands defined outside the loop into its single-edge preheader.
      */
@@ -2397,6 +2443,73 @@ int main(void)
     }
     vir_function_release(&func);
 
+    /* An input that reaches the header unchanged around the loop sits behind a
+     * header parameter, so LICM treats a value computed from it as
+     * loop-defined. Once parameter simplification drops it, the value moves to
+     * the preheader.
+     */
+    vir_function_init(&func, 64);
+    entry = vir_block_create(&func);
+    merge = vir_block_create(&func);
+    otherwise = vir_block_create(&func);
+    dead = vir_block_create(&func);
+    entry_param = vir_block_add_param(&func, entry, VIR_TYPE_I1);
+    one = vir_block_add_param(&func, entry, VIR_TYPE_I32);
+    two = vir_block_add_param(&func, entry, VIR_TYPE_I32);
+    param = vir_block_add_param(&func, merge, VIR_TYPE_I32);
+    product =
+        vir_binary(&func, otherwise, VIR_OP_MUL, VIR_TYPE_I32, param, two);
+    {
+        vir_value_t *entry_args[] = {one};
+        vir_value_t *backedge_args[] = {param};
+        vir_edge_args_t loop_true = {otherwise, NULL, 0};
+        vir_edge_args_t loop_false = {dead, NULL, 0};
+
+        assert(entry && merge && otherwise && dead && entry_param && one &&
+               two && param && product &&
+               vir_edge_create(&func, entry, merge, entry_args, 1) &&
+               vir_edge_create(&func, otherwise, merge, backedge_args, 1) &&
+               vir_block_set_branch(&func, merge, entry_param, &loop_true,
+                                    &loop_false) &&
+               vir_block_set_return(&func, dead, param) &&
+               vir_verify(&func, &error));
+        assert(vir_licm(&func, VIR_OPT_O2) == 0 && product->block == otherwise);
+        assert(vir_simplify_params(&func, VIR_OPT_O2) == 1 &&
+               merge->param_count == 0 && product->op0 == one);
+        assert(vir_licm(&func, VIR_OPT_O2) == 1 && product->block == entry &&
+               vir_verify(&func, &error));
+    }
+    vir_function_release(&func);
+
+    /* The entry block's parameters are the function's arguments. A back edge
+     * into the entry block passing one value must not replace an argument.
+     */
+    vir_function_init(&func, 64);
+    entry = vir_block_create(&func);
+    merge = vir_block_create(&func);
+    dead = vir_block_create(&func);
+    one = vir_block_add_param(&func, entry, VIR_TYPE_I32);
+    entry_param = vir_block_add_param(&func, entry, VIR_TYPE_I1);
+    two = vir_const_i32(&func, merge, 5);
+    sum = vir_binary(&func, merge, VIR_OP_ADD, VIR_TYPE_I32, one, two);
+    {
+        vir_value_t *backedge_args[] = {two, entry_param};
+        vir_edge_args_t loop_true = {entry, backedge_args, 2};
+        vir_edge_args_t loop_false = {dead, NULL, 0};
+
+        assert(entry && merge && dead && one && entry_param && two && sum &&
+               vir_edge_create(&func, entry, merge, NULL, 0) &&
+               vir_block_set_branch(&func, merge, entry_param, &loop_true,
+                                    &loop_false) &&
+               vir_block_set_return(&func, dead, sum) &&
+               vir_verify(&func, &error));
+        assert(vir_simplify_params(&func, VIR_OPT_O2) == 0 &&
+               entry->param_count == 2 &&
+               (sum->op0 == one || sum->op1 == one) &&
+               vir_verify(&func, &error));
+    }
+    vir_function_release(&func);
+
     /* LICM's pass-local dominator bitsets keep sparse large-CFG storage below
      * the former four-byte-per-block-pair matrix while retaining loop motion.
      */
@@ -2518,7 +2631,8 @@ int main(void)
     vir_function_release(&func);
 
     /* CFG simplification redirects each predecessor through an empty jump
-     * block, preserves a duplicate branch target, and removes the forwarder.
+     * block, which leaves both branch arms on one target, collapses that branch
+     * into a jump, and removes the forwarders.
      */
     vir_function_init(&func, 64);
     entry = vir_block_create(&func);
@@ -2538,12 +2652,48 @@ int main(void)
                                     &cfg_false_desc));
         assert(vir_simplify_cfg(&func, VIR_OPT_O0) == 0);
         assert(vir_simplify_cfg(&func, VIR_OPT_O1) > 0 &&
-               entry->true_edge->to == dead && entry->false_edge->to == dead &&
+               entry->terminator == VIR_TERM_JUMP && entry->outgoing &&
+               entry->outgoing->to == dead && !entry->outgoing->next_outgoing &&
                func.blocks == entry && entry->next == dead && !dead->next);
     }
     assert(vir_verify(&func, &error));
     vir_function_release(&func);
 
+    /* Both arms on one target fold to a jump only when they also pass the same
+     * arguments; different arguments still select a value and stay a branch.
+     */
+    for (int same = 0; same < 2; same++) {
+        vir_function_init(&func, 64);
+        entry = vir_block_create(&func);
+        merge = vir_block_create(&func);
+        entry_param = vir_block_add_param(&func, entry, VIR_TYPE_I1);
+        one = vir_const_i32(&func, entry, 1);
+        two = vir_const_i32(&func, entry, 2);
+        param = vir_block_add_param(&func, merge, VIR_TYPE_I32);
+        vir_edge_args_t arm_true = {merge, &one, 1};
+        vir_edge_args_t arm_false = {merge, same ? &one : &two, 1};
+
+        assert(entry && merge && entry_param && one && two && param &&
+               vir_block_set_branch(&func, entry, entry_param, &arm_true,
+                                    &arm_false) &&
+               vir_block_set_return(&func, merge, param));
+
+        /* The folded jump then feeds a block that only returns its parameter,
+         * so CFG simplification goes on to return the value from the entry.
+         */
+        if (same)
+            assert(vir_simplify_cfg(&func, VIR_OPT_O2) > 0 &&
+                   entry->terminator == VIR_TERM_RETURN &&
+                   entry->return_value == one && !entry_param->uses);
+        else
+            assert(vir_simplify_cfg(&func, VIR_OPT_O2) == 0 &&
+                   entry->terminator == VIR_TERM_BRANCH &&
+                   entry->true_edge->args[0] == one &&
+                   entry->false_edge->args[0] == two);
+        assert(vir_verify(&func, &error));
+        vir_function_release(&func);
+    }
+
     /* Branches on promoted booleans preserve polarity and reverse-use links. */
     for (int width = 0; width < 2; width++) {
         for (int constant = 0; constant < 3; constant++) {