MDEV-39014: FULL JOIN Phase 2

mysql-test/main/table_elim.result

@@ -1112,3 +1112,35 @@ DROP TABLE t1, t2;
 #
 # End of 10.11 tests
 #
+#
+# MDEV-38136 Prevent elimination of tables in a FULL OUTER JOIN
+#
+create table t1 (a int);
+insert into t1 values (0),(1),(2),(3);
+create table t2 (a int primary key, b int)
+as select a, a as b from t1 where a in (1,2);
+create table t3 (a int primary key, b int)
+as select a, a as b from t1 where a in (1,3);
+# These will not be eliminated because contains a FULL OUTER JOIN.
+explain extended select t1.a from t1 full join t2 on t2.a=t1.a;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	4	100.00	
+1	SIMPLE	t2	eq_ref	PRIMARY	PRIMARY	4	test.t1.a	1	100.00	Using where; Using index
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` full join `test`.`t2` on(`test`.`t2`.`a` = `test`.`t1`.`a` and `test`.`t1`.`a` is not null) where 1
+explain extended select t1.a from t1 full join (t2 full join t3 on t2.b=t3.b) on t2.a=t1.a and t3.a=t1.a;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	4	100.00	
+1	SIMPLE	t2	eq_ref	PRIMARY	PRIMARY	4	test.t1.a	1	100.00	Using where
+1	SIMPLE	t3	eq_ref	PRIMARY	PRIMARY	4	test.t2.a	1	100.00	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` full join (`test`.`t3` join `test`.`t2`) on(`test`.`t2`.`a` = `test`.`t1`.`a` and `test`.`t3`.`a` = `test`.`t1`.`a` and `test`.`t3`.`b` = `test`.`t2`.`b` and `test`.`t1`.`a` is not null and `test`.`t2`.`a` is not null) where 1
+explain extended select t1.a from t1 full join (t2 left join t3 on t2.b=t3.b) on t2.a=t1.a and t3.a=t1.a;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	4	100.00	
+1	SIMPLE	t2	eq_ref	PRIMARY	PRIMARY	4	test.t1.a	1	100.00	Using where
+1	SIMPLE	t3	eq_ref	PRIMARY	PRIMARY	4	test.t2.a	1	100.00	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` full join (`test`.`t2` join `test`.`t3`) on(`test`.`t2`.`a` = `test`.`t1`.`a` and `test`.`t3`.`a` = `test`.`t1`.`a` and `test`.`t3`.`b` = `test`.`t2`.`b` and `test`.`t1`.`a` is not null and `test`.`t2`.`a` is not null) where 1
+drop table t1, t2, t3;
+# End of 12.3  tests

mysql-test/main/table_elim.test

@@ -844,3 +844,24 @@ DROP TABLE t1, t2;
 --echo #
 --echo # End of 10.11 tests
 --echo #
+
+--echo #
+--echo # MDEV-38136 Prevent elimination of tables in a FULL OUTER JOIN
+--echo #
+create table t1 (a int);
+insert into t1 values (0),(1),(2),(3);
+
+create table t2 (a int primary key, b int)
+  as select a, a as b from t1 where a in (1,2);
+
+create table t3 (a int primary key, b int)
+  as select a, a as b from t1 where a in (1,3);
+
+--echo # These will not be eliminated because contains a FULL OUTER JOIN.
+explain extended select t1.a from t1 full join t2 on t2.a=t1.a;
+explain extended select t1.a from t1 full join (t2 full join t3 on t2.b=t3.b) on t2.a=t1.a and t3.a=t1.a;
+explain extended select t1.a from t1 full join (t2 left join t3 on t2.b=t3.b) on t2.a=t1.a and t3.a=t1.a;
+
+drop table t1, t2, t3;
+
+--echo # End of 12.3  tests

sql/opt_range.cc

@@ -1216,6 +1216,11 @@ SQL_SELECT *make_select(TABLE *head, table_map const_tables,
 
   *error=0;
 
+  /*
+    If there's no condition at all then NULLs could end up in
+    the result set.  However, we can disallow that with
+    allow_null_cond == false.
+   */
   if (!conds && !allow_null_cond)
     DBUG_RETURN(0);
   if (!(select= new (head->in_use->mem_root) SQL_SELECT))

sql/opt_table_elimination.cc

@@ -825,7 +825,8 @@ eliminate_tables_for_list(JOIN *join, List<TABLE_LIST> *join_list,
       if (tbl->nested_join)
       {
         /* This is  "... LEFT JOIN (join_nest) ON cond" */
-        if (eliminate_tables_for_list(join,
+        if (!(tbl->outer_join & JOIN_TYPE_FULL) &&
+            eliminate_tables_for_list(join,
                                       &tbl->nested_join->join_list, 
                                       tbl->nested_join->used_tables, 
                                       tbl->on_expr,
@@ -840,7 +841,8 @@ eliminate_tables_for_list(JOIN *join, List<TABLE_LIST> *join_list,
       else
       {
         /* This is  "... LEFT JOIN tbl ON cond" */
-        if (!(tbl->table->map & outside_used_tables) &&
+        if (!(tbl->outer_join & JOIN_TYPE_FULL) &&
+            !(tbl->table->map & outside_used_tables) &&
             check_func_dependency(join, tbl->table->map, NULL, tbl, 
                                   tbl->on_expr))
         {
@@ -2105,7 +2107,8 @@ void Dep_analysis_context::dbug_print_deps()
     char buf[128];
     String str(buf, sizeof(buf), &my_charset_bin);
     str.length(0);
-    eq_mod->expr->print(&str, QT_ORDINARY);
+    if (eq_mod->expr)
+      eq_mod->expr->print(&str, QT_ORDINARY);
     if (eq_mod->field)
     {
       fprintf(DBUG_FILE, "  equality%ld: %s -> %s.%s\n", 

sql/share/errmsg-utf8.txt

@@ -12392,3 +12392,5 @@ ER_CANNOT_CAST_ON_IDENT1_ASSIGNMENT_FOR_OPERATION
         eng "Cannot cast '%s' as '%s' in assignment of %sQ for operation %s"
 ER_CANNOT_CAST_ON_IDENT2_ASSIGNMENT_FOR_OPERATION
         eng "Cannot cast '%s' as '%s' in assignment of %sQ.%sQ for operation %s"
+ER_FULL_JOIN_BASE_TABLES_ONLY
+        eng "FULL JOIN is only supported with base tables on the right side; '%s' is not a base table"

sql/sql_base.cc

@@ -7562,8 +7562,15 @@ mark_common_columns(THD *thd, TABLE_LIST *table_ref_1, TABLE_LIST *table_ref_2,
 
     /*
       Create non-fixed fully qualified field and let fix_fields to
-      resolve it.
+      resolve it.  Clear the natural_full_join_field pointer--we recursed
+      on the nested join tree (store_top_level_join_columns) and picked
+      new common columns, so these natural_full_join_field items are invalid.
+      They will be updated to valid fields shortly, on the next call to
+      coalesce_natural_full_join via store_natural_using_join_columns
+      (case of NATURAL FULL JOIN only).
     */
+    nj_col_1->natural_full_join_field= nullptr;
+    nj_col_2->natural_full_join_field= nullptr;
     Item *item_1=   nj_col_1->create_item(thd);
     Item *item_2=   nj_col_2->create_item(thd);
     Item_ident *item_ident_1, *item_ident_2;
@@ -7648,6 +7655,56 @@ mark_common_columns(THD *thd, TABLE_LIST *table_ref_1, TABLE_LIST *table_ref_2,
 }
 
 
+/*
+  For some pair of tables (t1, t2) such that
+    t1 NATURAL FULL JOIN t2
+  generate a set of output columns
+    COALESCE(t1.x_1, t2.y_1), ..., COALESCE(t1.x_n, t2.y_n)
+  such that NULL results won't appear in the NATURAL FULL JOIN.
+
+  @parma thd                 the current thread
+  @param left_join_columns   common columns originating in t1
+  @param right_join_columns  common columns originating in t2
+ */
+void coalesce_natural_full_join(THD *thd,
+                                List<Natural_join_column> *left_join_columns,
+                                List<Natural_join_column> *right_join_columns)
+{
+  /*
+    It's a NATURAL JOIN so the number of columns from the left table better
+    match the number from the right table.
+  */
+  DBUG_ASSERT(left_join_columns->elements == right_join_columns->elements);
+
+  /*
+    Walk the left table and right table columns in lock-step, creating a
+    new COALESCE() over each pair of columns.  The calling function relies
+    on the state of left_join_columns, so set the COALESCE() item instance
+    on members of that list.
+   */
+  List_iterator<Natural_join_column> left(*left_join_columns);
+  List_iterator<Natural_join_column> right(*right_join_columns);
+  Natural_join_column *left_col= left++;
+  Natural_join_column *right_col= right++;
+  while (!left.at_end() && !right.at_end())
+  {
+    Item *left_field= left_col->get_item();
+    Item *right_field= right_col->get_item();
+    Item_func_coalesce *coal= new (thd->mem_root) Item_func_coalesce
+      (thd, left_field, right_field);
+    DBUG_ASSERT(coal);
+
+    // Makes the field `COALESCE(left, right) AS left`.
+    coal->set_name(thd, left_field->name);
+
+    // Save the result into the set of left_join_columns.
+    left_col->natural_full_join_field= coal;
+
+    left_col= left++;
+    right_col= right++;
+  }
+}
+
 
 /*
   Materialize and store the row type of NATURAL/USING join.
@@ -7696,15 +7753,17 @@ store_natural_using_join_columns(THD *thd, TABLE_LIST *natural_using_join,
   Query_arena *arena, backup;
   bool result= TRUE;
   List<Natural_join_column> *non_join_columns;
-  List<Natural_join_column> *join_columns;
+  List<Natural_join_column> *left_join_columns;
+  List<Natural_join_column> *right_join_columns;
   DBUG_ENTER("store_natural_using_join_columns");
 
   DBUG_ASSERT(!natural_using_join->join_columns);
 
   arena= thd->activate_stmt_arena_if_needed(&backup);
 
   if (!(non_join_columns= new List<Natural_join_column>) ||
-      !(join_columns= new List<Natural_join_column>))
+      !(left_join_columns= new List<Natural_join_column>) ||
+      !(right_join_columns= new List<Natural_join_column>))
     goto err;
 
   /* Append the columns of the first join operand. */
@@ -7713,7 +7772,7 @@ store_natural_using_join_columns(THD *thd, TABLE_LIST *natural_using_join,
     nj_col_1= it_1.get_natural_column_ref();
     if (nj_col_1->is_common)
     {
-      join_columns->push_back(nj_col_1, thd->mem_root);
+      left_join_columns->push_back(nj_col_1, thd->mem_root);
       /* Reset the common columns for the next call to mark_common_columns. */
       nj_col_1->is_common= FALSE;
     }
@@ -7733,7 +7792,7 @@ store_natural_using_join_columns(THD *thd, TABLE_LIST *natural_using_join,
     while ((using_field_name= using_fields_it++))
     {
       List_iterator_fast<Natural_join_column>
-        it(*join_columns);
+        it(*left_join_columns);
       Natural_join_column *common_field;
 
       for (;;)
@@ -7760,14 +7819,24 @@ store_natural_using_join_columns(THD *thd, TABLE_LIST *natural_using_join,
       non_join_columns->push_back(nj_col_2, thd->mem_root);
     else
     {
+      right_join_columns->push_back(nj_col_2, thd->mem_root);
+
       /* Reset the common columns for the next call to mark_common_columns. */
       nj_col_2->is_common= FALSE;
     }
   }
 
+  /*
+    If this is a NATURAL FULL JOIN, then create a COALESCE() for each pair of
+    columns from the two joined tables.
+  */
+  if ((table_ref_1->outer_join & JOIN_TYPE_FULL) &&
+      (table_ref_2->outer_join & JOIN_TYPE_FULL))
+    coalesce_natural_full_join(thd, left_join_columns, right_join_columns);
+
   if (non_join_columns->elements > 0)
-    join_columns->append(non_join_columns);
-  natural_using_join->join_columns= join_columns;
+    left_join_columns->append(non_join_columns);
+  natural_using_join->join_columns= left_join_columns;
   natural_using_join->is_join_columns_complete= TRUE;
 
   result= FALSE;
@@ -7858,7 +7927,8 @@ store_top_level_join_columns(THD *thd, TABLE_LIST *table_ref,
         swapped in the first loop.
       */
       if (same_level_left_neighbor &&
-          cur_table_ref->outer_join & JOIN_TYPE_RIGHT)
+          (cur_table_ref->outer_join & JOIN_TYPE_RIGHT) &&
+          !(cur_table_ref->outer_join & JOIN_TYPE_FULL))
       {
         /* This can happen only for JOIN ... ON. */
         DBUG_ASSERT(table_ref->nested_join->join_list.elements == 2);
@@ -8796,23 +8866,38 @@ insert_fields(THD *thd, Name_resolution_context *context,
       {
         DBUG_ASSERT((tables->field_translation == NULL && table) ||
                     tables->is_natural_join);
-        DBUG_ASSERT(item->type() == Item::FIELD_ITEM);
-        Item_field *fld= (Item_field*) item;
-        const char *field_db_name= field_iterator.get_db_name().str;
-        const char *field_table_name= field_iterator.get_table_name().str;
-
-        if (!tables->schema_table && 
-            !(fld->have_privileges=
-              (get_column_grant(thd, field_iterator.grant(),
-                                field_db_name,
-                                field_table_name, fld->field_name) &
-               VIEW_ANY_ACL)))
+        if (item->type() == Item::FIELD_ITEM)
         {
-          my_error(ER_TABLEACCESS_DENIED_ERROR, MYF(0), "ANY",
-                   thd->security_ctx->priv_user,
-                   thd->security_ctx->host_or_ip,
-                   field_db_name, field_table_name);
-          DBUG_RETURN(TRUE);
+          Item_field *fld= (Item_field*) item;
+          const char *field_db_name= field_iterator.get_db_name().str;
+          const char *field_table_name= field_iterator.get_table_name().str;
+
+          if (!tables->schema_table &&
+              !(fld->have_privileges=
+                (get_column_grant(thd, field_iterator.grant(),
+                                  field_db_name,
+                                  field_table_name, fld->field_name) &
+                 VIEW_ANY_ACL)))
+          {
+            my_error(ER_TABLEACCESS_DENIED_ERROR, MYF(0), "ANY",
+                     thd->security_ctx->priv_user,
+                     thd->security_ctx->host_or_ip,
+                     field_db_name, field_table_name);
+            DBUG_RETURN(TRUE);
+          }
+        }
+        else
+        {
+          /*
+            For NATURAL FULL JOIN, common columns are represented
+            as COALESCE expressions rather than plain Item_field.  The
+            per-column privilege check is skipped because the underlying
+            fields already had their privileges verified during name
+            resolution.  Assert that this is indeed a FULL JOIN context.
+          */
+          DBUG_ASSERT(tables->is_natural_join && tables->nested_join &&
+                      (tables->nested_join->join_list.head()->outer_join &
+                       JOIN_TYPE_FULL));
         }
       }
 #endif

sql/sql_lex.cc

@@ -1346,6 +1346,7 @@ void LEX::start(THD *thd_arg)
 
   memset(&trg_chistics, 0, sizeof(trg_chistics));
   selects_for_hint_resolution.empty();
+  full_join_count= 0;
   DBUG_VOID_RETURN;
 }
 
@@ -4903,11 +4904,26 @@ static void fix_prepare_info_in_table_list(THD *thd, TABLE_LIST *tbl)
 {
   for (; tbl; tbl= tbl->next_local)
   {
-    if (tbl->on_expr && !tbl->prep_on_expr)
+    /*
+      Walk up the nested JOINs so that upper-level ON expressions also
+      get saved into their respective prep_on_expr's.
+
+      We must do this to support PS:
+        prepare st from
+          'select ... from t1 full join t2 on t1.a = t2.a';
+    */
+    TABLE_LIST *embedding= tbl;
+    do
     {
-      thd->check_and_register_item_tree(&tbl->prep_on_expr, &tbl->on_expr);
-      tbl->on_expr= tbl->on_expr->copy_andor_structure(thd);
+      if (embedding->on_expr && !embedding->prep_on_expr)
+      {
+        thd->check_and_register_item_tree(&embedding->prep_on_expr,
+                                          &embedding->on_expr);
+        embedding->on_expr= embedding->on_expr->copy_andor_structure(thd);
+      }
     }
+    while ((embedding= embedding->embedding));
+
     if (tbl->is_view_or_derived() && tbl->is_merged_derived())
     {
       SELECT_LEX *sel= tbl->get_single_select();

sql/sql_lex.h

@@ -3526,6 +3526,9 @@ struct LEX: public Query_tables_list
   uint8 context_analysis_only;
   uint8 lex_options; // see OPTION_LEX_*
 
+  /* Zero by default, this counts the number of FULL JOINs in the query. */
+  uint16 full_join_count;
+
   Alter_info alter_info;
   Lex_prepared_stmt prepared_stmt;
   /*