MDEV-40417 Fix default value for compressed columns

mysql-test/main/column_compression.result

@@ -1323,7 +1323,7 @@ LENGTH(a)
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `a` blob /*M!100301 COMPRESSED*/ NOT NULL DEFAULT ''
+  `a` blob /*M!100301 COMPRESSED*/ NOT NULL
 ) ENGINE=CSV DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 DROP TABLE t1;
@@ -3051,3 +3051,127 @@ REPLACE INTO t VALUES ('abcdefghijklm');
 UPDATE t SET c=MID(c,2,4);
 DROP TABLE t;
 # End of 10.6 tests
+#
+# MDEV-40417 A column with compression enabled automatically adds DEFAULT ''
+#
+CREATE TABLE t (
+c1 LONGTEXT COMPRESSED NOT NULL,
+nc1 LONGTEXT NOT NULL,
+c2 LONGTEXT COMPRESSED NOT NULL DEFAULT '',
+nc2 LONGTEXT NOT NULL DEFAULT '',
+c3 VARCHAR(100) COMPRESSED NOT NULL,
+nc3 VARCHAR(100) NOT NULL
+) ENGINE=InnoDB;
+SHOW CREATE TABLE t;
+Table	Create Table
+t	CREATE TABLE `t` (
+  `c1` longtext /*M!100301 COMPRESSED*/ NOT NULL,
+  `nc1` longtext NOT NULL,
+  `c2` longtext /*M!100301 COMPRESSED*/ NOT NULL DEFAULT '',
+  `nc2` longtext NOT NULL DEFAULT '',
+  `c3` varchar(100) /*M!100301 COMPRESSED*/ NOT NULL,
+  `nc3` varchar(100) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
+SELECT COLUMN_NAME, COLUMN_DEFAULT FROM INFORMATION_SCHEMA.COLUMNS
+WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t' ORDER BY ORDINAL_POSITION;
+COLUMN_NAME	COLUMN_DEFAULT
+c1	NULL
+nc1	NULL
+c2	''
+nc2	''
+c3	NULL
+nc3	NULL
+# A compressed column must be as mandatory as an uncompressed one
+INSERT INTO t (nc1) VALUES ('x');
+ERROR HY000: Field 'c1' doesn't have a default value
+INSERT INTO t (c1, nc1, nc3) VALUES ('x', 'x', 'x');
+ERROR HY000: Field 'c3' doesn't have a default value
+INSERT INTO t (c1, nc1, c3, nc3) VALUES ('x', 'x', 'x', 'x');
+SELECT * FROM t;
+c1	nc1	c2	nc2	c3	nc3
+x	x			x	x
+DROP TABLE t;
+# ALTER TABLE must not add an implicit default either
+CREATE TABLE t (c LONGTEXT COMPRESSED NOT NULL DEFAULT 'x') ENGINE=InnoDB;
+ALTER TABLE t MODIFY c LONGTEXT COMPRESSED NOT NULL;
+ALTER TABLE t ADD c2 VARCHAR(100) COMPRESSED NOT NULL;
+SHOW CREATE TABLE t;
+Table	Create Table
+t	CREATE TABLE `t` (
+  `c` longtext /*M!100301 COMPRESSED*/ NOT NULL,
+  `c2` varchar(100) /*M!100301 COMPRESSED*/ NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
+DROP TABLE t;
+#
+# MDEV-40417 upgrade: a table created before the fix is repaired when
+# its FRM is read.  Only blobs can be repaired - for VARCHAR the FRM
+# does not tell an explicit DEFAULT '' from the wrong implicit one,
+# so those columns are deliberately left alone.
+#
+# MDEV-40417.frm was created by 10.11.19 as:
+#   CREATE TABLE mdev40417 (
+#     blob_nodef  LONGTEXT COMPRESSED NOT NULL,
+#     blob_def    LONGTEXT COMPRESSED NOT NULL DEFAULT 'x',
+#     vc_nodef    VARCHAR(100) COMPRESSED NOT NULL,
+#     vc_def      VARCHAR(100) COMPRESSED NOT NULL DEFAULT '',
+#     plain_nodef LONGTEXT NOT NULL,
+#     pad         INT) ENGINE=MyISAM CHARSET=latin1;
+#   INSERT INTO mdev40417 (plain_nodef, pad) VALUES ('p', 1);
+#
+call mtr.add_suppression("Found wrong implicit DEFAULT");
+SHOW CREATE TABLE mdev40417;
+Table	Create Table
+mdev40417	CREATE TABLE `mdev40417` (
+  `blob_nodef` longtext /*M!100301 COMPRESSED*/ NOT NULL,
+  `blob_def` longtext /*M!100301 COMPRESSED*/ NOT NULL DEFAULT 'x',
+  `vc_nodef` varchar(100) /*M!100301 COMPRESSED*/ NOT NULL DEFAULT '',
+  `vc_def` varchar(100) /*M!100301 COMPRESSED*/ NOT NULL DEFAULT '',
+  `plain_nodef` longtext NOT NULL,
+  `pad` int(11) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
+# For the error log reporting see main.column_compression_errlog
+SELECT COLUMN_NAME, COLUMN_DEFAULT FROM INFORMATION_SCHEMA.COLUMNS
+WHERE TABLE_SCHEMA='test' AND TABLE_NAME='mdev40417' ORDER BY ORDINAL_POSITION;
+COLUMN_NAME	COLUMN_DEFAULT
+blob_nodef	NULL
+blob_def	'x'
+vc_nodef	''
+vc_def	''
+plain_nodef	NULL
+pad	NULL
+# The row that the old server allowed to be inserted is still readable
+SELECT * FROM mdev40417;
+blob_nodef	blob_def	vc_nodef	vc_def	plain_nodef	pad
+	x			p	1
+# blob_nodef is mandatory again
+INSERT INTO mdev40417 (blob_def, vc_nodef, vc_def, plain_nodef)
+VALUES ('a', 'b', 'c', 'd');
+ERROR HY000: Field 'blob_nodef' doesn't have a default value
+# plain_nodef was never affected by the bug
+INSERT INTO mdev40417 (blob_nodef, blob_def, vc_nodef, vc_def)
+VALUES ('a', 'b', 'c', 'd');
+ERROR HY000: Field 'plain_nodef' doesn't have a default value
+# blob_def keeps its explicit DEFAULT
+INSERT INTO mdev40417 (blob_nodef, vc_nodef, vc_def, plain_nodef)
+VALUES ('a', 'b', 'c', 'd');
+# vc_nodef is not touched, it still has the old implicit DEFAULT ''
+INSERT INTO mdev40417 (blob_nodef, blob_def, plain_nodef) VALUES ('a', 'b', 'd');
+SELECT blob_nodef, blob_def, vc_nodef, vc_def, plain_nodef FROM mdev40417;
+blob_nodef	blob_def	vc_nodef	vc_def	plain_nodef
+	x			p
+a	b			d
+a	x	b	c	d
+# ALTER TABLE ... FORCE writes the repaired flag back into the FRM
+ALTER TABLE mdev40417 FORCE;
+SHOW CREATE TABLE mdev40417;
+Table	Create Table
+mdev40417	CREATE TABLE `mdev40417` (
+  `blob_nodef` longtext /*M!100301 COMPRESSED*/ NOT NULL,
+  `blob_def` longtext /*M!100301 COMPRESSED*/ NOT NULL DEFAULT 'x',
+  `vc_nodef` varchar(100) /*M!100301 COMPRESSED*/ NOT NULL DEFAULT '',
+  `vc_def` varchar(100) /*M!100301 COMPRESSED*/ NOT NULL DEFAULT '',
+  `plain_nodef` longtext NOT NULL,
+  `pad` int(11) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
+DROP TABLE mdev40417;
+# End of 10.11 tests

mysql-test/main/column_compression.test

@@ -594,3 +594,88 @@ UPDATE t SET c=MID(c,2,4);
 DROP TABLE t;
 
 --echo # End of 10.6 tests
+
+--echo #
+--echo # MDEV-40417 A column with compression enabled automatically adds DEFAULT ''
+--echo #
+
+CREATE TABLE t (
+  c1 LONGTEXT COMPRESSED NOT NULL,
+  nc1 LONGTEXT NOT NULL,
+  c2 LONGTEXT COMPRESSED NOT NULL DEFAULT '',
+  nc2 LONGTEXT NOT NULL DEFAULT '',
+  c3 VARCHAR(100) COMPRESSED NOT NULL,
+  nc3 VARCHAR(100) NOT NULL
+) ENGINE=InnoDB;
+SHOW CREATE TABLE t;
+SELECT COLUMN_NAME, COLUMN_DEFAULT FROM INFORMATION_SCHEMA.COLUMNS
+WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t' ORDER BY ORDINAL_POSITION;
+
+--echo # A compressed column must be as mandatory as an uncompressed one
+--error ER_NO_DEFAULT_FOR_FIELD
+INSERT INTO t (nc1) VALUES ('x');
+--error ER_NO_DEFAULT_FOR_FIELD
+INSERT INTO t (c1, nc1, nc3) VALUES ('x', 'x', 'x');
+INSERT INTO t (c1, nc1, c3, nc3) VALUES ('x', 'x', 'x', 'x');
+SELECT * FROM t;
+DROP TABLE t;
+
+--echo # ALTER TABLE must not add an implicit default either
+CREATE TABLE t (c LONGTEXT COMPRESSED NOT NULL DEFAULT 'x') ENGINE=InnoDB;
+ALTER TABLE t MODIFY c LONGTEXT COMPRESSED NOT NULL;
+ALTER TABLE t ADD c2 VARCHAR(100) COMPRESSED NOT NULL;
+SHOW CREATE TABLE t;
+DROP TABLE t;
+
+--echo #
+--echo # MDEV-40417 upgrade: a table created before the fix is repaired when
+--echo # its FRM is read.  Only blobs can be repaired - for VARCHAR the FRM
+--echo # does not tell an explicit DEFAULT '' from the wrong implicit one,
+--echo # so those columns are deliberately left alone.
+--echo #
+--echo # MDEV-40417.frm was created by 10.11.19 as:
+--echo #   CREATE TABLE mdev40417 (
+--echo #     blob_nodef  LONGTEXT COMPRESSED NOT NULL,
+--echo #     blob_def    LONGTEXT COMPRESSED NOT NULL DEFAULT 'x',
+--echo #     vc_nodef    VARCHAR(100) COMPRESSED NOT NULL,
+--echo #     vc_def      VARCHAR(100) COMPRESSED NOT NULL DEFAULT '',
+--echo #     plain_nodef LONGTEXT NOT NULL,
+--echo #     pad         INT) ENGINE=MyISAM CHARSET=latin1;
+--echo #   INSERT INTO mdev40417 (plain_nodef, pad) VALUES ('p', 1);
+--echo #
+
+call mtr.add_suppression("Found wrong implicit DEFAULT");
+
+--copy_file std_data/MDEV-40417.frm $MYSQLD_DATADIR/test/mdev40417.frm
+--copy_file std_data/MDEV-40417.MYD $MYSQLD_DATADIR/test/mdev40417.MYD
+--copy_file std_data/MDEV-40417.MYI $MYSQLD_DATADIR/test/mdev40417.MYI
+
+SHOW CREATE TABLE mdev40417;
+--echo # For the error log reporting see main.column_compression_errlog
+SELECT COLUMN_NAME, COLUMN_DEFAULT FROM INFORMATION_SCHEMA.COLUMNS
+WHERE TABLE_SCHEMA='test' AND TABLE_NAME='mdev40417' ORDER BY ORDINAL_POSITION;
+--echo # The row that the old server allowed to be inserted is still readable
+SELECT * FROM mdev40417;
+
+--echo # blob_nodef is mandatory again
+--error ER_NO_DEFAULT_FOR_FIELD
+INSERT INTO mdev40417 (blob_def, vc_nodef, vc_def, plain_nodef)
+VALUES ('a', 'b', 'c', 'd');
+--echo # plain_nodef was never affected by the bug
+--error ER_NO_DEFAULT_FOR_FIELD
+INSERT INTO mdev40417 (blob_nodef, blob_def, vc_nodef, vc_def)
+VALUES ('a', 'b', 'c', 'd');
+--echo # blob_def keeps its explicit DEFAULT
+INSERT INTO mdev40417 (blob_nodef, vc_nodef, vc_def, plain_nodef)
+VALUES ('a', 'b', 'c', 'd');
+--echo # vc_nodef is not touched, it still has the old implicit DEFAULT ''
+INSERT INTO mdev40417 (blob_nodef, blob_def, plain_nodef) VALUES ('a', 'b', 'd');
+--sorted_result
+SELECT blob_nodef, blob_def, vc_nodef, vc_def, plain_nodef FROM mdev40417;
+
+--echo # ALTER TABLE ... FORCE writes the repaired flag back into the FRM
+ALTER TABLE mdev40417 FORCE;
+SHOW CREATE TABLE mdev40417;
+DROP TABLE mdev40417;
+
+--echo # End of 10.11 tests

mysql-test/main/column_compression_errlog.result

@@ -0,0 +1,12 @@
+call mtr.add_suppression("Found wrong implicit DEFAULT");
+# std_data/MDEV-40417.* was created by 10.11.19 before the fix, see the
+# MDEV-40417 section of main.column_compression for its definition.
+# Reading the old .frm reports the repaired blob column
+SELECT COUNT(*) FROM mdev40417;
+COUNT(*)
+1
+FOUND 1 /Found wrong implicit DEFAULT '' for compressed field 'blob_nodef'/ in mysqld.1.err
+# ... and only that one, vc_nodef cannot be recognized as affected
+NOT FOUND /Found wrong implicit DEFAULT '' for compressed field 'vc_nodef'/ in mysqld.1.err
+DROP TABLE mdev40417;
+# End of 10.11 tests

mysql-test/main/column_compression_errlog.test

@@ -0,0 +1,33 @@
+#
+# MDEV-40417 A column with compression enabled automatically adds DEFAULT ''
+#
+# Error log reporting for a table created before the fix.  This lives in a
+# file of its own because reading the server error log needs
+# include/not_embedded.inc, and sourcing that from main.column_compression
+# would skip that whole test for embedded server runs.
+#
+--source include/not_embedded.inc
+
+call mtr.add_suppression("Found wrong implicit DEFAULT");
+
+let $MYSQLD_DATADIR= `select @@datadir`;
+
+--echo # std_data/MDEV-40417.* was created by 10.11.19 before the fix, see the
+--echo # MDEV-40417 section of main.column_compression for its definition.
+--copy_file std_data/MDEV-40417.frm $MYSQLD_DATADIR/test/mdev40417.frm
+--copy_file std_data/MDEV-40417.MYD $MYSQLD_DATADIR/test/mdev40417.MYD
+--copy_file std_data/MDEV-40417.MYI $MYSQLD_DATADIR/test/mdev40417.MYI
+
+--echo # Reading the old .frm reports the repaired blob column
+SELECT COUNT(*) FROM mdev40417;
+--let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err
+--let SEARCH_PATTERN= Found wrong implicit DEFAULT '' for compressed field 'blob_nodef'
+--source include/search_pattern_in_file.inc
+
+--echo # ... and only that one, vc_nodef cannot be recognized as affected
+--let SEARCH_PATTERN= Found wrong implicit DEFAULT '' for compressed field 'vc_nodef'
+--source include/search_pattern_in_file.inc
+
+DROP TABLE mdev40417;
+
+--echo # End of 10.11 tests

sql/field.cc

@@ -10937,7 +10937,7 @@ bool Column_definition::check(THD *thd)
     We need to do this check here and in mysql_create_prepare_table() as
     sp_head::fill_field_definition() calls this function.
   */
-  if (!default_value && unireg_check == Field::NONE && (flags & NOT_NULL_FLAG))
+  if (!default_value && !has_default_function() && (flags & NOT_NULL_FLAG))
   {
     /*
       TIMESTAMP columns get implicit DEFAULT value when

sql/field.h

@@ -5486,7 +5486,12 @@ class Column_definition: public Sql_alloc,
 
   bool has_default_function() const
   {
-    return unireg_check != Field::NONE;
+    /*
+      TMYSQL_COMPRESSED is not a default function, it is stored in
+      unireg_check only to remember that the column is COMPRESSED.
+    */
+    return unireg_check != Field::NONE &&
+           unireg_check != Field::TMYSQL_COMPRESSED;
   }
 
   Field *make_field(TABLE_SHARE *share, MEM_ROOT *mem_root,

sql/table.cc

@@ -3351,7 +3351,38 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
     for (k=0, ptr= share->field ; *ptr ; ptr++, k++)
     {
       if ((*ptr)->flags & BLOB_FLAG)
+      {
+        Field *field= *ptr;
 	(*save++)= k;
+
+        /*
+          Tables created before MDEV-40417 was fixed have no
+          FIELDFLAG_NO_DEFAULT in the FRM for a COMPRESSED NOT NULL column
+          without an explicit DEFAULT clause, so such a column wrongly looks
+          like it has DEFAULT ''.  Repair the flag here, an explicit DEFAULT
+          of a blob column is always stored in the FRM as an expression, see
+          Column_definition::has_default_expression(), hence a blob that has
+          no default_value provably had no DEFAULT clause.  For FRMs written
+          after the fix the flag is set already and nothing is done.
+
+          VARCHAR and VARBINARY cannot be repaired at all: a constant DEFAULT
+          of a non-blob column is stored in the default record, exactly like
+          the wrong implicit default, which makes the two indistinguishable.
+        */
+        if (field->compression_method() && !field->default_value &&
+            !field->vcol_info &&
+            (field->flags & (NOT_NULL_FLAG | NO_DEFAULT_VALUE_FLAG)) ==
+              NOT_NULL_FLAG)
+        {
+          field->flags|= NO_DEFAULT_VALUE_FLAG;
+          sql_print_warning("Found wrong implicit DEFAULT '' for compressed "
+                            "field '%s' of %`s.%`s; Please do "
+                            "\"ALTER TABLE %`s FORCE\" to fix it.",
+                            field->field_name.str,
+                            share->db.str, share->table_name.str,
+                            share->table_name.str);
+        }
+      }
     }
   }