From 9802c95c1dd5c300ba1932c8f2ab69a888735f69 Mon Sep 17 00:00:00 2001 From: Mark Callaghan Date: Thu, 11 May 2023 13:13:38 -0700 Subject: [PATCH] Fix issues 1200 and 1295, add rocksdb_compact_lzero_now global variable (#1309) Summary: This fixes issue 1295 and has a workaround for issue 1200. https://github.com/facebook/mysql-5.6/issues/1200 https://github.com/facebook/mysql-5.6/issues/1295 The fix for issue 1295 is to get the value of the base level from RocksDB rather than assuming that L0 compacts into L1 because L1 might not be the base level when dynamic leveled compaction is used -- in that case the base level can be Ln where n > 1 because L1 isn't needed yet. There are two workarounds for issue 1200. The hacky one is to sleep for 1 second between requesting memtable flush and L0 -> base_level compaction. The alternative is to add a new global variable, rocksdb_compact_lzero_now, that when set will request L0 -> base_level compaction. This allows a client to first set rocksdb_force_flush_memtable_now, wait for that to finish, then set rocksdb_compact_lzero_now. Other changes: * rocksdb_force_flush_memtable_now default value changed to OFF to match what is done for rocksdb_force_flush_memtable_and_lzero_now and rocksdb_compact_lzero_now * confirm that the value to which these variables are set can be parsed, that wasn't done for all of them * don't raise an error when these are set to OFF, that will be a no-op Note: these variables are triggers, an action is taken when this is done: set global var = ON | true | 1 But these variables always show the value 0 (OFF, false). Their value doesn't change. Pull Request resolved: https://github.com/facebook/mysql-5.6/pull/1309 Differential Revision: D45789784 --- mysql-test/r/mysqld--help-notwin.result | 6 +- mysql-test/suite/rocksdb/r/rocksdb.result | 3 +- .../r/rocksdb_compact_lzero_now_basic.result | 83 +++++++++++++ ..._flush_memtable_and_lzero_now_basic.result | 39 +++++- ...ksdb_force_flush_memtable_now_basic.result | 54 +++++---- .../t/rocksdb_compact_lzero_now_basic.test | 20 ++++ ...ce_flush_memtable_and_lzero_now_basic.test | 5 +- ...ocksdb_force_flush_memtable_now_basic.test | 6 +- storage/rocksdb/ha_rocksdb.cc | 111 +++++++++++++++--- 9 files changed, 278 insertions(+), 49 deletions(-) create mode 100644 mysql-test/suite/rocksdb_sys_vars/r/rocksdb_compact_lzero_now_basic.result create mode 100644 mysql-test/suite/rocksdb_sys_vars/t/rocksdb_compact_lzero_now_basic.test diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result index 0acaaa9eebbb..a160e833ea22 100644 --- a/mysql-test/r/mysqld--help-notwin.result +++ b/mysql-test/r/mysqld--help-notwin.result @@ -1975,6 +1975,8 @@ The following options may be given as the first argument: (Defaults to on; use --skip-rocksdb-commit-time-batch-for-recovery to disable.) --rocksdb-compact-cf=name Compact column family + --rocksdb-compact-lzero-now + Compacts all L0 files. --rocksdb-compaction-history[=name] Enable or disable ROCKSDB_COMPACTION_HISTORY plugin. Possible values are ON, OFF, FORCE (don't start if the @@ -2147,7 +2149,6 @@ The following options may be given as the first argument: --rocksdb-force-flush-memtable-now Forces memstore flush which may block all write requests so be careful - (Defaults to on; use --skip-rocksdb-force-flush-memtable-now to disable.) --rocksdb-force-index-records-in-range=# Used to override the result of records_in_range() when FORCE INDEX is used. @@ -3633,6 +3634,7 @@ rocksdb-column-default-value-as-expression TRUE rocksdb-commit-in-the-middle FALSE rocksdb-commit-time-batch-for-recovery TRUE rocksdb-compact-cf +rocksdb-compact-lzero-now FALSE rocksdb-compaction-history ON rocksdb-compaction-readahead-size 2097152 rocksdb-compaction-sequential-deletes 0 @@ -3686,7 +3688,7 @@ rocksdb-flush-log-at-trx-commit 1 rocksdb-force-compute-memtable-stats TRUE rocksdb-force-compute-memtable-stats-cachetime 60000000 rocksdb-force-flush-memtable-and-lzero-now FALSE -rocksdb-force-flush-memtable-now TRUE +rocksdb-force-flush-memtable-now FALSE rocksdb-force-index-records-in-range 0 rocksdb-global-info ON rocksdb-ignore-unknown-options TRUE diff --git a/mysql-test/suite/rocksdb/r/rocksdb.result b/mysql-test/suite/rocksdb/r/rocksdb.result index a2b55cbf6cfb..a2da4795ac8c 100644 --- a/mysql-test/suite/rocksdb/r/rocksdb.result +++ b/mysql-test/suite/rocksdb/r/rocksdb.result @@ -936,6 +936,7 @@ rocksdb_column_default_value_as_expression ON rocksdb_commit_in_the_middle OFF rocksdb_commit_time_batch_for_recovery ON rocksdb_compact_cf +rocksdb_compact_lzero_now OFF rocksdb_compaction_readahead_size 2097152 rocksdb_compaction_sequential_deletes 0 rocksdb_compaction_sequential_deletes_count_sd OFF @@ -984,7 +985,7 @@ rocksdb_flush_log_at_trx_commit 1 rocksdb_force_compute_memtable_stats ON rocksdb_force_compute_memtable_stats_cachetime 0 rocksdb_force_flush_memtable_and_lzero_now OFF -rocksdb_force_flush_memtable_now ON +rocksdb_force_flush_memtable_now OFF rocksdb_force_index_records_in_range 0 rocksdb_ignore_unknown_options ON rocksdb_index_type kBinarySearch diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_compact_lzero_now_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_compact_lzero_now_basic.result new file mode 100644 index 000000000000..28302a0a4671 --- /dev/null +++ b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_compact_lzero_now_basic.result @@ -0,0 +1,83 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +SET @start_global_value = @@global.ROCKSDB_COMPACT_LZERO_NOW; +SELECT @start_global_value; +@start_global_value +0 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to 1" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = 1; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to on" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = on; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to true" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = true; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to 0" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = 0; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to off" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = off; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to false" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = false; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@session.ROCKSDB_COMPACT_LZERO_NOW to 444. It should fail because it is not session." +SET @@session.ROCKSDB_COMPACT_LZERO_NOW = 444; +ERROR HY000: Variable 'rocksdb_compact_lzero_now' is a GLOBAL variable and should be set with SET GLOBAL +'# Testing with invalid values in global scope #' +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = @start_global_value; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result index 68cfeb07fc71..a3e506753d90 100644 --- a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result +++ b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result @@ -1,7 +1,10 @@ CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; INSERT INTO valid_values VALUES(1); -INSERT INTO valid_values VALUES(0); INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; SET @start_global_value = @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; SELECT @start_global_value; @@ -18,6 +21,26 @@ SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW 0 +"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to on" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = on; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to true" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = true; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to 0" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = 0; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; @@ -28,8 +51,18 @@ SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW 0 -"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to on" -SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = on; +"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to off" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = off; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to false" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = false; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW 0 diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_now_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_now_basic.result index 53665a7cedaa..59f6c832da39 100644 --- a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_now_basic.result +++ b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_now_basic.result @@ -2,70 +2,82 @@ CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; INSERT INTO valid_values VALUES(1); INSERT INTO valid_values VALUES('on'); INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; -INSERT INTO invalid_values VALUES(0); -INSERT INTO invalid_values VALUES('off'); -INSERT INTO invalid_values VALUES('false'); SET @start_global_value = @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; SELECT @start_global_value; @start_global_value -1 +0 '# Setting to valid values in global scope#' "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to 1" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = 1; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Setting the global scope variable back to default" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to on" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = on; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Setting the global scope variable back to default" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to true" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = true; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Setting the global scope variable back to default" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 -"Trying to set variable @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to 444. It should fail because it is not session." -SET @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = 444; -ERROR HY000: Variable 'rocksdb_force_flush_memtable_now' is a GLOBAL variable and should be set with SET GLOBAL -'# Testing with invalid values in global scope #' +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to 0" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = 0; -Got one of the listed errors SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to off" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = off; -Got one of the listed errors SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to false" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = false; -Got one of the listed errors SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW +0 +"Trying to set variable @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to 444. It should fail because it is not session." +SET @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = 444; +ERROR HY000: Variable 'rocksdb_force_flush_memtable_now' is a GLOBAL variable and should be set with SET GLOBAL +'# Testing with invalid values in global scope #' SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = @start_global_value; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 DROP TABLE valid_values; DROP TABLE invalid_values; diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_compact_lzero_now_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_compact_lzero_now_basic.test new file mode 100644 index 000000000000..eb9431a7d09d --- /dev/null +++ b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_compact_lzero_now_basic.test @@ -0,0 +1,20 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; + +--let $sys_var=ROCKSDB_COMPACT_LZERO_NOW +--let $read_only=0 +--let $session=0 +--let $sticky=1 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test index 5eeac5010407..39bbe0eb6e4d 100644 --- a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test +++ b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test @@ -2,8 +2,11 @@ CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; INSERT INTO valid_values VALUES(1); -INSERT INTO valid_values VALUES(0); INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_now_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_now_basic.test index b493d5a80eef..4761f9b44406 100644 --- a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_now_basic.test +++ b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_now_basic.test @@ -4,11 +4,11 @@ CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; INSERT INTO valid_values VALUES(1); INSERT INTO valid_values VALUES('on'); INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; -INSERT INTO invalid_values VALUES(0); -INSERT INTO invalid_values VALUES('off'); -INSERT INTO invalid_values VALUES('false'); --let $sys_var=ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW --let $read_only=0 diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index cbed6fa875b8..f97fcfe2de51 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -542,6 +542,8 @@ static void rocksdb_select_bypass_rejected_query_history_size_update( my_core::THD *const thd, my_core::SYS_VAR *const /* unused */, void *const var_ptr, const void *const save); +static int delete_range(const std::unordered_set &indices); + static void rocksdb_force_flush_memtable_now_stub( THD *const thd MY_ATTRIBUTE((__unused__)), struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), @@ -554,8 +556,11 @@ static int rocksdb_force_flush_memtable_now( void *const var_ptr MY_ATTRIBUTE((__unused__)), struct st_mysql_value *const value MY_ATTRIBUTE((__unused__))) { bool parsed_value = false; - if (mysql_value_to_bool(value, &parsed_value) != 0 || !parsed_value) { + if (mysql_value_to_bool(value, &parsed_value) != 0) { return 1; + } else if (!parsed_value) { + // Setting to OFF is a no-op and this supports mtr tests + return HA_EXIT_SUCCESS; } // NO_LINT_DEBUG @@ -564,22 +569,10 @@ static int rocksdb_force_flush_memtable_now( rocksdb_flush_all_memtables(); return HA_EXIT_SUCCESS; } -static int delete_range(const std::unordered_set &indices); -static void rocksdb_force_flush_memtable_and_lzero_now_stub( - THD *const thd MY_ATTRIBUTE((__unused__)), - struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), - void *const var_ptr MY_ATTRIBUTE((__unused__)), - const void *const save MY_ATTRIBUTE((__unused__))) {} -static int rocksdb_force_flush_memtable_and_lzero_now( - THD *const thd MY_ATTRIBUTE((__unused__)), - struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), - void *const var_ptr MY_ATTRIBUTE((__unused__)), - struct st_mysql_value *const value MY_ATTRIBUTE((__unused__))) { +static int rocksdb_compact_lzero() { // NO_LINT_DEBUG - LogPluginErrMsg(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, - "RocksDB: Manual memtable and L0 flush."); - rocksdb_flush_all_memtables(); + LogPluginErrMsg(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, "RocksDB: compact L0."); const Rdb_cf_manager &cf_manager = rdb_get_cf_manager(); rocksdb::CompactionOptions c_options = rocksdb::CompactionOptions(); @@ -594,6 +587,25 @@ static int rocksdb_force_flush_memtable_and_lzero_now( cf_handle->GetDescriptor(&cf_descr); c_options.output_file_size_limit = cf_descr.options.target_file_size_base; + // Lets RocksDB use the configured compression for this level + c_options.compression = rocksdb::kDisableCompressionOption; + + uint64_t base_level; + if (!rdb->GetIntProperty(cf_handle.get(), + rocksdb::DB::Properties::kBaseLevel, + &base_level)) { + LogPluginErrMsg(ERROR_LEVEL, ER_LOG_PRINTF_MSG, + "MyRocks: compact L0 cannot get base level"); + break; + } + + if (base_level == 0) { + LogPluginErrMsg( + ERROR_LEVEL, ER_LOG_PRINTF_MSG, + "MyRocks: compact L0 cannot flush to base level when 0"); + break; + } + assert(metadata.levels[0].level == 0); std::vector file_names; for (auto &file : metadata.levels[0].files) { @@ -601,11 +613,13 @@ static int rocksdb_force_flush_memtable_and_lzero_now( } if (file_names.empty()) { + LogPluginErrMsg(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, + "MyRocks: no files in L0"); break; } rocksdb::Status s; - s = rdb->CompactFiles(c_options, cf_handle.get(), file_names, 1); + s = rdb->CompactFiles(c_options, cf_handle.get(), file_names, base_level); if (!s.ok()) { std::shared_ptr cfh = @@ -643,6 +657,60 @@ static int rocksdb_force_flush_memtable_and_lzero_now( return num_errors == 0 ? HA_EXIT_SUCCESS : HA_EXIT_FAILURE; } +static void rocksdb_compact_lzero_now_stub( + THD *const thd MY_ATTRIBUTE((__unused__)), + struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), + void *const var_ptr MY_ATTRIBUTE((__unused__)), + const void *const save MY_ATTRIBUTE((__unused__))) {} + +static int rocksdb_compact_lzero_now( + THD *const thd MY_ATTRIBUTE((__unused__)), + struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), + void *const var_ptr MY_ATTRIBUTE((__unused__)), + struct st_mysql_value *const value) { + bool parsed_value = false; + + if (mysql_value_to_bool(value, &parsed_value) != 0) { + return 1; + } else if (!parsed_value) { + // Setting to OFF is a no-op and this supports mtr tests + return HA_EXIT_SUCCESS; + } + + return rocksdb_compact_lzero(); +} + +static void rocksdb_force_flush_memtable_and_lzero_now_stub( + THD *const thd MY_ATTRIBUTE((__unused__)), + struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), + void *const var_ptr MY_ATTRIBUTE((__unused__)), + const void *const save MY_ATTRIBUTE((__unused__))) {} + +static int rocksdb_force_flush_memtable_and_lzero_now( + THD *const thd MY_ATTRIBUTE((__unused__)), + struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), + void *const var_ptr MY_ATTRIBUTE((__unused__)), + struct st_mysql_value *const value) { + bool parsed_value = false; + + if (mysql_value_to_bool(value, &parsed_value) != 0) { + return 1; + } else if (!parsed_value) { + // Setting to OFF is a no-op and this supports mtr tests + return HA_EXIT_SUCCESS; + } + + // NO_LINT_DEBUG + LogPluginErrMsg(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, + "RocksDB: Manual memtable and L0 flush."); + rocksdb_flush_all_memtables(); + + // Try to avoid https://github.com/facebook/mysql-5.6/issues/1200 + my_sleep(1000000); + + return rocksdb_compact_lzero(); +} + static void rocksdb_cancel_manual_compactions_stub( THD *const thd MY_ATTRIBUTE((__unused__)), struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), @@ -824,8 +892,9 @@ static bool rocksdb_ignore_unknown_options = 1; static bool rocksdb_enable_2pc = 0; static char *rocksdb_strict_collation_exceptions; static bool rocksdb_collect_sst_properties = 1; -static bool rocksdb_force_flush_memtable_now_var = 1; +static bool rocksdb_force_flush_memtable_now_var = 0; static bool rocksdb_force_flush_memtable_and_lzero_now_var = 0; +static bool rocksdb_compact_lzero_now_var = 0; static bool rocksdb_cancel_manual_compactions_var = 0; static bool rocksdb_enable_ttl = 1; static bool rocksdb_enable_ttl_read_filtering = 1; @@ -2440,7 +2509,12 @@ static MYSQL_SYSVAR_BOOL( PLUGIN_VAR_RQCMDARG, "Forces memstore flush which may block all write requests so be careful", rocksdb_force_flush_memtable_now, rocksdb_force_flush_memtable_now_stub, - true); + false); + +static MYSQL_SYSVAR_BOOL(compact_lzero_now, rocksdb_compact_lzero_now_var, + PLUGIN_VAR_RQCMDARG, "Compacts all L0 files.", + rocksdb_compact_lzero_now, + rocksdb_compact_lzero_now_stub, false); static MYSQL_SYSVAR_BOOL( force_flush_memtable_and_lzero_now, @@ -2979,6 +3053,7 @@ static struct SYS_VAR *rocksdb_system_variables[] = { MYSQL_SYSVAR(collect_sst_properties), MYSQL_SYSVAR(force_flush_memtable_now), MYSQL_SYSVAR(force_flush_memtable_and_lzero_now), + MYSQL_SYSVAR(compact_lzero_now), MYSQL_SYSVAR(cancel_manual_compactions), MYSQL_SYSVAR(enable_ttl), MYSQL_SYSVAR(enable_ttl_read_filtering),