From a9a136b23c1718ec8f2bef1697a1142b663f2f49 Mon Sep 17 00:00:00 2001 From: Mark Callaghan Date: Thu, 11 May 2023 13:13:38 -0700 Subject: [PATCH] Fix issues 1200 and 1295, add rocksdb_compact_lzero_now global variable (#1309) Upstream commit ID: https://github.com/facebook/mysql-5.6/commit/3ba4f39d4ef9a7389a92938997ae02de953de718 PS-8951: Merge percona-202305 (https://jira.percona.com/browse/PS-8951) Summary: This fixes issue 1295 and has a workaround for issue 1200. https://github.com/facebook/mysql-5.6/issues/1200 https://github.com/facebook/mysql-5.6/issues/1295 The fix for issue 1295 is to get the value of the base level from RocksDB rather than assuming that L0 compacts into L1 because L1 might not be the base level when dynamic leveled compaction is used -- in that case the base level can be Ln where n > 1 because L1 isn't needed yet. There are two workarounds for issue 1200. The hacky one is to sleep for 1 second between requesting memtable flush and L0 -> base_level compaction. The alternative is to add a new global variable, rocksdb_compact_lzero_now, that when set will request L0 -> base_level compaction. This allows a client to first set rocksdb_force_flush_memtable_now, wait for that to finish, then set rocksdb_compact_lzero_now. Other changes: * rocksdb_force_flush_memtable_now default value changed to OFF to match what is done for rocksdb_force_flush_memtable_and_lzero_now and rocksdb_compact_lzero_now * confirm that the value to which these variables are set can be parsed, that wasn't done for all of them * don't raise an error when these are set to OFF, that will be a no-op Note: these variables are triggers, an action is taken when this is done: set global var = ON | true | 1 But these variables always show the value 0 (OFF, false). Their value doesn't change. Pull Request resolved: https://github.com/facebook/mysql-5.6/pull/1309 Differential Revision: D45789784 fbshipit-source-id: b5ab4d03a984cd0615ad2a828af80b7389b2508f --- mysql-test/suite/rocksdb/r/rocksdb.result | 3 +- .../r/rocksdb_compact_lzero_now_basic.result | 83 ++++++++++++++ ..._flush_memtable_and_lzero_now_basic.result | 39 ++++++- ...ksdb_force_flush_memtable_now_basic.result | 54 +++++---- .../t/rocksdb_compact_lzero_now_basic.test | 20 ++++ ...ce_flush_memtable_and_lzero_now_basic.test | 5 +- ...ocksdb_force_flush_memtable_now_basic.test | 6 +- storage/rocksdb/ha_rocksdb.cc | 105 ++++++++++++++++-- 8 files changed, 274 insertions(+), 41 deletions(-) create mode 100644 mysql-test/suite/rocksdb_sys_vars/r/rocksdb_compact_lzero_now_basic.result create mode 100644 mysql-test/suite/rocksdb_sys_vars/t/rocksdb_compact_lzero_now_basic.test diff --git a/mysql-test/suite/rocksdb/r/rocksdb.result b/mysql-test/suite/rocksdb/r/rocksdb.result index 7647223d7758..b0be17854463 100644 --- a/mysql-test/suite/rocksdb/r/rocksdb.result +++ b/mysql-test/suite/rocksdb/r/rocksdb.result @@ -926,6 +926,7 @@ rocksdb_column_default_value_as_expression ON rocksdb_commit_in_the_middle OFF rocksdb_commit_time_batch_for_recovery OFF rocksdb_compact_cf +rocksdb_compact_lzero_now OFF rocksdb_compaction_readahead_size 0 rocksdb_compaction_sequential_deletes 0 rocksdb_compaction_sequential_deletes_count_sd OFF @@ -974,7 +975,7 @@ rocksdb_flush_log_at_trx_commit 1 rocksdb_force_compute_memtable_stats ON rocksdb_force_compute_memtable_stats_cachetime 0 rocksdb_force_flush_memtable_and_lzero_now OFF -rocksdb_force_flush_memtable_now ON +rocksdb_force_flush_memtable_now OFF rocksdb_force_index_records_in_range 0 rocksdb_ignore_unknown_options ON rocksdb_index_type kBinarySearch diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_compact_lzero_now_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_compact_lzero_now_basic.result new file mode 100644 index 000000000000..28302a0a4671 --- /dev/null +++ b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_compact_lzero_now_basic.result @@ -0,0 +1,83 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +SET @start_global_value = @@global.ROCKSDB_COMPACT_LZERO_NOW; +SELECT @start_global_value; +@start_global_value +0 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to 1" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = 1; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to on" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = on; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to true" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = true; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to 0" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = 0; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to off" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = off; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to false" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = false; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@session.ROCKSDB_COMPACT_LZERO_NOW to 444. It should fail because it is not session." +SET @@session.ROCKSDB_COMPACT_LZERO_NOW = 444; +ERROR HY000: Variable 'rocksdb_compact_lzero_now' is a GLOBAL variable and should be set with SET GLOBAL +'# Testing with invalid values in global scope #' +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = @start_global_value; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result index 68cfeb07fc71..a3e506753d90 100644 --- a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result +++ b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result @@ -1,7 +1,10 @@ CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; INSERT INTO valid_values VALUES(1); -INSERT INTO valid_values VALUES(0); INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; SET @start_global_value = @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; SELECT @start_global_value; @@ -18,6 +21,26 @@ SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW 0 +"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to on" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = on; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to true" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = true; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to 0" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = 0; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; @@ -28,8 +51,18 @@ SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW 0 -"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to on" -SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = on; +"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to off" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = off; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to false" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = false; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW 0 diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_now_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_now_basic.result index 53665a7cedaa..59f6c832da39 100644 --- a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_now_basic.result +++ b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_now_basic.result @@ -2,70 +2,82 @@ CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; INSERT INTO valid_values VALUES(1); INSERT INTO valid_values VALUES('on'); INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; -INSERT INTO invalid_values VALUES(0); -INSERT INTO invalid_values VALUES('off'); -INSERT INTO invalid_values VALUES('false'); SET @start_global_value = @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; SELECT @start_global_value; @start_global_value -1 +0 '# Setting to valid values in global scope#' "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to 1" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = 1; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Setting the global scope variable back to default" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to on" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = on; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Setting the global scope variable back to default" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to true" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = true; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Setting the global scope variable back to default" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 -"Trying to set variable @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to 444. It should fail because it is not session." -SET @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = 444; -ERROR HY000: Variable 'rocksdb_force_flush_memtable_now' is a GLOBAL variable and should be set with SET GLOBAL -'# Testing with invalid values in global scope #' +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to 0" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = 0; -Got one of the listed errors SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to off" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = off; -Got one of the listed errors SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to false" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = false; -Got one of the listed errors SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW +0 +"Trying to set variable @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to 444. It should fail because it is not session." +SET @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = 444; +ERROR HY000: Variable 'rocksdb_force_flush_memtable_now' is a GLOBAL variable and should be set with SET GLOBAL +'# Testing with invalid values in global scope #' SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = @start_global_value; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 DROP TABLE valid_values; DROP TABLE invalid_values; diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_compact_lzero_now_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_compact_lzero_now_basic.test new file mode 100644 index 000000000000..eb9431a7d09d --- /dev/null +++ b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_compact_lzero_now_basic.test @@ -0,0 +1,20 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; + +--let $sys_var=ROCKSDB_COMPACT_LZERO_NOW +--let $read_only=0 +--let $session=0 +--let $sticky=1 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test index c0f5cd21b36b..c9273f8cfc22 100644 --- a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test +++ b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test @@ -3,8 +3,11 @@ CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; INSERT INTO valid_values VALUES(1); -INSERT INTO valid_values VALUES(0); INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_now_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_now_basic.test index d02e1ce7c695..7077772bbccf 100644 --- a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_now_basic.test +++ b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_now_basic.test @@ -5,11 +5,11 @@ CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; INSERT INTO valid_values VALUES(1); INSERT INTO valid_values VALUES('on'); INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; -INSERT INTO invalid_values VALUES(0); -INSERT INTO invalid_values VALUES('off'); -INSERT INTO invalid_values VALUES('false'); --let $sys_var=ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW --let $read_only=0 diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index dd713e8a5791..2f5de90f0b4f 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -403,6 +403,8 @@ static bool parse_fault_injection_params(bool *retryable, uint32_t *failure_ratio, std::vector *types); +static int delete_range(const std::unordered_set &indices); + static void rocksdb_force_flush_memtable_now_stub( THD *const thd MY_ATTRIBUTE((__unused__)), struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), @@ -413,24 +415,21 @@ static int rocksdb_force_flush_memtable_now( THD *const thd, struct SYS_VAR *const var, void *const var_ptr, struct st_mysql_value *const value) { bool parsed_value = false; - if (mysql_value_to_bool(value, &parsed_value) != 0 || !parsed_value) { + if (mysql_value_to_bool(value, &parsed_value) != 0) { return 1; + } else if (!parsed_value) { + // Setting to OFF is a no-op and this supports mtr tests + return HA_EXIT_SUCCESS; } LogPluginErrMsg(INFORMATION_LEVEL, 0, "Manual memtable flush."); rocksdb_flush_all_memtables(); return HA_EXIT_SUCCESS; } -static int delete_range(const std::unordered_set &indices); -static void rocksdb_force_flush_memtable_and_lzero_now_stub( - THD *const thd, struct SYS_VAR *const var, void *const var_ptr, - const void *const save) {} -static int rocksdb_force_flush_memtable_and_lzero_now( - THD *const thd, struct SYS_VAR *const var, void *const var_ptr, - struct st_mysql_value *const value) { - LogPluginErrMsg(INFORMATION_LEVEL, 0, "Manual memtable and L0 flush."); - rocksdb_flush_all_memtables(); +static int rocksdb_compact_lzero() { + // NO_LINT_DEBUG + LogPluginErrMsg(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, "RocksDB: compact L0."); const Rdb_cf_manager &cf_manager = rdb_get_cf_manager(); rocksdb::CompactionOptions c_options = rocksdb::CompactionOptions(); @@ -446,6 +445,25 @@ static int rocksdb_force_flush_memtable_and_lzero_now( cf_handle->GetDescriptor(&cf_descr); c_options.output_file_size_limit = cf_descr.options.target_file_size_base; + // Lets RocksDB use the configured compression for this level + c_options.compression = rocksdb::kDisableCompressionOption; + + uint64_t base_level; + if (!rdb->GetIntProperty(cf_handle.get(), + rocksdb::DB::Properties::kBaseLevel, + &base_level)) { + LogPluginErrMsg(ERROR_LEVEL, ER_LOG_PRINTF_MSG, + "MyRocks: compact L0 cannot get base level"); + break; + } + + if (base_level == 0) { + LogPluginErrMsg( + ERROR_LEVEL, ER_LOG_PRINTF_MSG, + "MyRocks: compact L0 cannot flush to base level when 0"); + break; + } + assert(metadata.levels[0].level == 0); std::vector file_names; for (const auto &file : metadata.levels[0].files) { @@ -453,11 +471,13 @@ static int rocksdb_force_flush_memtable_and_lzero_now( } if (file_names.empty()) { + LogPluginErrMsg(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, + "MyRocks: no files in L0"); break; } rocksdb::Status s; - s = rdb->CompactFiles(c_options, cf_handle.get(), file_names, 1); + s = rdb->CompactFiles(c_options, cf_handle.get(), file_names, base_level); if (!s.ok()) { std::shared_ptr cfh = @@ -494,6 +514,60 @@ static int rocksdb_force_flush_memtable_and_lzero_now( return num_errors == 0 ? HA_EXIT_SUCCESS : HA_EXIT_FAILURE; } +static void rocksdb_compact_lzero_now_stub( + THD *const thd MY_ATTRIBUTE((__unused__)), + struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), + void *const var_ptr MY_ATTRIBUTE((__unused__)), + const void *const save MY_ATTRIBUTE((__unused__))) {} + +static int rocksdb_compact_lzero_now( + THD *const thd MY_ATTRIBUTE((__unused__)), + struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), + void *const var_ptr MY_ATTRIBUTE((__unused__)), + struct st_mysql_value *const value) { + bool parsed_value = false; + + if (mysql_value_to_bool(value, &parsed_value) != 0) { + return 1; + } else if (!parsed_value) { + // Setting to OFF is a no-op and this supports mtr tests + return HA_EXIT_SUCCESS; + } + + return rocksdb_compact_lzero(); +} + +static void rocksdb_force_flush_memtable_and_lzero_now_stub( + THD *const thd MY_ATTRIBUTE((__unused__)), + struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), + void *const var_ptr MY_ATTRIBUTE((__unused__)), + const void *const save MY_ATTRIBUTE((__unused__))) {} + +static int rocksdb_force_flush_memtable_and_lzero_now( + THD *const thd MY_ATTRIBUTE((__unused__)), + struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), + void *const var_ptr MY_ATTRIBUTE((__unused__)), + struct st_mysql_value *const value) { + bool parsed_value = false; + + if (mysql_value_to_bool(value, &parsed_value) != 0) { + return 1; + } else if (!parsed_value) { + // Setting to OFF is a no-op and this supports mtr tests + return HA_EXIT_SUCCESS; + } + + // NO_LINT_DEBUG + LogPluginErrMsg(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, + "RocksDB: Manual memtable and L0 flush."); + rocksdb_flush_all_memtables(); + + // Try to avoid https://github.com/facebook/mysql-5.6/issues/1200 + my_sleep(1000000); + + return rocksdb_compact_lzero(); +} + static void rocksdb_cancel_manual_compactions_stub( THD *const thd MY_ATTRIBUTE((__unused__)), struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), @@ -697,6 +771,7 @@ static bool rocksdb_ignore_unknown_options = true; static bool rocksdb_collect_sst_properties = true; static bool rocksdb_force_flush_memtable_now_var = true; static bool rocksdb_force_flush_memtable_and_lzero_now_var = false; +static bool rocksdb_compact_lzero_now_var = false; static bool rocksdb_cancel_manual_compactions_var = false; static bool rocksdb_enable_ttl = true; static bool rocksdb_enable_ttl_read_filtering = true; @@ -2182,7 +2257,12 @@ static MYSQL_SYSVAR_BOOL( PLUGIN_VAR_RQCMDARG, "Forces memstore flush which may block all write requests so be careful", rocksdb_force_flush_memtable_now, rocksdb_force_flush_memtable_now_stub, - true); + false); + +static MYSQL_SYSVAR_BOOL(compact_lzero_now, rocksdb_compact_lzero_now_var, + PLUGIN_VAR_RQCMDARG, "Compacts all L0 files.", + rocksdb_compact_lzero_now, + rocksdb_compact_lzero_now_stub, false); static MYSQL_SYSVAR_BOOL( force_flush_memtable_and_lzero_now, @@ -2630,6 +2710,7 @@ static struct SYS_VAR *rocksdb_system_variables[] = { MYSQL_SYSVAR(collect_sst_properties), MYSQL_SYSVAR(force_flush_memtable_now), MYSQL_SYSVAR(force_flush_memtable_and_lzero_now), + MYSQL_SYSVAR(compact_lzero_now), MYSQL_SYSVAR(cancel_manual_compactions), MYSQL_SYSVAR(enable_ttl), MYSQL_SYSVAR(enable_ttl_read_filtering),