From 61f78b85f9e7058a452568d2af36d4cb5c1199fd Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 13 Feb 2023 19:40:50 +0000 Subject: [PATCH] Include keyspace range checks on flow tests --- tests/include.py | 26 ++++++- tests/tests_oss_simple_flow.py | 124 +++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 3 deletions(-) diff --git a/tests/include.py b/tests/include.py index 80441fa1..8a4b96da 100644 --- a/tests/include.py +++ b/tests/include.py @@ -59,13 +59,15 @@ def debugPrintMemtierOnError(config, env): env.debugPrint(line.rstrip(), True) -def get_expected_request_count(config): +def get_expected_request_count(config, key_minimum=0, key_maximum=1000000): result = -1 if 'memtier_benchmark' in config: mt = config['memtier_benchmark'] if 'threads' in mt and 'clients' in mt and 'requests' in mt: - result = config['memtier_benchmark']['threads'] * config['memtier_benchmark']['clients'] * \ - config['memtier_benchmark']['requests'] + if mt['requests'] != 'allkeys': + result = mt['threads'] * mt['clients'] * mt['requests'] + else: + result = key_maximum - key_minimum + 1 return result @@ -112,3 +114,21 @@ def ensure_clean_benchmark_folder(dirname): os.removedirs(dirname) os.makedirs(dirname) + +def assert_keyspace_range(env, key_max, key_min, master_nodes_connections): + expected_keyspace_range = key_max - key_min + 1 + overall_keyspace_range = agg_keyspace_range(master_nodes_connections) + # assert we have the expected keyspace range + env.assertEqual(expected_keyspace_range, overall_keyspace_range) + + +def agg_keyspace_range(master_nodes_connections): + overall_keyspace_range = 0 + for master_connection in master_nodes_connections: + shard_reply = master_connection.execute_command("INFO", "KEYSPACE") + shard_count = 0 + if 'db0' in shard_reply: + if 'keys' in shard_reply['db0']: + shard_count = int(shard_reply['db0']['keys']) + overall_keyspace_range = overall_keyspace_range + shard_count + return overall_keyspace_range diff --git a/tests/tests_oss_simple_flow.py b/tests/tests_oss_simple_flow.py index 5f772e28..c229e7df 100644 --- a/tests/tests_oss_simple_flow.py +++ b/tests/tests_oss_simple_flow.py @@ -5,6 +5,130 @@ from mbdirector.runner import RunConfig +def test_preload_and_set_get(env): + key_max = 500000 + key_min = 1 + benchmark_specs = {"name": env.testName, "args": ['--pipeline=10','--ratio=1:0','--key-pattern=P:P','--key-minimum={}'.format(key_min),'--key-maximum={}'.format(key_max)]} + addTLSArgs(benchmark_specs, env) + config = get_default_memtier_config(threads=2, clients=10, requests='allkeys') + master_nodes_list = env.getMasterNodesList() + overall_expected_request_count = get_expected_request_count(config,key_min, key_max) + + add_required_env_arguments(benchmark_specs, config, env, master_nodes_list) + + # Create a temporary directory + test_dir = tempfile.mkdtemp() + + config = RunConfig(test_dir, env.testName, config, {}) + ensure_clean_benchmark_folder(config.results_dir) + + benchmark = Benchmark.from_json(config, benchmark_specs) + + # benchmark.run() returns True if the return code of memtier_benchmark was 0 + memtier_ok = benchmark.run() + debugPrintMemtierOnError(config, env) + + master_nodes_connections = env.getOSSMasterNodesConnectionList() + merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}} + assert_keyspace_range(env, key_max, key_min, master_nodes_connections) + + overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats) + assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, + overall_request_count) + json_filename = '{0}/mb.json'.format(config.results_dir) + + for master_connection in master_nodes_connections: + master_connection.execute_command("CONFIG", "RESETSTAT") + + benchmark_specs = {"name": env.testName, "args": ['--pipeline=10','--ratio=1:1','--key-pattern=R:R','--key-minimum={}'.format(key_min),'--key-maximum={}'.format(key_max)]} + addTLSArgs(benchmark_specs, env) + config = get_default_memtier_config(threads=2, clients=10, requests=200000) + master_nodes_list = env.getMasterNodesList() + overall_expected_request_count = get_expected_request_count(config,key_min, key_max) + + add_required_env_arguments(benchmark_specs, config, env, master_nodes_list) + + # Create a temporary directory + test_dir = tempfile.mkdtemp() + + config = RunConfig(test_dir, env.testName, config, {}) + ensure_clean_benchmark_folder(config.results_dir) + + benchmark = Benchmark.from_json(config, benchmark_specs) + + # benchmark.run() returns True if the return code of memtier_benchmark was 0 + memtier_ok = benchmark.run() + debugPrintMemtierOnError(config, env) + + merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}} + assert_keyspace_range(env, key_max, key_min, master_nodes_connections) + + overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats) + assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, + overall_request_count) + + +def test_default_set(env): + key_max = 500000 + key_min = 1 + benchmark_specs = {"name": env.testName, "args": ['--pipeline=10','--ratio=1:0','--key-pattern=P:P','--key-minimum={}'.format(key_min),'--key-maximum={}'.format(key_max)]} + addTLSArgs(benchmark_specs, env) + config = get_default_memtier_config(threads=2, clients=10, requests='allkeys') + master_nodes_list = env.getMasterNodesList() + overall_expected_request_count = get_expected_request_count(config,key_min, key_max) + + add_required_env_arguments(benchmark_specs, config, env, master_nodes_list) + + # Create a temporary directory + test_dir = tempfile.mkdtemp() + + config = RunConfig(test_dir, env.testName, config, {}) + ensure_clean_benchmark_folder(config.results_dir) + + benchmark = Benchmark.from_json(config, benchmark_specs) + + # benchmark.run() returns True if the return code of memtier_benchmark was 0 + memtier_ok = benchmark.run() + master_nodes_connections = env.getOSSMasterNodesConnectionList() + + merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}} + assert_keyspace_range(env, key_max, key_min, master_nodes_connections) + + overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats) + assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, + overall_request_count) + + # ensure if we run again on a different key pattern the dataset doesn't grow + for master_connection in master_nodes_connections: + master_connection.execute_command("CONFIG", "RESETSTAT") + + benchmark_specs = {"name": env.testName, "args": ['--pipeline=10','--ratio=1:0','--key-pattern=R:R','--key-minimum={}'.format(key_min),'--key-maximum={}'.format(key_max)]} + addTLSArgs(benchmark_specs, env) + config = get_default_memtier_config(threads=2, clients=10, requests=200000) + master_nodes_list = env.getMasterNodesList() + overall_expected_request_count = get_expected_request_count(config,key_min, key_max) + + add_required_env_arguments(benchmark_specs, config, env, master_nodes_list) + + # Create a temporary directory + test_dir = tempfile.mkdtemp() + + config = RunConfig(test_dir, env.testName, config, {}) + ensure_clean_benchmark_folder(config.results_dir) + + benchmark = Benchmark.from_json(config, benchmark_specs) + + # benchmark.run() returns True if the return code of memtier_benchmark was 0 + memtier_ok = benchmark.run() + + master_nodes_connections = env.getOSSMasterNodesConnectionList() + merged_command_stats = {'cmdstat_set': {'calls': 0}} + assert_keyspace_range(env, key_max, key_min, master_nodes_connections) + + overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats) + assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, + overall_request_count) + def test_default_set_get(env): benchmark_specs = {"name": env.testName, "args": []} addTLSArgs(benchmark_specs, env)