Upload files to "tests"
This commit is contained in:
411
tests/autoscaling_test.lua
Normal file
411
tests/autoscaling_test.lua
Normal file
@@ -0,0 +1,411 @@
|
||||
-- tests/autoscaling_test.lua
|
||||
-- Тестирование автоматического масштабирования кластера
|
||||
|
||||
local function setup_test_environment()
|
||||
print("=== Setup Test Environment ===")
|
||||
|
||||
-- Создаём тестовую базу данных
|
||||
local result = db.execute("CREATE DATABASE IF NOT EXISTS autoscale_test")
|
||||
assert(result, "Failed to create database")
|
||||
|
||||
result = db.execute("USE autoscale_test")
|
||||
assert(result, "Failed to use database")
|
||||
|
||||
-- Создаём тестовую коллекцию
|
||||
result = db.execute([[
|
||||
CREATE COLLECTION IF NOT EXISTS metrics_collection
|
||||
WITH SETTINGS {
|
||||
max_documents = 1000000,
|
||||
validate_schema = true
|
||||
}
|
||||
]])
|
||||
assert(result, "Failed to create collection")
|
||||
|
||||
-- Получаем конфигурацию автомасштабирования
|
||||
local config = db.execute("SHOW AUTOSCALING CONFIG")
|
||||
print("Current autoscaling config:")
|
||||
print(config)
|
||||
|
||||
return true
|
||||
end
|
||||
|
||||
local function generate_test_load(duration_seconds, target_qps)
|
||||
print(string.format("Generating %.2f QPS load for %d seconds", target_qps, duration_seconds))
|
||||
|
||||
local start_time = os.time()
|
||||
local requests = 0
|
||||
local errors = 0
|
||||
local latencies = {}
|
||||
|
||||
local interval = 1.0 / target_qps
|
||||
|
||||
while os.time() - start_time < duration_seconds do
|
||||
local req_start = os.clock()
|
||||
|
||||
-- Симуляция запроса на запись
|
||||
local success, err = pcall(function()
|
||||
local doc = {
|
||||
_id = string.format("doc_%d_%d", os.time(), math.random(1, 10000)),
|
||||
timestamp = os.time() * 1000,
|
||||
value = math.random(1, 1000),
|
||||
type = "test_metric"
|
||||
}
|
||||
return db.execute(string.format(
|
||||
"INSERT INTO metrics_collection VALUES (%s)",
|
||||
db.json_encode(doc)
|
||||
))
|
||||
end)
|
||||
|
||||
local req_end = os.clock()
|
||||
local latency = (req_end - req_start) * 1000 -- в миллисекундах
|
||||
|
||||
table.insert(latencies, latency)
|
||||
requests = requests + 1
|
||||
|
||||
if not success then
|
||||
errors = errors + 1
|
||||
print("Request failed: " .. tostring(err))
|
||||
end
|
||||
|
||||
-- Контроль скорости
|
||||
local elapsed = os.clock() - req_start
|
||||
if elapsed < interval then
|
||||
local sleep_time = interval - elapsed
|
||||
db.sleep(sleep_time)
|
||||
end
|
||||
end
|
||||
|
||||
-- Вычисляем статистику
|
||||
local total_latency = 0
|
||||
for _, lat in ipairs(latencies) do
|
||||
total_latency = total_latency + lat
|
||||
end
|
||||
local avg_latency = total_latency / #latencies
|
||||
|
||||
-- Сортируем для перцентилей
|
||||
table.sort(latencies)
|
||||
local p95_latency = latencies[math.floor(#latencies * 0.95)]
|
||||
local p99_latency = latencies[math.floor(#latencies * 0.99)]
|
||||
|
||||
print(string.format([[
|
||||
=== Load Test Results ===
|
||||
Duration: %d seconds
|
||||
Total Requests: %d
|
||||
Successful: %d
|
||||
Errors: %d
|
||||
Actual QPS: %.2f
|
||||
Avg Latency: %.2f ms
|
||||
P95 Latency: %.2f ms
|
||||
P99 Latency: %.2f ms
|
||||
]], duration_seconds, requests, requests - errors, errors,
|
||||
requests / duration_seconds, avg_latency, p95_latency, p99_latency))
|
||||
|
||||
return {
|
||||
requests = requests,
|
||||
errors = errors,
|
||||
avg_latency = avg_latency,
|
||||
p95_latency = p95_latency,
|
||||
p99_latency = p99_latency,
|
||||
actual_qps = requests / duration_seconds
|
||||
}
|
||||
end
|
||||
|
||||
local function monitor_cluster_size(interval_seconds, duration_seconds)
|
||||
print(string.format("Monitoring cluster size every %d seconds for %d seconds",
|
||||
interval_seconds, duration_seconds))
|
||||
|
||||
local history = {}
|
||||
local start_time = os.time()
|
||||
|
||||
while os.time() - start_time < duration_seconds do
|
||||
local status = db.execute("SHOW CLUSTER STATUS")
|
||||
local cluster_info = db.json_decode(status)
|
||||
|
||||
table.insert(history, {
|
||||
timestamp = os.time(),
|
||||
nodes = cluster_info.total_nodes,
|
||||
active_nodes = cluster_info.active_nodes,
|
||||
leader = cluster_info.leader_id,
|
||||
health = cluster_info.health
|
||||
})
|
||||
|
||||
print(string.format("Time %ds: Nodes=%d, Active=%d, Health=%s",
|
||||
os.time() - start_time,
|
||||
cluster_info.total_nodes,
|
||||
cluster_info.active_nodes,
|
||||
cluster_info.health))
|
||||
|
||||
db.sleep(interval_seconds)
|
||||
end
|
||||
|
||||
return history
|
||||
end
|
||||
|
||||
local function test_scale_up_on_high_load()
|
||||
print("\n=== Test 1: Scale Up on High Load ===")
|
||||
|
||||
-- Проверяем начальное состояние
|
||||
local initial_status = db.execute("SHOW CLUSTER STATUS")
|
||||
local initial_info = db.json_decode(initial_status)
|
||||
local initial_nodes = initial_info.total_nodes
|
||||
print(string.format("Initial cluster size: %d nodes", initial_nodes))
|
||||
|
||||
-- Запускаем мониторинг в фоне
|
||||
local monitor_thread = coroutine.create(function()
|
||||
return monitor_cluster_size(5, 180) -- мониторим 3 минуты
|
||||
end)
|
||||
|
||||
-- Генерируем высокую нагрузку
|
||||
local load_results = generate_test_load(60, 500) -- 60 секунд, 500 QPS
|
||||
|
||||
-- Ожидаем возможного масштабирования
|
||||
print("Waiting for potential scale up...")
|
||||
db.sleep(120) -- ждём 2 минуты
|
||||
|
||||
-- Проверяем результат
|
||||
local final_status = db.execute("SHOW CLUSTER STATUS")
|
||||
local final_info = db.json_decode(final_status)
|
||||
local final_nodes = final_info.total_nodes
|
||||
|
||||
print(string.format("Final cluster size: %d nodes", final_nodes))
|
||||
|
||||
if final_nodes > initial_nodes then
|
||||
print("✅ Scale Up successful: cluster expanded from " ..
|
||||
initial_nodes .. " to " .. final_nodes .. " nodes")
|
||||
return true
|
||||
else
|
||||
print("⚠️ No scale up detected (may be at max capacity or cooldown)")
|
||||
return false
|
||||
end
|
||||
end
|
||||
|
||||
local function test_scale_down_on_low_load()
|
||||
print("\n=== Test 2: Scale Down on Low Load ===")
|
||||
|
||||
local initial_status = db.execute("SHOW CLUSTER STATUS")
|
||||
local initial_info = db.json_decode(initial_status)
|
||||
local initial_nodes = initial_info.total_nodes
|
||||
|
||||
if initial_nodes <= 1 then
|
||||
print("⚠️ Cannot test scale down: only one node in cluster")
|
||||
return false
|
||||
end
|
||||
|
||||
print(string.format("Initial cluster size: %d nodes", initial_nodes))
|
||||
|
||||
-- Ждём снижения нагрузки
|
||||
print("Waiting for load to decrease and scale down...")
|
||||
db.sleep(300) -- ждём 5 минут
|
||||
|
||||
local final_status = db.execute("SHOW CLUSTER STATUS")
|
||||
local final_info = db.json_decode(final_status)
|
||||
local final_nodes = final_info.total_nodes
|
||||
|
||||
print(string.format("Final cluster size: %d nodes", final_nodes))
|
||||
|
||||
if final_nodes < initial_nodes then
|
||||
print("✅ Scale Down successful: cluster reduced from " ..
|
||||
initial_nodes .. " to " .. final_nodes .. " nodes")
|
||||
return true
|
||||
else
|
||||
print("⚠️ No scale down detected (may be at min capacity or cooldown)")
|
||||
return false
|
||||
end
|
||||
end
|
||||
|
||||
local function test_predictive_scaling()
|
||||
print("\n=== Test 3: Predictive Scaling ===")
|
||||
|
||||
-- Включаем прогнозирование
|
||||
db.execute("SET AUTOSCALING PREDICTIVE true")
|
||||
|
||||
-- Создаём растущую нагрузку
|
||||
print("Generating increasing load pattern...")
|
||||
|
||||
local function generate_ramp_load(initial_qps, final_qps, duration_seconds)
|
||||
local step_duration = duration_seconds / 10
|
||||
for step = 0, 9 do
|
||||
local target_qps = initial_qps + (final_qps - initial_qps) * step / 10
|
||||
print(string.format("Step %d/10: %.2f QPS", step + 1, target_qps))
|
||||
generate_test_load(step_duration, target_qps)
|
||||
end
|
||||
end
|
||||
|
||||
generate_ramp_load(50, 600, 120)
|
||||
|
||||
-- Проверяем, произошло ли масштабирование до достижения пика
|
||||
local scaling_history = db.execute("SHOW AUTOSCALING HISTORY")
|
||||
local history = db.json_decode(scaling_history)
|
||||
|
||||
local scaled_before_peak = false
|
||||
for _, event in ipairs(history) do
|
||||
if event.decision == "scale_up" then
|
||||
local load_at_scale = event.avg_load
|
||||
if load_at_scale < 0.85 then -- Масштабирование до достижения 85% нагрузки
|
||||
scaled_before_peak = true
|
||||
break
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if scaled_before_peak then
|
||||
print("✅ Predictive scaling successful: scaled before reaching peak load")
|
||||
return true
|
||||
else
|
||||
print("⚠️ Predictive scaling may not be working optimally")
|
||||
return false
|
||||
end
|
||||
end
|
||||
|
||||
local function test_autoscaling_metrics()
|
||||
print("\n=== Test 4: Autoscaling Metrics ===")
|
||||
|
||||
local metrics = db.execute("SHOW AUTOSCALING METRICS")
|
||||
local metrics_data = db.json_decode(metrics)
|
||||
|
||||
print("Current autoscaling metrics:")
|
||||
print(string.format(" Current nodes: %d", metrics_data.current_nodes or 0))
|
||||
print(string.format(" Min nodes: %d", metrics_data.min_nodes or 0))
|
||||
print(string.format(" Max nodes: %d", metrics_data.max_nodes or 0))
|
||||
print(string.format(" Last scale up: %s", metrics_data.last_scale_up or "never"))
|
||||
print(string.format(" Last scale down: %s", metrics_data.last_scale_down or "never"))
|
||||
print(string.format(" Evaluation count: %d", metrics_data.evaluation_count or 0))
|
||||
print(string.format(" Predictive enabled: %s", tostring(metrics_data.predictive_enabled)))
|
||||
|
||||
-- Проверяем наличие критических метрик
|
||||
local required_fields = {
|
||||
"current_nodes", "min_nodes", "max_nodes",
|
||||
"evaluation_count", "predictive_enabled"
|
||||
}
|
||||
|
||||
for _, field in ipairs(required_fields) do
|
||||
if metrics_data[field] == nil then
|
||||
print("❌ Missing metric: " .. field)
|
||||
return false
|
||||
end
|
||||
end
|
||||
|
||||
print("✅ All required metrics present")
|
||||
return true
|
||||
end
|
||||
|
||||
local function test_cooldown_behavior()
|
||||
print("\n=== Test 5: Cooldown Behavior ===")
|
||||
|
||||
-- Быстро генерируем две всплески нагрузки
|
||||
for i = 1, 2 do
|
||||
print(string.format("Load spike %d/2", i))
|
||||
generate_test_load(30, 800)
|
||||
db.sleep(10) -- Короткая пауза между всплесками
|
||||
end
|
||||
|
||||
local history = db.execute("SHOW AUTOSCALING HISTORY")
|
||||
local history_data = db.json_decode(history)
|
||||
|
||||
-- Проверяем, что масштабирование не происходило слишком часто
|
||||
local scale_ups = {}
|
||||
for _, event in ipairs(history_data) do
|
||||
if event.decision == "scale_up" then
|
||||
table.insert(scale_ups, event.timestamp)
|
||||
end
|
||||
end
|
||||
|
||||
local cooldown_ok = true
|
||||
for i = 2, #scale_ups do
|
||||
local time_diff = scale_ups[i] - scale_ups[i-1]
|
||||
if time_diff < 300 then -- 5 минут cooldown
|
||||
cooldown_ok = false
|
||||
print(string.format("⚠️ Scale ups too close: %d seconds apart", time_diff))
|
||||
end
|
||||
end
|
||||
|
||||
if cooldown_ok then
|
||||
print("✅ Cooldown mechanism working properly")
|
||||
else
|
||||
print("❌ Cooldown mechanism may be misconfigured")
|
||||
end
|
||||
|
||||
return cooldown_ok
|
||||
end
|
||||
|
||||
local function test_node_selection_for_scale_down()
|
||||
print("\n=== Test 6: Node Selection for Scale Down ===")
|
||||
|
||||
-- Получаем нагрузку на узлах
|
||||
local nodes_load = db.execute("SHOW NODES LOAD")
|
||||
local load_data = db.json_decode(nodes_load)
|
||||
|
||||
print("Node load distribution:")
|
||||
local loads = {}
|
||||
for _, node in ipairs(load_data) do
|
||||
table.insert(loads, node.load)
|
||||
print(string.format(" %s: %.2f%%", node.id, node.load * 100))
|
||||
end
|
||||
|
||||
-- Сортируем узлы по нагрузке
|
||||
table.sort(load_data, function(a, b) return a.load < b.load end)
|
||||
|
||||
-- Наименее загруженный узел должен быть кандидатом на удаление
|
||||
local lowest_load_node = load_data[1]
|
||||
print(string.format("Lowest load node: %s (%.2f%%)",
|
||||
lowest_load_node.id, lowest_load_node.load * 100))
|
||||
|
||||
-- Запрашиваем кандидата на удаление у системы
|
||||
local scale_down_candidate = db.execute("SHOW SCALE_DOWN_CANDIDATE")
|
||||
local candidate = db.json_decode(scale_down_candidate)
|
||||
|
||||
if candidate.id == lowest_load_node.id then
|
||||
print("✅ Correct node selected for scale down")
|
||||
return true
|
||||
else
|
||||
print(string.format("❌ Wrong node selected. Expected: %s, Got: %s",
|
||||
lowest_load_node.id, candidate.id))
|
||||
return false
|
||||
end
|
||||
end
|
||||
|
||||
-- Функция очистки
|
||||
local function cleanup()
|
||||
print("\n=== Cleanup ===")
|
||||
db.execute("DROP DATABASE IF EXISTS autoscale_test")
|
||||
print("Test database removed")
|
||||
end
|
||||
|
||||
-- Запуск всех тестов
|
||||
local function run_all_autoscaling_tests()
|
||||
print("╔══════════════════════════════════════════════════════════════╗")
|
||||
print("║ AUTOSCALING TESTS SUITE ║")
|
||||
print("╚══════════════════════════════════════════════════════════════╝")
|
||||
|
||||
setup_test_environment()
|
||||
|
||||
local results = {
|
||||
test_scale_up_on_high_load(),
|
||||
test_scale_down_on_low_load(),
|
||||
test_predictive_scaling(),
|
||||
test_autoscaling_metrics(),
|
||||
test_cooldown_behavior(),
|
||||
test_node_selection_for_scale_down()
|
||||
}
|
||||
|
||||
-- Подведение итогов
|
||||
print("\n╔══════════════════════════════════════════════════════════════╗")
|
||||
print("║ TEST RESULTS ║")
|
||||
print("╚══════════════════════════════════════════════════════════════╝")
|
||||
|
||||
local passed = 0
|
||||
for i, result in ipairs(results) do
|
||||
local status = result and "✅ PASSED" or "⚠️ SKIPPED/FAILED"
|
||||
print(string.format("Test %d: %s", i, status))
|
||||
if result then passed = passed + 1 end
|
||||
end
|
||||
|
||||
print(string.format("\nTotal: %d/%d tests passed", passed, #results))
|
||||
|
||||
cleanup()
|
||||
|
||||
return passed == #results
|
||||
end
|
||||
|
||||
-- Запуск
|
||||
return run_all_autoscaling_tests()
|
||||
Reference in New Issue
Block a user