futriix/tests/autoscaling_test.lua

-- tests/autoscaling_test.lua
-- Тестирование автоматического масштабирования кластера

local function setup_test_environment()
    print("=== Setup Test Environment ===")
    
    -- Создаём тестовую базу данных
    local result = db.execute("CREATE DATABASE IF NOT EXISTS autoscale_test")
    assert(result, "Failed to create database")
    
    result = db.execute("USE autoscale_test")
    assert(result, "Failed to use database")
    
    -- Создаём тестовую коллекцию
    result = db.execute([[
        CREATE COLLECTION IF NOT EXISTS metrics_collection
        WITH SETTINGS {
            max_documents = 1000000,
            validate_schema = true
        }
    ]])
    assert(result, "Failed to create collection")
    
    -- Получаем конфигурацию автомасштабирования
    local config = db.execute("SHOW AUTOSCALING CONFIG")
    print("Current autoscaling config:")
    print(config)
    
    return true
end

local function generate_test_load(duration_seconds, target_qps)
    print(string.format("Generating %.2f QPS load for %d seconds", target_qps, duration_seconds))
    
    local start_time = os.time()
    local requests = 0
    local errors = 0
    local latencies = {}
    
    local interval = 1.0 / target_qps
    
    while os.time() - start_time < duration_seconds do
        local req_start = os.clock()
        
        -- Симуляция запроса на запись
        local success, err = pcall(function()
            local doc = {
                _id = string.format("doc_%d_%d", os.time(), math.random(1, 10000)),
                timestamp = os.time() * 1000,
                value = math.random(1, 1000),
                type = "test_metric"
            }
            return db.execute(string.format(
                "INSERT INTO metrics_collection VALUES (%s)",
                db.json_encode(doc)
            ))
        end)
        
        local req_end = os.clock()
        local latency = (req_end - req_start) * 1000 -- в миллисекундах
        
        table.insert(latencies, latency)
        requests = requests + 1
        
        if not success then
            errors = errors + 1
            print("Request failed: " .. tostring(err))
        end
        
        -- Контроль скорости
        local elapsed = os.clock() - req_start
        if elapsed < interval then
            local sleep_time = interval - elapsed
            db.sleep(sleep_time)
        end
    end
    
    -- Вычисляем статистику
    local total_latency = 0
    for _, lat in ipairs(latencies) do
        total_latency = total_latency + lat
    end
    local avg_latency = total_latency / #latencies
    
    -- Сортируем для перцентилей
    table.sort(latencies)
    local p95_latency = latencies[math.floor(#latencies * 0.95)]
    local p99_latency = latencies[math.floor(#latencies * 0.99)]
    
    print(string.format([[
    === Load Test Results ===
    Duration: %d seconds
    Total Requests: %d
    Successful: %d
    Errors: %d
    Actual QPS: %.2f
    Avg Latency: %.2f ms
    P95 Latency: %.2f ms
    P99 Latency: %.2f ms
    ]], duration_seconds, requests, requests - errors, errors,
        requests / duration_seconds, avg_latency, p95_latency, p99_latency))
    
    return {
        requests = requests,
        errors = errors,
        avg_latency = avg_latency,
        p95_latency = p95_latency,
        p99_latency = p99_latency,
        actual_qps = requests / duration_seconds
    }
end

local function monitor_cluster_size(interval_seconds, duration_seconds)
    print(string.format("Monitoring cluster size every %d seconds for %d seconds", 
        interval_seconds, duration_seconds))
    
    local history = {}
    local start_time = os.time()
    
    while os.time() - start_time < duration_seconds do
        local status = db.execute("SHOW CLUSTER STATUS")
        local cluster_info = db.json_decode(status)
        
        table.insert(history, {
            timestamp = os.time(),
            nodes = cluster_info.total_nodes,
            active_nodes = cluster_info.active_nodes,
            leader = cluster_info.leader_id,
            health = cluster_info.health
        })
        
        print(string.format("Time %ds: Nodes=%d, Active=%d, Health=%s",
            os.time() - start_time,
            cluster_info.total_nodes,
            cluster_info.active_nodes,
            cluster_info.health))
        
        db.sleep(interval_seconds)
    end
    
    return history
end

local function test_scale_up_on_high_load()
    print("\n=== Test 1: Scale Up on High Load ===")
    
    -- Проверяем начальное состояние
    local initial_status = db.execute("SHOW CLUSTER STATUS")
    local initial_info = db.json_decode(initial_status)
    local initial_nodes = initial_info.total_nodes
    print(string.format("Initial cluster size: %d nodes", initial_nodes))
    
    -- Запускаем мониторинг в фоне
    local monitor_thread = coroutine.create(function()
        return monitor_cluster_size(5, 180) -- мониторим 3 минуты
    end)
    
    -- Генерируем высокую нагрузку
    local load_results = generate_test_load(60, 500) -- 60 секунд, 500 QPS
    
    -- Ожидаем возможного масштабирования
    print("Waiting for potential scale up...")
    db.sleep(120) -- ждём 2 минуты
    
    -- Проверяем результат
    local final_status = db.execute("SHOW CLUSTER STATUS")
    local final_info = db.json_decode(final_status)
    local final_nodes = final_info.total_nodes
    
    print(string.format("Final cluster size: %d nodes", final_nodes))
    
    if final_nodes > initial_nodes then
        print("✅ Scale Up successful: cluster expanded from " .. 
              initial_nodes .. " to " .. final_nodes .. " nodes")
        return true
    else
        print("⚠️ No scale up detected (may be at max capacity or cooldown)")
        return false
    end
end

local function test_scale_down_on_low_load()
    print("\n=== Test 2: Scale Down on Low Load ===")
    
    local initial_status = db.execute("SHOW CLUSTER STATUS")
    local initial_info = db.json_decode(initial_status)
    local initial_nodes = initial_info.total_nodes
    
    if initial_nodes <= 1 then
        print("⚠️ Cannot test scale down: only one node in cluster")
        return false
    end
    
    print(string.format("Initial cluster size: %d nodes", initial_nodes))
    
    -- Ждём снижения нагрузки
    print("Waiting for load to decrease and scale down...")
    db.sleep(300) -- ждём 5 минут
    
    local final_status = db.execute("SHOW CLUSTER STATUS")
    local final_info = db.json_decode(final_status)
    local final_nodes = final_info.total_nodes
    
    print(string.format("Final cluster size: %d nodes", final_nodes))
    
    if final_nodes < initial_nodes then
        print("✅ Scale Down successful: cluster reduced from " .. 
              initial_nodes .. " to " .. final_nodes .. " nodes")
        return true
    else
        print("⚠️ No scale down detected (may be at min capacity or cooldown)")
        return false
    end
end

local function test_predictive_scaling()
    print("\n=== Test 3: Predictive Scaling ===")
    
    -- Включаем прогнозирование
    db.execute("SET AUTOSCALING PREDICTIVE true")
    
    -- Создаём растущую нагрузку
    print("Generating increasing load pattern...")
    
    local function generate_ramp_load(initial_qps, final_qps, duration_seconds)
        local step_duration = duration_seconds / 10
        for step = 0, 9 do
            local target_qps = initial_qps + (final_qps - initial_qps) * step / 10
            print(string.format("Step %d/10: %.2f QPS", step + 1, target_qps))
            generate_test_load(step_duration, target_qps)
        end
    end
    
    generate_ramp_load(50, 600, 120)
    
    -- Проверяем, произошло ли масштабирование до достижения пика
    local scaling_history = db.execute("SHOW AUTOSCALING HISTORY")
    local history = db.json_decode(scaling_history)
    
    local scaled_before_peak = false
    for _, event in ipairs(history) do
        if event.decision == "scale_up" then
            local load_at_scale = event.avg_load
            if load_at_scale < 0.85 then -- Масштабирование до достижения 85% нагрузки
                scaled_before_peak = true
                break
            end
        end
    end
    
    if scaled_before_peak then
        print("✅ Predictive scaling successful: scaled before reaching peak load")
        return true
    else
        print("⚠️ Predictive scaling may not be working optimally")
        return false
    end
end

local function test_autoscaling_metrics()
    print("\n=== Test 4: Autoscaling Metrics ===")
    
    local metrics = db.execute("SHOW AUTOSCALING METRICS")
    local metrics_data = db.json_decode(metrics)
    
    print("Current autoscaling metrics:")
    print(string.format("  Current nodes: %d", metrics_data.current_nodes or 0))
    print(string.format("  Min nodes: %d", metrics_data.min_nodes or 0))
    print(string.format("  Max nodes: %d", metrics_data.max_nodes or 0))
    print(string.format("  Last scale up: %s", metrics_data.last_scale_up or "never"))
    print(string.format("  Last scale down: %s", metrics_data.last_scale_down or "never"))
    print(string.format("  Evaluation count: %d", metrics_data.evaluation_count or 0))
    print(string.format("  Predictive enabled: %s", tostring(metrics_data.predictive_enabled)))
    
    -- Проверяем наличие критических метрик
    local required_fields = {
        "current_nodes", "min_nodes", "max_nodes",
        "evaluation_count", "predictive_enabled"
    }
    
    for _, field in ipairs(required_fields) do
        if metrics_data[field] == nil then
            print("❌ Missing metric: " .. field)
            return false
        end
    end
    
    print("✅ All required metrics present")
    return true
end

local function test_cooldown_behavior()
    print("\n=== Test 5: Cooldown Behavior ===")
    
    -- Быстро генерируем две всплески нагрузки
    for i = 1, 2 do
        print(string.format("Load spike %d/2", i))
        generate_test_load(30, 800)
        db.sleep(10) -- Короткая пауза между всплесками
    end
    
    local history = db.execute("SHOW AUTOSCALING HISTORY")
    local history_data = db.json_decode(history)
    
    -- Проверяем, что масштабирование не происходило слишком часто
    local scale_ups = {}
    for _, event in ipairs(history_data) do
        if event.decision == "scale_up" then
            table.insert(scale_ups, event.timestamp)
        end
    end
    
    local cooldown_ok = true
    for i = 2, #scale_ups do
        local time_diff = scale_ups[i] - scale_ups[i-1]
        if time_diff < 300 then -- 5 минут cooldown
            cooldown_ok = false
            print(string.format("⚠️ Scale ups too close: %d seconds apart", time_diff))
        end
    end
    
    if cooldown_ok then
        print("✅ Cooldown mechanism working properly")
    else
        print("❌ Cooldown mechanism may be misconfigured")
    end
    
    return cooldown_ok
end

local function test_node_selection_for_scale_down()
    print("\n=== Test 6: Node Selection for Scale Down ===")
    
    -- Получаем нагрузку на узлах
    local nodes_load = db.execute("SHOW NODES LOAD")
    local load_data = db.json_decode(nodes_load)
    
    print("Node load distribution:")
    local loads = {}
    for _, node in ipairs(load_data) do
        table.insert(loads, node.load)
        print(string.format("  %s: %.2f%%", node.id, node.load * 100))
    end
    
    -- Сортируем узлы по нагрузке
    table.sort(load_data, function(a, b) return a.load < b.load end)
    
    -- Наименее загруженный узел должен быть кандидатом на удаление
    local lowest_load_node = load_data[1]
    print(string.format("Lowest load node: %s (%.2f%%)", 
        lowest_load_node.id, lowest_load_node.load * 100))
    
    -- Запрашиваем кандидата на удаление у системы
    local scale_down_candidate = db.execute("SHOW SCALE_DOWN_CANDIDATE")
    local candidate = db.json_decode(scale_down_candidate)
    
    if candidate.id == lowest_load_node.id then
        print("✅ Correct node selected for scale down")
        return true
    else
        print(string.format("❌ Wrong node selected. Expected: %s, Got: %s",
            lowest_load_node.id, candidate.id))
        return false
    end
end

-- Функция очистки
local function cleanup()
    print("\n=== Cleanup ===")
    db.execute("DROP DATABASE IF EXISTS autoscale_test")
    print("Test database removed")
end

-- Запуск всех тестов
local function run_all_autoscaling_tests()
    print("╔══════════════════════════════════════════════════════════════╗")
    print("║     AUTOSCALING TESTS SUITE                                   ║")
    print("╚══════════════════════════════════════════════════════════════╝")
    
    setup_test_environment()
    
    local results = {
        test_scale_up_on_high_load(),
        test_scale_down_on_low_load(),
        test_predictive_scaling(),
        test_autoscaling_metrics(),
        test_cooldown_behavior(),
        test_node_selection_for_scale_down()
    }
    
    -- Подведение итогов
    print("\n╔══════════════════════════════════════════════════════════════╗")
    print("║                     TEST RESULTS                              ║")
    print("╚══════════════════════════════════════════════════════════════╝")
    
    local passed = 0
    for i, result in ipairs(results) do
        local status = result and "✅ PASSED" or "⚠️ SKIPPED/FAILED"
        print(string.format("Test %d: %s", i, status))
        if result then passed = passed + 1 end
    end
    
    print(string.format("\nTotal: %d/%d tests passed", passed, #results))
    
    cleanup()
    
    return passed == #results
end

-- Запуск
return run_all_autoscaling_tests()