From 12db4ddd6e0c8a4fd179eafcc523ebe0508c8fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=93=D1=80=D0=B8=D0=B3=D0=BE=D1=80=D0=B8=D0=B9=20=D0=A1?= =?UTF-8?q?=D0=B0=D1=84=D1=80=D0=BE=D0=BD=D0=BE=D0=B2?= Date: Thu, 17 Sep 2026 22:05:23 +0000 Subject: [PATCH] Upload files to "internal/storage" --- internal/storage/collection.go | 1902 +++++++++++++ internal/storage/runtime_limits.go | 446 +++ internal/storage/transactions.go | 4121 ++++++++++++++++++++++++++++ 3 files changed, 6469 insertions(+) create mode 100644 internal/storage/collection.go create mode 100644 internal/storage/runtime_limits.go create mode 100644 internal/storage/transactions.go diff --git a/internal/storage/collection.go b/internal/storage/collection.go new file mode 100644 index 0000000..dc03fd9 --- /dev/null +++ b/internal/storage/collection.go @@ -0,0 +1,1902 @@ +/* + * Copyright 2026 Safronov Grigorii + * + * Licensed under the CDDL, Version 1.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at + * https://opensource.org/licenses/CDDL-1.0 + */ + +// Файл: internal/storage/collection.go +// Назначение: Реализация коллекции с индексами (первичными и вторичными). +// Индексы хранятся отдельно от документов, обеспечивают wait-free доступ. +// Исправлено: корректная работа уникальных индексов, удаление из индексов при обновлении. +// Lock-free: ACL и Constraints переведены на атомарные операции. +// MVCC: добавлена поддержка многоверсионности. + +package storage + +import ( + "fmt" + "sync" + "sync/atomic" + "time" + "strings" + + "futriis/internal/serializer" +) + +// ============================================================================= +// КОНСТАНТЫ (удалены дублирующиеся, используются из trigger.go) +// ============================================================================= + +// MaxVersionsPerDoc, VersionRetentionDays, VisibilityMapSize +// определены в trigger.go + +// ============================================================================= +// Collection - ОСНОВНАЯ СТРУКТУРА +// ============================================================================= + +// Collection представляет коллекцию документов (аналог таблицы) +type Collection struct { + dbName string // имя базы данных + name string // имя коллекции + docs sync.Map // map[string]*Document - wait-free хранилище документов + indexes sync.Map // map[string]*Index - индексы для быстрого поиска (lock-free) + metadata *CollectionMetadata // Метаданные коллекции + docCount atomic.Int64 // Атомарный счётчик документов + sizeBytes atomic.Int64 // Атомарный размер коллекции в байтах + mu sync.RWMutex // Для операций, изменяющих структуру коллекции + constraints *Constraints // Ограничения коллекции (lock-free) + acl *CollectionACL // ACL для коллекции (lock-free) + triggers sync.Map // map[string]*Trigger - триггеры коллекции (использует тип из trigger.go) + + // MVCC - многоверсионность + mvccManager *MVCCManager // MVCC менеджер для управления версиями + mvccEnabled bool // Включен ли MVCC + + // ИСПРАВЛЕНО: Блокировка для согласованности индексов при SoftDelete + indexMu sync.Mutex // Защита атомарности операций с индексами +} + +// CollectionMetadata содержит метаданные коллекции +type CollectionMetadata struct { + Name string `msgpack:"name"` + CreatedAt int64 `msgpack:"created_at"` + UpdatedAt int64 `msgpack:"updated_at"` + DeletedAt int64 `msgpack:"deleted_at"` + DocumentCount int64 `msgpack:"document_count"` + SizeBytes int64 `msgpack:"size_bytes"` + IndexCount int `msgpack:"index_count"` + Settings *CollectionSettings `msgpack:"settings"` +} + +// CollectionSettings содержит настройки коллекции +type CollectionSettings struct { + MaxDocuments int `msgpack:"max_documents"` // Максимальное количество документов (0 = безлимит) + ValidateSchema bool `msgpack:"validate_schema"` // Валидировать схему документов + AutoIndexID bool `msgpack:"auto_index_id"` // Автоматически индексировать поле _id + TTLSeconds int `msgpack:"ttl_seconds"` // Время жизни документов (0 = бессрочно) + SoftDelete bool `msgpack:"soft_delete"` // Мягкое удаление документов + MVCCEnabled bool `msgpack:"mvcc_enabled"` // Включить MVCC + MaxVersions int `msgpack:"max_versions"` // Максимальное количество версий на документ +} + +// Index представляет индекс для ускорения поиска (хранится отдельно от документов) +type Index struct { + Name string `msgpack:"name"` + Fields []string `msgpack:"fields"` // Поля для индексации + Unique bool `msgpack:"unique"` // Уникальный индекс + CreatedAt int64 `msgpack:"created_at"` // Время создания индекса + UpdatedAt int64 `msgpack:"updated_at"` // Время последнего обновления индекса + data sync.Map // map[interface{}]string - значение индекса -> ID документа (lock-free) +} + +// Constraints представляет ограничения на коллекцию (lock-free версия) +type Constraints struct { + // Атомарные указатели для lock-free доступа к данным ограничений + requiredFieldsPtr atomic.Value // map[string]bool + uniqueFieldsPtr atomic.Value // map[string]bool + minValuesPtr atomic.Value // map[string]float64 + maxValuesPtr atomic.Value // map[string]float64 + patternFieldsPtr atomic.Value // map[string]string + enumFieldsPtr atomic.Value // map[string][]interface{} + + createdAt int64 + updatedAt atomic.Int64 + constraintHistory atomic.Value // []ConstraintChange +} + +// ConstraintChange представляет изменение ограничения +type ConstraintChange struct { + Timestamp int64 `msgpack:"timestamp"` + TimestampStr string `msgpack:"timestamp_str"` + Action string `msgpack:"action"` // ADD, REMOVE, MODIFY + ConstraintType string `msgpack:"constraint_type"` // required, unique, min, max, enum, regex + Field string `msgpack:"field"` + OldValue interface{} `msgpack:"old_value,omitempty"` + NewValue interface{} `msgpack:"new_value,omitempty"` +} + +// CollectionACL представляет список контроля доступа для коллекции (lock-free версия) +type CollectionACL struct { + // Атомарные указатели для lock-free доступа к ACL + readRolesPtr atomic.Value // map[string]bool + writeRolesPtr atomic.Value // map[string]bool + deleteRolesPtr atomic.Value // map[string]bool + adminRolesPtr atomic.Value // map[string]bool + + createdAt int64 + updatedAt atomic.Int64 + aclHistory atomic.Value // []ACLChange +} + +// ACLChange представляет изменение ACL +type ACLChange struct { + Timestamp int64 `msgpack:"timestamp"` + TimestampStr string `msgpack:"timestamp_str"` + Action string `msgpack:"action"` // GRANT, REVOKE, SET + Role string `msgpack:"role"` + Permission string `msgpack:"permission"` // read, write, delete, admin + Granted bool `msgpack:"granted"` +} + +// NewConstraints создаёт новый экземпляр Constraints с lock-free реализацией +func NewConstraints() *Constraints { + c := &Constraints{ + createdAt: time.Now().UnixMilli(), + } + + // Инициализация атомарных указателей с пустыми map + c.requiredFieldsPtr.Store(make(map[string]bool)) + c.uniqueFieldsPtr.Store(make(map[string]bool)) + c.minValuesPtr.Store(make(map[string]float64)) + c.maxValuesPtr.Store(make(map[string]float64)) + c.patternFieldsPtr.Store(make(map[string]string)) + c.enumFieldsPtr.Store(make(map[string][]interface{})) + c.constraintHistory.Store(make([]ConstraintChange, 0)) + c.updatedAt.Store(c.createdAt) + + return c +} + +// loadRequiredFields загружает карту обязательных полей (lock-free) +func (cons *Constraints) loadRequiredFields() map[string]bool { + val := cons.requiredFieldsPtr.Load() + if val == nil { + return make(map[string]bool) + } + return val.(map[string]bool) +} + +// storeRequiredFields сохраняет карту обязательных полей (lock-free) +func (cons *Constraints) storeRequiredFields(newMap map[string]bool) { + cons.requiredFieldsPtr.Store(newMap) +} + +// loadUniqueFields загружает карту уникальных полей (lock-free) +func (cons *Constraints) loadUniqueFields() map[string]bool { + val := cons.uniqueFieldsPtr.Load() + if val == nil { + return make(map[string]bool) + } + return val.(map[string]bool) +} + +// storeUniqueFields сохраняет карту уникальных полей (lock-free) +func (cons *Constraints) storeUniqueFields(newMap map[string]bool) { + cons.uniqueFieldsPtr.Store(newMap) +} + +// loadMinValues загружает карту минимальных значений (lock-free) +func (cons *Constraints) loadMinValues() map[string]float64 { + val := cons.minValuesPtr.Load() + if val == nil { + return make(map[string]float64) + } + return val.(map[string]float64) +} + +// storeMinValues сохраняет карту минимальных значений (lock-free) +func (cons *Constraints) storeMinValues(newMap map[string]float64) { + cons.minValuesPtr.Store(newMap) +} + +// loadMaxValues загружает карту максимальных значений (lock-free) +func (cons *Constraints) loadMaxValues() map[string]float64 { + val := cons.maxValuesPtr.Load() + if val == nil { + return make(map[string]float64) + } + return val.(map[string]float64) +} + +// storeMaxValues сохраняет карту максимальных значений (lock-free) +func (cons *Constraints) storeMaxValues(newMap map[string]float64) { + cons.maxValuesPtr.Store(newMap) +} + +// loadPatternFields загружает карту regex паттернов (lock-free) +func (cons *Constraints) loadPatternFields() map[string]string { + val := cons.patternFieldsPtr.Load() + if val == nil { + return make(map[string]string) + } + return val.(map[string]string) +} + +// storePatternFields сохраняет карту regex паттернов (lock-free) +func (cons *Constraints) storePatternFields(newMap map[string]string) { + cons.patternFieldsPtr.Store(newMap) +} + +// loadEnumFields загружает карту enum ограничений (lock-free) +func (cons *Constraints) loadEnumFields() map[string][]interface{} { + val := cons.enumFieldsPtr.Load() + if val == nil { + return make(map[string][]interface{}) + } + // Возвращаем копию, чтобы избежать модификации + original := val.(map[string][]interface{}) + result := make(map[string][]interface{}) + for k, v := range original { + copied := make([]interface{}, len(v)) + copy(copied, v) + result[k] = copied + } + return result +} + +// storeEnumFields сохраняет карту enum ограничений (lock-free) +func (cons *Constraints) storeEnumFields(newMap map[string][]interface{}) { + cons.enumFieldsPtr.Store(newMap) +} + +// loadConstraintHistory загружает историю изменений (lock-free) +func (cons *Constraints) loadConstraintHistory() []ConstraintChange { + val := cons.constraintHistory.Load() + if val == nil { + return make([]ConstraintChange, 0) + } + return val.([]ConstraintChange) +} + +// storeConstraintHistory сохраняет историю изменений (lock-free) +func (cons *Constraints) storeConstraintHistory(history []ConstraintChange) { + cons.constraintHistory.Store(history) +} + +// NewCollectionACL создаёт новый экземпляр CollectionACL с lock-free реализацией +func NewCollectionACL() *CollectionACL { + acl := &CollectionACL{ + createdAt: time.Now().UnixMilli(), + } + + acl.readRolesPtr.Store(make(map[string]bool)) + acl.writeRolesPtr.Store(make(map[string]bool)) + acl.deleteRolesPtr.Store(make(map[string]bool)) + acl.adminRolesPtr.Store(make(map[string]bool)) + acl.aclHistory.Store(make([]ACLChange, 0)) + acl.updatedAt.Store(acl.createdAt) + + return acl +} + +// loadReadRoles загружает роли чтения (lock-free) +func (acl *CollectionACL) loadReadRoles() map[string]bool { + val := acl.readRolesPtr.Load() + if val == nil { + return make(map[string]bool) + } + return val.(map[string]bool) +} + +// storeReadRoles сохраняет роли чтения (lock-free) +func (acl *CollectionACL) storeReadRoles(roles map[string]bool) { + acl.readRolesPtr.Store(roles) +} + +// loadWriteRoles загружает роли записи (lock-free) +func (acl *CollectionACL) loadWriteRoles() map[string]bool { + val := acl.writeRolesPtr.Load() + if val == nil { + return make(map[string]bool) + } + return val.(map[string]bool) +} + +// storeWriteRoles сохраняет роли записи (lock-free) +func (acl *CollectionACL) storeWriteRoles(roles map[string]bool) { + acl.writeRolesPtr.Store(roles) +} + +// loadDeleteRoles загружает роли удаления (lock-free) +func (acl *CollectionACL) loadDeleteRoles() map[string]bool { + val := acl.deleteRolesPtr.Load() + if val == nil { + return make(map[string]bool) + } + return val.(map[string]bool) +} + +// storeDeleteRoles сохраняет роли удаления (lock-free) +func (acl *CollectionACL) storeDeleteRoles(roles map[string]bool) { + acl.deleteRolesPtr.Store(roles) +} + +// loadAdminRoles загружает административные роли (lock-free) +func (acl *CollectionACL) loadAdminRoles() map[string]bool { + val := acl.adminRolesPtr.Load() + if val == nil { + return make(map[string]bool) + } + return val.(map[string]bool) +} + +// storeAdminRoles сохраняет административные роли (lock-free) +func (acl *CollectionACL) storeAdminRoles(roles map[string]bool) { + acl.adminRolesPtr.Store(roles) +} + +// loadACLHistory загружает историю ACL (lock-free) +func (acl *CollectionACL) loadACLHistory() []ACLChange { + val := acl.aclHistory.Load() + if val == nil { + return make([]ACLChange, 0) + } + return val.([]ACLChange) +} + +// storeACLHistory сохраняет историю ACL (lock-free) +func (acl *CollectionACL) storeACLHistory(history []ACLChange) { + acl.aclHistory.Store(history) +} + +// CheckPermission проверяет наличие разрешения у роли (lock-free) +func (acl *CollectionACL) CheckPermission(role, operation string) bool { + // Администратор имеет все права (lock-free чтение) + adminRoles := acl.loadAdminRoles() + if adminRoles[role] { + return true + } + + switch operation { + case "read": + readRoles := acl.loadReadRoles() + return readRoles[role] + case "write": + writeRoles := acl.loadWriteRoles() + return writeRoles[role] + case "delete": + deleteRoles := acl.loadDeleteRoles() + return deleteRoles[role] + default: + return false + } +} + +// NewCollection создаёт новую коллекцию +func NewCollection(dbName, name string, settings *CollectionSettings) *Collection { + if settings == nil { + settings = &CollectionSettings{ + MaxDocuments: 0, + ValidateSchema: false, + AutoIndexID: true, + TTLSeconds: 0, + SoftDelete: false, + MVCCEnabled: false, + MaxVersions: MaxVersionsPerDoc, + } + } + + now := time.Now().UnixMilli() + coll := &Collection{ + dbName: dbName, + name: name, + metadata: &CollectionMetadata{ + Name: name, + CreatedAt: now, + UpdatedAt: now, + DeletedAt: 0, + DocumentCount: 0, + SizeBytes: 0, + IndexCount: 0, + Settings: settings, + }, + constraints: NewConstraints(), + acl: NewCollectionACL(), + mvccEnabled: settings.MVCCEnabled, + } + + // Создаём MVCC менеджер, если включен + if settings.MVCCEnabled { + maxVersions := settings.MaxVersions + if maxVersions <= 0 { + maxVersions = MaxVersionsPerDoc + } + coll.mvccManager = NewMVCCManager(maxVersions, VersionRetentionDays) + } + + // Автоматически создаём первичный индекс по _id + if settings.AutoIndexID { + coll.CreateIndex("_id_", []string{"_id"}, true) + } + + // Запускаем фоновую задачу для удаления просроченных документов + if settings.TTLSeconds > 0 { + go coll.ttlCleanupLoop() + } + + // Аудит создания коллекции - используем функцию из audit.go + AuditCollectionOperation("CREATE", dbName, name, settings) + + return coll +} + +// ============================================================================= +// МЕТОДЫ ДЛЯ РАБОТЫ С ТРИГГЕРАМИ (используют тип Trigger из trigger.go) +// ============================================================================= + +// AddTrigger добавляет триггер в коллекцию +func (c *Collection) AddTrigger(trigger *Trigger) error { + if trigger.Name == "" { + return fmt.Errorf("trigger name cannot be empty") + } + + if _, exists := c.triggers.Load(trigger.Name); exists { + return fmt.Errorf("trigger '%s' already exists", trigger.Name) + } + + trigger.CreatedAt = time.Now().UnixMilli() + trigger.UpdatedAt = trigger.CreatedAt + + c.triggers.Store(trigger.Name, trigger) + c.mu.Lock() + c.metadata.UpdatedAt = time.Now().UnixMilli() + c.mu.Unlock() + + AuditDocumentOperation("CREATE_TRIGGER", c.dbName, c.name, trigger.Name, map[string]interface{}{ + "event": trigger.Event, + "action": trigger.Action, + }) + + return nil +} + +// DropTrigger удаляет триггер из коллекции +func (c *Collection) DropTrigger(name string) error { + if _, exists := c.triggers.LoadAndDelete(name); !exists { + return fmt.Errorf("trigger '%s' not found", name) + } + + c.mu.Lock() + c.metadata.UpdatedAt = time.Now().UnixMilli() + c.mu.Unlock() + + AuditDocumentOperation("DROP_TRIGGER", c.dbName, c.name, name, nil) + + return nil +} + +// ListTriggers возвращает список всех триггеров в коллекции +func (c *Collection) ListTriggers() []*Trigger { + triggers := make([]*Trigger, 0) + c.triggers.Range(func(key, value interface{}) bool { + triggers = append(triggers, value.(*Trigger)) + return true + }) + return triggers +} + +// GetTrigger возвращает триггер по имени +func (c *Collection) GetTrigger(name string) (*Trigger, bool) { + if val, ok := c.triggers.Load(name); ok { + return val.(*Trigger), true + } + return nil, false +} + +// EnableTrigger включает триггер +func (c *Collection) EnableTrigger(name string) error { + val, ok := c.triggers.Load(name) + if !ok { + return fmt.Errorf("trigger '%s' not found", name) + } + + trigger := val.(*Trigger) + trigger.Enabled = true + trigger.UpdatedAt = time.Now().UnixMilli() + c.triggers.Store(name, trigger) + + c.mu.Lock() + c.metadata.UpdatedAt = time.Now().UnixMilli() + c.mu.Unlock() + + AuditDocumentOperation("ENABLE_TRIGGER", c.dbName, c.name, name, nil) + + return nil +} + +// DisableTrigger отключает триггер +func (c *Collection) DisableTrigger(name string) error { + val, ok := c.triggers.Load(name) + if !ok { + return fmt.Errorf("trigger '%s' not found", name) + } + + trigger := val.(*Trigger) + trigger.Enabled = false + trigger.UpdatedAt = time.Now().UnixMilli() + c.triggers.Store(name, trigger) + + c.mu.Lock() + c.metadata.UpdatedAt = time.Now().UnixMilli() + c.mu.Unlock() + + AuditDocumentOperation("DISABLE_TRIGGER", c.dbName, c.name, name, nil) + + return nil +} + +// executeTriggers выполняет триггеры для заданного события +func (c *Collection) executeTriggers(event string, docID string, data map[string]interface{}) error { + triggers := c.ListTriggers() + for _, trigger := range triggers { + if !trigger.Enabled || trigger.Event != event { + continue + } + + // Логируем выполнение триггера + LogTriggerExecution(trigger.Name, event, c.name, docID, trigger.Action, true, nil) + + // Здесь должна быть логика выполнения действий триггера + // В зависимости от trigger.Action: abort, skip, modify, log, notify + } + return nil +} + +// ============================================================================= +// МЕТОДЫ ДЛЯ РАБОТЫ С MVCC +// ============================================================================= + +// GetMVCCManager возвращает MVCC менеджер +func (c *Collection) GetMVCCManager() *MVCCManager { + return c.mvccManager +} + +// IsMVCCEnabled проверяет, включен ли MVCC +func (c *Collection) IsMVCCEnabled() bool { + return c.mvccEnabled +} + +// GetVersionAt возвращает версию документа на указанный момент времени +func (c *Collection) GetVersionAt(id string, timestamp int64) (*Document, error) { + if !c.mvccEnabled || c.mvccManager == nil { + return nil, fmt.Errorf("MVCC is not enabled for this collection") + } + + doc := c.mvccManager.GetVersionAt(id, timestamp) + if doc == nil { + return nil, fmt.Errorf("version not found for document %s at timestamp %d", id, timestamp) + } + return doc, nil +} + +// GetAllVersions возвращает все версии документа +func (c *Collection) GetAllVersions(id string) ([]*DocumentVersion, error) { + if !c.mvccEnabled || c.mvccManager == nil { + return nil, fmt.Errorf("MVCC is not enabled for this collection") + } + + versions := c.mvccManager.GetAllVersions(id) + if versions == nil { + return nil, fmt.Errorf("document %s not found", id) + } + return versions, nil +} + +// GetLatestVersion возвращает последнюю версию документа +func (c *Collection) GetLatestVersion(id string) (*Document, error) { + if !c.mvccEnabled || c.mvccManager == nil { + return nil, fmt.Errorf("MVCC is not enabled for this collection") + } + + doc := c.mvccManager.GetLatestVersion(id) + if doc == nil { + return nil, fmt.Errorf("document %s not found", id) + } + return doc, nil +} + +// GetMVCCStats возвращает статистику MVCC +func (c *Collection) GetMVCCStats() map[string]interface{} { + if !c.mvccEnabled || c.mvccManager == nil { + return map[string]interface{}{ + "enabled": false, + } + } + stats := c.mvccManager.GetMVCCStats() + stats["enabled"] = true + return stats +} + +// ============================================================================= +// БАЗОВЫЕ МЕТОДЫ КОЛЛЕКЦИИ +// ============================================================================= + +// Name возвращает имя коллекции +func (c *Collection) Name() string { + return c.name +} + +// DBName возвращает имя базы данных +func (c *Collection) DBName() string { + return c.dbName +} + +// Insert вставляет документ в коллекцию (wait-free) +func (c *Collection) Insert(doc *Document) error { + // Проверка ограничений + if err := c.constraints.ValidateDocument(doc); err != nil { + return fmt.Errorf("constraint violation: %v", err) + } + + // Проверка на максимальное количество документов + if c.metadata.Settings.MaxDocuments > 0 { + if c.docCount.Load() >= int64(c.metadata.Settings.MaxDocuments) { + return fmt.Errorf("collection is full: max documents %d reached", c.metadata.Settings.MaxDocuments) + } + } + + // Валидация схемы (если включена) + if c.metadata.Settings.ValidateSchema { + if err := c.validateDocument(doc); err != nil { + return fmt.Errorf("document validation failed: %v", err) + } + } + + // Проверка уникальных индексов + if err := c.checkUniqueConstraints(doc); err != nil { + return err + } + + // Обновляем временные метки документа + doc.CreatedAt = time.Now().UnixMilli() + doc.UpdatedAt = doc.CreatedAt + doc.DeletedAt = 0 + + // Сериализация для проверки (опционально) + data, err := serializer.Marshal(doc) + if err != nil { + return fmt.Errorf("failed to serialize document: %v", err) + } + + // Атомарное сохранение документа + if _, loaded := c.docs.LoadOrStore(doc.ID, doc); loaded { + return fmt.Errorf("document with id %s already exists", doc.ID) + } + + // Если MVCC включен, создаём версию + if c.mvccEnabled && c.mvccManager != nil { + c.mvccManager.CreateVersion(doc, 0) // 0 = системная транзакция + } + + // Обновление индексов (wait-free) + c.updateIndexes(doc, true) + + // Обновление метаданных + c.docCount.Add(1) + c.sizeBytes.Add(int64(len(data))) + + c.mu.Lock() + c.metadata.DocumentCount = c.docCount.Load() + c.metadata.SizeBytes = c.sizeBytes.Load() + c.metadata.UpdatedAt = time.Now().UnixMilli() + c.mu.Unlock() + + // Аудит вставки документа + AuditDocumentOperation("INSERT", c.dbName, c.name, doc.ID, doc.GetFields()) + + // Выполняем триггеры AFTER_INSERT + c.executeTriggers("AFTER_INSERT", doc.ID, doc.GetFields()) + + return nil +} + +// InsertFromMap создаёт и вставляет документ из map +func (c *Collection) InsertFromMap(fields map[string]interface{}) error { + doc := NewDocument() + for k, v := range fields { + doc.SetField(k, v) + } + return c.Insert(doc) +} + +// Find находит документ по ID (с использованием первичного индекса) +func (c *Collection) Find(id string) (*Document, error) { + if val, ok := c.docs.Load(id); ok { + doc := val.(*Document) + + // Проверяем мягкое удаление + if doc.IsDeleted() && c.metadata.Settings.SoftDelete { + return nil, fmt.Errorf("document deleted at %s", doc.GetDeletedAtStr()) + } + + // Проверяем, не истёк ли TTL + if c.metadata.Settings.TTLSeconds > 0 { + if time.Now().UnixMilli()-doc.CreatedAt > int64(c.metadata.Settings.TTLSeconds*1000) { + if c.metadata.Settings.SoftDelete { + doc.SoftDelete() + c.docs.Store(id, doc) + } else { + c.Delete(id) + } + return nil, fmt.Errorf("key not found") + } + } + return doc, nil + } + return nil, fmt.Errorf("key not found") +} + +// FindIncludingDeleted находит документ даже если он мягко удалён +func (c *Collection) FindIncludingDeleted(id string) (*Document, error) { + if val, ok := c.docs.Load(id); ok { + return val.(*Document), nil + } + return nil, fmt.Errorf("key not found") +} + +// compareValues сравнивает два значения с учётом типа +func compareValues(a, b interface{}) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + + // Пробуем прямое сравнение + if a == b { + return true + } + + // Пробуем сравнение через строковое представление для разных типов + aStr := fmt.Sprintf("%v", a) + bStr := fmt.Sprintf("%v", b) + return aStr == bStr +} + +// FindByIndex находит документы по значению индексированного поля +func (c *Collection) FindByIndex(indexName string, value interface{}) ([]*Document, error) { + idxVal, ok := c.indexes.Load(indexName) + if !ok { + return nil, fmt.Errorf("index not found: %s", indexName) + } + + index := idxVal.(*Index) + docs := make([]*Document, 0) + + if index.Unique { + // Уникальный индекс возвращает один документ + if docID, ok := index.data.Load(value); ok { + if doc, err := c.Find(docID.(string)); err == nil { + docs = append(docs, doc) + } + } + } else { + // Для неуникального индекса используем Range с правильным сравнением + index.data.Range(func(key, val interface{}) bool { + if compareValues(key, value) { + if doc, err := c.Find(val.(string)); err == nil { + docs = append(docs, doc) + } + } + return true + }) + } + + return docs, nil +} + +// FindByIndexPrefix находит документы по префиксу индекса (для строковых полей) +func (c *Collection) FindByIndexPrefix(indexName string, prefix string) ([]*Document, error) { + idxVal, ok := c.indexes.Load(indexName) + if !ok { + return nil, fmt.Errorf("index not found: %s", indexName) + } + + index := idxVal.(*Index) + docs := make([]*Document, 0) + + index.data.Range(func(key, val interface{}) bool { + if keyStr, ok := key.(string); ok { + if strings.HasPrefix(keyStr, prefix) { + if doc, err := c.Find(val.(string)); err == nil { + docs = append(docs, doc) + } + } + } + return true + }) + + return docs, nil +} + +// Update обновляет документ по ID +func (c *Collection) Update(id string, updates map[string]interface{}) error { + val, ok := c.docs.Load(id) + if !ok { + return fmt.Errorf("key not found") + } + + oldDoc := val.(*Document) + + // Проверяем, не удалён ли документ мягко + if oldDoc.IsDeleted() && c.metadata.Settings.SoftDelete { + return fmt.Errorf("cannot update deleted document") + } + + // Создаём копию для проверки уникальности + newDoc := oldDoc.Clone() + if err := newDoc.Update(updates); err != nil { + return err + } + + // Обновляем временную метку + newDoc.UpdatedAt = time.Now().UnixMilli() + + // Проверяем ограничения + if err := c.constraints.ValidateDocument(newDoc); err != nil { + return fmt.Errorf("constraint violation: %v", err) + } + + // Проверяем уникальные индексы + if err := c.checkUniqueConstraintsUpdate(oldDoc, newDoc); err != nil { + return err + } + + // Выполняем триггеры BEFORE_UPDATE + c.executeTriggers("BEFORE_UPDATE", id, newDoc.GetFields()) + + // ИСПРАВЛЕНО: Блокируем индексы для атомарности + c.indexMu.Lock() + // Сначала удаляем старые индексы, потом добавляем новые + c.removeFromIndexes(oldDoc) + c.addToIndexes(newDoc) + c.indexMu.Unlock() + + // Сохраняем обновлённый документ + c.docs.Store(id, newDoc) + + // Если MVCC включен, создаём версию + if c.mvccEnabled && c.mvccManager != nil { + c.mvccManager.CreateVersion(newDoc, 0) + } + + c.mu.Lock() + c.metadata.UpdatedAt = time.Now().UnixMilli() + c.mu.Unlock() + + // Аудит обновления документа + AuditDocumentOperation("UPDATE", c.dbName, c.name, id, updates) + + // Выполняем триггеры AFTER_UPDATE + c.executeTriggers("AFTER_UPDATE", id, newDoc.GetFields()) + + return nil +} + +// Delete удаляет документ по ID +// ИСПРАВЛЕНО: Атомарность SoftDelete и удаления из индексов +func (c *Collection) Delete(id string) error { + val, ok := c.docs.Load(id) + if !ok { + return fmt.Errorf("key not found") + } + + doc := val.(*Document) + + // Выполняем триггеры BEFORE_DELETE + c.executeTriggers("BEFORE_DELETE", id, doc.GetFields()) + + if c.metadata.Settings.SoftDelete { + // ИСПРАВЛЕНО: Блокируем индексы для атомарности SoftDelete и удаления из индексов + c.indexMu.Lock() + + // Мягкое удаление + doc.SoftDelete() + c.docs.Store(id, doc) + + // Удаляем из индексов при мягком удалении + c.removeFromIndexes(doc) + + c.indexMu.Unlock() + + // Аудит мягкого удаления + AuditDocumentOperation("SOFT_DELETE", c.dbName, c.name, id, map[string]interface{}{ + "deleted_at": doc.DeletedAt, + }) + } else { + // ИСПРАВЛЕНО: Блокируем индексы для атомарности физического удаления + c.indexMu.Lock() + + // Физическое удаление + // Удаляем из индексов + c.removeFromIndexes(doc) + + // Удаляем документ + c.docs.Delete(id) + + c.indexMu.Unlock() + + // Аудит физического удаления + AuditDocumentOperation("DELETE", c.dbName, c.name, id, nil) + } + + // Обновляем метаданные + c.docCount.Add(-1) + c.mu.Lock() + c.metadata.DocumentCount = c.docCount.Load() + c.metadata.UpdatedAt = time.Now().UnixMilli() + c.mu.Unlock() + + // Выполняем триггеры AFTER_DELETE + c.executeTriggers("AFTER_DELETE", id, nil) + + return nil +} + +// PermanentDelete выполняет физическое удаление мягко удалённого документа +func (c *Collection) PermanentDelete(id string) error { + val, ok := c.docs.Load(id) + if !ok { + return fmt.Errorf("key not found") + } + + doc := val.(*Document) + + if !doc.IsDeleted() { + return fmt.Errorf("document is not soft deleted, use Delete() instead") + } + + // ИСПРАВЛЕНО: Блокируем индексы для атомарности + c.indexMu.Lock() + + // Удаляем из индексов + c.removeFromIndexes(doc) + + // Удаляем документ + c.docs.Delete(id) + + c.indexMu.Unlock() + + // Обновляем метаданные + c.docCount.Add(-1) + c.mu.Lock() + c.metadata.DocumentCount = c.docCount.Load() + c.metadata.UpdatedAt = time.Now().UnixMilli() + c.mu.Unlock() + + // Аудит физического удаления + AuditDocumentOperation("PERMANENT_DELETE", c.dbName, c.name, id, nil) + + return nil +} + +// RestoreDeleted восстанавливает мягко удалённый документ +// ИСПРАВЛЕНО: Атомарность восстановления и добавления в индексы +func (c *Collection) RestoreDeleted(id string) error { + val, ok := c.docs.Load(id) + if !ok { + return fmt.Errorf("key not found") + } + + doc := val.(*Document) + + if !doc.IsDeleted() { + return fmt.Errorf("document is not deleted") + } + + // ИСПРАВЛЕНО: Блокируем индексы для атомарности + c.indexMu.Lock() + + doc.Restore() + + // Восстанавливаем индексы + c.addToIndexes(doc) + + c.docs.Store(id, doc) + + c.indexMu.Unlock() + + c.mu.Lock() + c.metadata.UpdatedAt = time.Now().UnixMilli() + c.mu.Unlock() + + // Аудит восстановления + AuditDocumentOperation("RESTORE", c.dbName, c.name, id, nil) + + return nil +} + +// removeFromIndexes удаляет документ из всех индексов (wait-free) +func (c *Collection) removeFromIndexes(doc *Document) { + c.indexes.Range(func(key, value interface{}) bool { + index := value.(*Index) + indexValue := c.extractIndexValue(doc, index.Fields) + index.data.Delete(indexValue) + return true + }) +} + +// addToIndexes добавляет документ во все индексы (wait-free) +func (c *Collection) addToIndexes(doc *Document) { + c.indexes.Range(func(key, value interface{}) bool { + index := value.(*Index) + indexValue := c.extractIndexValue(doc, index.Fields) + if index.Unique { + index.data.LoadOrStore(indexValue, doc.ID) + } else { + index.data.Store(indexValue, doc.ID) + } + return true + }) +} + +// CreateIndex создаёт новый индекс на коллекции +func (c *Collection) CreateIndex(name string, fields []string, unique bool) error { + c.mu.Lock() + defer c.mu.Unlock() + + if _, exists := c.indexes.Load(name); exists { + return fmt.Errorf("index %s already exists", name) + } + + now := time.Now().UnixMilli() + index := &Index{ + Name: name, + Fields: fields, + Unique: unique, + CreatedAt: now, + UpdatedAt: now, + } + + // Строим индекс на существующих документах (wait-free) + c.docs.Range(func(key, value interface{}) bool { + doc := value.(*Document) + // Пропускаем мягко удалённые документы + if doc.IsDeleted() && c.metadata.Settings.SoftDelete { + return true + } + indexValue := c.extractIndexValue(doc, fields) + if unique { + if _, loaded := index.data.LoadOrStore(indexValue, doc.ID); loaded { + // Найден дубликат - откатываем создание индекса + c.mu.Unlock() + return false + } + } else { + index.data.Store(indexValue, doc.ID) + } + return true + }) + + c.indexes.Store(name, index) + c.metadata.IndexCount++ + + // Аудит создания индекса + AuditIndexOperation("CREATE_INDEX", c.dbName, c.name, name, fields, unique) + + return nil +} + +// DropIndex удаляет индекс +func (c *Collection) DropIndex(name string) error { + if _, exists := c.indexes.LoadAndDelete(name); !exists { + return fmt.Errorf("index not found: %s", name) + } + c.metadata.IndexCount-- + + // Аудит удаления индекса + AuditIndexOperation("DROP_INDEX", c.dbName, c.name, name, nil, false) + + return nil +} + +// GetIndexes возвращает список всех индексов +func (c *Collection) GetIndexes() []string { + names := make([]string, 0) + c.indexes.Range(func(key, value interface{}) bool { + names = append(names, key.(string)) + return true + }) + return names +} + +// GetIndexesInfo возвращает подробную информацию об индексах (для API) +func (c *Collection) GetIndexesInfo() []map[string]interface{} { + indexes := make([]map[string]interface{}, 0) + c.indexes.Range(func(key, value interface{}) bool { + idx := value.(*Index) + indexes = append(indexes, map[string]interface{}{ + "name": idx.Name, + "fields": idx.Fields, + "unique": idx.Unique, + "created_at": idx.CreatedAt, + "updated_at": idx.UpdatedAt, + }) + return true + }) + return indexes +} + +// extractIndexValue извлекает значение из документа для индексации +func (c *Collection) extractIndexValue(doc *Document, fields []string) interface{} { + if len(fields) == 1 { + val, _ := doc.GetField(fields[0]) + return val + } + + // Составной индекс - возвращаем строковое представление + parts := make([]string, 0, len(fields)) + for _, field := range fields { + if val, err := doc.GetField(field); err == nil { + parts = append(parts, fmt.Sprintf("%v", val)) + } else { + parts = append(parts, "NULL") + } + } + return strings.Join(parts, "|") +} + +// updateIndexes обновляет индексы для документа +func (c *Collection) updateIndexes(doc *Document, add bool) { + if add { + c.addToIndexes(doc) + } else { + c.removeFromIndexes(doc) + } +} + +// checkUniqueConstraints проверяет уникальные индексы перед вставкой +func (c *Collection) checkUniqueConstraints(doc *Document) error { + var errs []string + + c.indexes.Range(func(key, value interface{}) bool { + index := value.(*Index) + if index.Unique { + indexValue := c.extractIndexValue(doc, index.Fields) + if _, exists := index.data.Load(indexValue); exists { + errs = append(errs, fmt.Sprintf("duplicate key for index %s: %v", index.Name, indexValue)) + } + } + return true + }) + + if len(errs) > 0 { + return fmt.Errorf("%s", strings.Join(errs, "; ")) + } + return nil +} + +// checkUniqueConstraintsUpdate проверяет уникальность при обновлении +func (c *Collection) checkUniqueConstraintsUpdate(oldDoc, newDoc *Document) error { + var errs []string + + c.indexes.Range(func(key, value interface{}) bool { + index := value.(*Index) + if index.Unique { + oldValue := c.extractIndexValue(oldDoc, index.Fields) + newValue := c.extractIndexValue(newDoc, index.Fields) + + if fmt.Sprintf("%v", oldValue) != fmt.Sprintf("%v", newValue) { + if _, exists := index.data.Load(newValue); exists { + errs = append(errs, fmt.Sprintf("duplicate key for index %s: %v", index.Name, newValue)) + } + } + } + return true + }) + + if len(errs) > 0 { + return fmt.Errorf("%s", strings.Join(errs, "; ")) + } + return nil +} + +// validateDocument валидирует документ согласно схеме коллекции +func (c *Collection) validateDocument(doc *Document) error { + if doc.ID == "" { + return fmt.Errorf("document must have _id field") + } + return nil +} + +// ttlCleanupLoop периодически удаляет просроченные документы +func (c *Collection) ttlCleanupLoop() { + ticker := time.NewTicker(time.Duration(c.metadata.Settings.TTLSeconds/2) * time.Second) + defer ticker.Stop() + + for range ticker.C { + now := time.Now().UnixMilli() + toDelete := make([]string, 0) + + c.docs.Range(func(key, value interface{}) bool { + doc := value.(*Document) + if !doc.IsDeleted() && now-doc.CreatedAt > int64(c.metadata.Settings.TTLSeconds*1000) { + toDelete = append(toDelete, doc.ID) + } + return true + }) + + for _, id := range toDelete { + c.Delete(id) + } + } +} + +// Count возвращает количество активных документов в коллекции +func (c *Collection) Count() int64 { + if c.metadata.Settings.SoftDelete { + // Подсчитываем только не удалённые документы + count := int64(0) + c.docs.Range(func(key, value interface{}) bool { + doc := value.(*Document) + if !doc.IsDeleted() { + count++ + } + return true + }) + return count + } + return c.docCount.Load() +} + +// CountAll возвращает общее количество документов (включая мягко удалённые) +func (c *Collection) CountAll() int64 { + return c.docCount.Load() +} + +// CountDeleted возвращает количество мягко удалённых документов +func (c *Collection) CountDeleted() int64 { + if !c.metadata.Settings.SoftDelete { + return 0 + } + + count := int64(0) + c.docs.Range(func(key, value interface{}) bool { + doc := value.(*Document) + if doc.IsDeleted() { + count++ + } + return true + }) + return count +} + +// Size возвращает размер коллекции в байтах +func (c *Collection) Size() int64 { + return c.sizeBytes.Load() +} + +// GetAllDocuments возвращает все активные документы коллекции +func (c *Collection) GetAllDocuments() []*Document { + docs := make([]*Document, 0, c.docCount.Load()) + c.docs.Range(func(key, value interface{}) bool { + doc := value.(*Document) + if !c.metadata.Settings.SoftDelete || !doc.IsDeleted() { + docs = append(docs, doc) + } + return true + }) + return docs +} + +// GetAllDocumentsIncludingDeleted возвращает все документы (включая мягко удалённые) +func (c *Collection) GetAllDocumentsIncludingDeleted() []*Document { + docs := make([]*Document, 0, c.docCount.Load()) + c.docs.Range(func(key, value interface{}) bool { + docs = append(docs, value.(*Document)) + return true + }) + return docs +} + +// FindByFilter находит документы по произвольному фильтру +func (c *Collection) FindByFilter(filter func(*Document) bool) []*Document { + results := make([]*Document, 0) + c.docs.Range(func(key, value interface{}) bool { + doc := value.(*Document) + if (!c.metadata.Settings.SoftDelete || !doc.IsDeleted()) && filter(doc) { + results = append(results, doc) + } + return true + }) + return results +} + +// GetMetadata возвращает метаданные коллекции +func (c *Collection) GetMetadata() *CollectionMetadata { + c.mu.RLock() + defer c.mu.RUnlock() + return c.metadata +} + +// GetTimestamps возвращает временные метки коллекции +func (c *Collection) GetTimestamps() map[string]int64 { + c.mu.RLock() + defer c.mu.RUnlock() + return map[string]int64{ + "created_at": c.metadata.CreatedAt, + "updated_at": c.metadata.UpdatedAt, + "deleted_at": c.metadata.DeletedAt, + } +} + +// Drop удаляет все документы из коллекции +func (c *Collection) Drop() error { + c.mu.Lock() + defer c.mu.Unlock() + + c.docs = sync.Map{} + c.indexes = sync.Map{} + c.triggers = sync.Map{} + + if c.metadata.Settings.AutoIndexID { + c.CreateIndex("_id_", []string{"_id"}, true) + } + + c.docCount.Store(0) + c.sizeBytes.Store(0) + c.metadata.DocumentCount = 0 + c.metadata.SizeBytes = 0 + c.metadata.UpdatedAt = time.Now().UnixMilli() + + // Аудит удаления коллекции + AuditCollectionOperation("DROP", c.dbName, c.name, nil) + + return nil +} + +// logConstraintChange логирует изменение ограничения (lock-free) +func (c *Collection) logConstraintChange(action, constraintType, field string, oldVal, newVal interface{}) { + // Используем функцию LogAudit из audit.go + LogAudit("CONSTRAINT_"+action, "CONSTRAINT", fmt.Sprintf("%s.%s", c.dbName, c.name), map[string]interface{}{ + "constraint_type": constraintType, + "field": field, + "old_value": oldVal, + "new_value": newVal, + }) +} + +// logACLChange логирует изменение ACL (lock-free) +func (c *Collection) logACLChange(action, role, permission string, granted bool) { + // Используем функцию LogAudit из audit.go + LogAudit("ACL_"+action, "ACL", fmt.Sprintf("%s.%s", c.dbName, c.name), map[string]interface{}{ + "role": role, + "permission": permission, + "granted": granted, + }) +} + +// ============================================================================= +// МЕТОДЫ ОГРАНИЧЕНИЙ (Constraints) +// ============================================================================= + +// AddRequiredField добавляет обязательное поле (lock-free) +func (c *Collection) AddRequiredField(field string) { + oldMap := c.constraints.loadRequiredFields() + oldVal := oldMap[field] + + newMap := make(map[string]bool) + for k, v := range oldMap { + newMap[k] = v + } + newMap[field] = true + + c.constraints.storeRequiredFields(newMap) + c.logConstraintChange("ADD", "required", field, oldVal, true) +} + +// AddUniqueConstraint добавляет ограничение уникальности (lock-free) +func (c *Collection) AddUniqueConstraint(field string) { + oldMap := c.constraints.loadUniqueFields() + oldVal := oldMap[field] + + newMap := make(map[string]bool) + for k, v := range oldMap { + newMap[k] = v + } + newMap[field] = true + + c.constraints.storeUniqueFields(newMap) + c.logConstraintChange("ADD", "unique", field, oldVal, true) + // Также создаём уникальный индекс + c.CreateIndex("unique_"+field, []string{field}, true) +} + +// AddMinConstraint добавляет минимальное значение (lock-free) +func (c *Collection) AddMinConstraint(field string, min float64) { + oldMap := c.constraints.loadMinValues() + oldVal := oldMap[field] + + newMap := make(map[string]float64) + for k, v := range oldMap { + newMap[k] = v + } + newMap[field] = min + + c.constraints.storeMinValues(newMap) + c.logConstraintChange("ADD", "min", field, oldVal, min) +} + +// AddMaxConstraint добавляет максимальное значение (lock-free) +func (c *Collection) AddMaxConstraint(field string, max float64) { + oldMap := c.constraints.loadMaxValues() + oldVal := oldMap[field] + + newMap := make(map[string]float64) + for k, v := range oldMap { + newMap[k] = v + } + newMap[field] = max + + c.constraints.storeMaxValues(newMap) + c.logConstraintChange("ADD", "max", field, oldVal, max) +} + +// AddRegexConstraint добавляет regexp паттерн (lock-free) +func (c *Collection) AddRegexConstraint(field string, pattern string) { + oldMap := c.constraints.loadPatternFields() + oldVal := oldMap[field] + + newMap := make(map[string]string) + for k, v := range oldMap { + newMap[k] = v + } + newMap[field] = pattern + + c.constraints.storePatternFields(newMap) + c.logConstraintChange("ADD", "regex", field, oldVal, pattern) +} + +// AddEnumConstraint добавляет допустимые значения (lock-free) +func (c *Collection) AddEnumConstraint(field string, values []interface{}) { + oldMap := c.constraints.loadEnumFields() + oldVal := oldMap[field] + + newMap := make(map[string][]interface{}) + for k, v := range oldMap { + // Копируем слайс + copiedVals := make([]interface{}, len(v)) + copy(copiedVals, v) + newMap[k] = copiedVals + } + // Копируем новые значения + copiedValues := make([]interface{}, len(values)) + copy(copiedValues, values) + newMap[field] = copiedValues + + c.constraints.storeEnumFields(newMap) + c.logConstraintChange("ADD", "enum", field, oldVal, values) +} + +// RemoveRequiredField удаляет обязательное поле (lock-free) +func (c *Collection) RemoveRequiredField(field string) { + oldMap := c.constraints.loadRequiredFields() + oldVal := oldMap[field] + + newMap := make(map[string]bool) + for k, v := range oldMap { + if k != field { + newMap[k] = v + } + } + + c.constraints.storeRequiredFields(newMap) + c.logConstraintChange("REMOVE", "required", field, oldVal, nil) +} + +// RemoveUniqueConstraint удаляет ограничение уникальности (lock-free) +func (c *Collection) RemoveUniqueConstraint(field string) { + oldMap := c.constraints.loadUniqueFields() + oldVal := oldMap[field] + + newMap := make(map[string]bool) + for k, v := range oldMap { + if k != field { + newMap[k] = v + } + } + + c.constraints.storeUniqueFields(newMap) + c.logConstraintChange("REMOVE", "unique", field, oldVal, nil) + // Также удаляем индекс + c.DropIndex("unique_" + field) +} + +// RemoveMinConstraint удаляет минимальное значение (lock-free) +func (c *Collection) RemoveMinConstraint(field string) { + oldMap := c.constraints.loadMinValues() + oldVal := oldMap[field] + + newMap := make(map[string]float64) + for k, v := range oldMap { + if k != field { + newMap[k] = v + } + } + + c.constraints.storeMinValues(newMap) + c.logConstraintChange("REMOVE", "min", field, oldVal, nil) +} + +// RemoveMaxConstraint удаляет максимальное значение (lock-free) +func (c *Collection) RemoveMaxConstraint(field string) { + oldMap := c.constraints.loadMaxValues() + oldVal := oldMap[field] + + newMap := make(map[string]float64) + for k, v := range oldMap { + if k != field { + newMap[k] = v + } + } + + c.constraints.storeMaxValues(newMap) + c.logConstraintChange("REMOVE", "max", field, oldVal, nil) +} + +// RemoveRegexConstraint удаляет regexp паттерн (lock-free) +func (c *Collection) RemoveRegexConstraint(field string) { + oldMap := c.constraints.loadPatternFields() + oldVal := oldMap[field] + + newMap := make(map[string]string) + for k, v := range oldMap { + if k != field { + newMap[k] = v + } + } + + c.constraints.storePatternFields(newMap) + c.logConstraintChange("REMOVE", "regex", field, oldVal, nil) +} + +// RemoveEnumConstraint удаляет допустимые значения (lock-free) +func (c *Collection) RemoveEnumConstraint(field string) { + oldMap := c.constraints.loadEnumFields() + oldVal := oldMap[field] + + newMap := make(map[string][]interface{}) + for k, v := range oldMap { + if k != field { + copiedVals := make([]interface{}, len(v)) + copy(copiedVals, v) + newMap[k] = copiedVals + } + } + + c.constraints.storeEnumFields(newMap) + c.logConstraintChange("REMOVE", "enum", field, oldVal, nil) +} + +// GetRequiredFields возвращает список обязательных полей (lock-free) +func (c *Collection) GetRequiredFields() []string { + fieldsMap := c.constraints.loadRequiredFields() + fields := make([]string, 0, len(fieldsMap)) + for field := range fieldsMap { + fields = append(fields, field) + } + return fields +} + +// GetUniqueConstraints возвращает список уникальных полей (lock-free) +func (c *Collection) GetUniqueConstraints() []string { + fieldsMap := c.constraints.loadUniqueFields() + fields := make([]string, 0, len(fieldsMap)) + for field := range fieldsMap { + fields = append(fields, field) + } + return fields +} + +// GetMinConstraints возвращает карту минимальных значений (lock-free) +func (c *Collection) GetMinConstraints() map[string]float64 { + result := c.constraints.loadMinValues() + // Возвращаем копию + resultCopy := make(map[string]float64) + for k, v := range result { + resultCopy[k] = v + } + return resultCopy +} + +// GetMaxConstraints возвращает карту максимальных значений (lock-free) +func (c *Collection) GetMaxConstraints() map[string]float64 { + result := c.constraints.loadMaxValues() + resultCopy := make(map[string]float64) + for k, v := range result { + resultCopy[k] = v + } + return resultCopy +} + +// GetEnumConstraints возвращает карту enum ограничений (lock-free) +func (c *Collection) GetEnumConstraints() map[string][]interface{} { + result := c.constraints.loadEnumFields() + resultCopy := make(map[string][]interface{}) + for k, v := range result { + copied := make([]interface{}, len(v)) + copy(copied, v) + resultCopy[k] = copied + } + return resultCopy +} + +// GetRegexConstraints возвращает карту regex паттернов (lock-free) +func (c *Collection) GetRegexConstraints() map[string]string { + result := c.constraints.loadPatternFields() + resultCopy := make(map[string]string) + for k, v := range result { + resultCopy[k] = v + } + return resultCopy +} + +// GetConstraints возвращает все ограничения коллекции (для API) +func (c *Collection) GetConstraints() map[string]interface{} { + return map[string]interface{}{ + "required_fields": c.GetRequiredFields(), + "unique_fields": c.GetUniqueConstraints(), + "min_values": c.GetMinConstraints(), + "max_values": c.GetMaxConstraints(), + "enum_values": c.GetEnumConstraints(), + "regex_patterns": c.GetRegexConstraints(), + } +} + +// GetConstraintTimestamps возвращает временные метки ограничений (lock-free) +func (c *Collection) GetConstraintTimestamps() map[string]interface{} { + return map[string]interface{}{ + "created_at": c.constraints.createdAt, + "updated_at": c.constraints.updatedAt.Load(), + "history_count": len(c.constraints.loadConstraintHistory()), + } +} + +// GetConstraintHistory возвращает историю изменений ограничений (lock-free) +func (c *Collection) GetConstraintHistory() []ConstraintChange { + history := c.constraints.loadConstraintHistory() + result := make([]ConstraintChange, len(history)) + copy(result, history) + return result +} + +// ValidateDocument проверяет документ на соответствие ограничениям (lock-free) +func (cons *Constraints) ValidateDocument(doc *Document) error { + // Проверка обязательных полей (lock-free чтение) + requiredFields := cons.loadRequiredFields() + for field := range requiredFields { + if !doc.HasField(field) { + return fmt.Errorf("required field '%s' is missing", field) + } + } + + // Проверка числовых ограничений (lock-free чтение) + minValues := cons.loadMinValues() + for field, minVal := range minValues { + if val, err := doc.GetField(field); err == nil { + if numVal, ok := toFloat64(val); ok { + if numVal < minVal { + return fmt.Errorf("field '%s' value %v is less than minimum %v", field, numVal, minVal) + } + } + } + } + + maxValues := cons.loadMaxValues() + for field, maxVal := range maxValues { + if val, err := doc.GetField(field); err == nil { + if numVal, ok := toFloat64(val); ok { + if numVal > maxVal { + return fmt.Errorf("field '%s' value %v exceeds maximum %v", field, numVal, maxVal) + } + } + } + } + + // Проверка regex паттернов (lock-free чтение) + patternFields := cons.loadPatternFields() + for field, pattern := range patternFields { + if val, err := doc.GetField(field); err == nil { + if strVal, ok := val.(string); ok { + if pattern != "" && !strings.Contains(strVal, pattern) { + return fmt.Errorf("field '%s' value '%s' does not match pattern '%s'", field, strVal, pattern) + } + } + } + } + + // Проверка enum (lock-free чтение) + enumFields := cons.loadEnumFields() + for field, allowedValues := range enumFields { + if val, err := doc.GetField(field); err == nil { + found := false + for _, allowed := range allowedValues { + if fmt.Sprintf("%v", val) == fmt.Sprintf("%v", allowed) { + found = true + break + } + } + if !found { + return fmt.Errorf("field '%s' value '%v' not in allowed list", field, val) + } + } + } + + return nil +} + +// ============================================================================= +// МЕТОДЫ ACL +// ============================================================================= + +// SetACL устанавливает ACL для коллекции (lock-free) +func (c *Collection) SetACL(role string, canRead, canWrite, canDelete, isAdmin bool) { + c.acl.updatedAt.Store(time.Now().UnixMilli()) + + if canRead { + oldRoles := c.acl.loadReadRoles() + if !oldRoles[role] { + newRoles := make(map[string]bool) + for k, v := range oldRoles { + newRoles[k] = v + } + newRoles[role] = true + c.acl.storeReadRoles(newRoles) + c.logACLChange("GRANT", role, "read", true) + } + } + if canWrite { + oldRoles := c.acl.loadWriteRoles() + if !oldRoles[role] { + newRoles := make(map[string]bool) + for k, v := range oldRoles { + newRoles[k] = v + } + newRoles[role] = true + c.acl.storeWriteRoles(newRoles) + c.logACLChange("GRANT", role, "write", true) + } + } + if canDelete { + oldRoles := c.acl.loadDeleteRoles() + if !oldRoles[role] { + newRoles := make(map[string]bool) + for k, v := range oldRoles { + newRoles[k] = v + } + newRoles[role] = true + c.acl.storeDeleteRoles(newRoles) + c.logACLChange("GRANT", role, "delete", true) + } + } + if isAdmin { + oldRoles := c.acl.loadAdminRoles() + if !oldRoles[role] { + newRoles := make(map[string]bool) + for k, v := range oldRoles { + newRoles[k] = v + } + newRoles[role] = true + c.acl.storeAdminRoles(newRoles) + c.logACLChange("GRANT", role, "admin", true) + } + } +} + +// RevokeACL отзывает разрешения у роли (lock-free) +func (c *Collection) RevokeACL(role string, permission string) { + c.acl.updatedAt.Store(time.Now().UnixMilli()) + + switch permission { + case "read": + oldRoles := c.acl.loadReadRoles() + if oldRoles[role] { + newRoles := make(map[string]bool) + for k, v := range oldRoles { + if k != role { + newRoles[k] = v + } + } + c.acl.storeReadRoles(newRoles) + c.logACLChange("REVOKE", role, "read", false) + } + case "write": + oldRoles := c.acl.loadWriteRoles() + if oldRoles[role] { + newRoles := make(map[string]bool) + for k, v := range oldRoles { + if k != role { + newRoles[k] = v + } + } + c.acl.storeWriteRoles(newRoles) + c.logACLChange("REVOKE", role, "write", false) + } + case "delete": + oldRoles := c.acl.loadDeleteRoles() + if oldRoles[role] { + newRoles := make(map[string]bool) + for k, v := range oldRoles { + if k != role { + newRoles[k] = v + } + } + c.acl.storeDeleteRoles(newRoles) + c.logACLChange("REVOKE", role, "delete", false) + } + case "admin": + oldRoles := c.acl.loadAdminRoles() + if oldRoles[role] { + newRoles := make(map[string]bool) + for k, v := range oldRoles { + if k != role { + newRoles[k] = v + } + } + c.acl.storeAdminRoles(newRoles) + c.logACLChange("REVOKE", role, "admin", false) + } + } +} + +// CheckPermission проверяет наличие разрешения у роли (lock-free) +func (c *Collection) CheckPermission(role, operation string) bool { + return c.acl.CheckPermission(role, operation) +} + +// GetACLTimestamps возвращает временные метки ACL (lock-free) +func (c *Collection) GetACLTimestamps() map[string]interface{} { + return map[string]interface{}{ + "created_at": c.acl.createdAt, + "updated_at": c.acl.updatedAt.Load(), + } +} + +// GetACLHistory возвращает историю изменений ACL (lock-free) +func (c *Collection) GetACLHistory() []ACLChange { + history := c.acl.loadACLHistory() + result := make([]ACLChange, len(history)) + copy(result, history) + return result +} + +// GetACLUpdatedAt возвращает время последнего обновления ACL (lock-free) +func (c *Collection) GetACLUpdatedAt() int64 { + return c.acl.updatedAt.Load() +} + +// GetACLCreatedAt возвращает время создания ACL (lock-free) +func (c *Collection) GetACLCreatedAt() int64 { + return c.acl.createdAt +} + +// toFloat64 конвертирует interface{} в float64 +func toFloat64(val interface{}) (float64, bool) { + switch v := val.(type) { + case int: + return float64(v), true + case int64: + return float64(v), true + case float64: + return v, true + case float32: + return float64(v), true + default: + return 0, false + } +} diff --git a/internal/storage/runtime_limits.go b/internal/storage/runtime_limits.go new file mode 100644 index 0000000..39d680d --- /dev/null +++ b/internal/storage/runtime_limits.go @@ -0,0 +1,446 @@ +/* + * Copyright 2026 Safronov Grigorii + * + * Licensed under the CDDL, Version 1.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at + * https://opensource.org/licenses/CDDL-1.0 + */ + +// Файл: internal/storage/runtime_limits.go +// Назначение: Ограничения на размер коллекции/документа в рантайме +// Добавлен механизм eviction при нехватке памяти (OOM protection) +// Eviction теперь проверяет активные транзакции перед удалением + +package storage + +import ( + "fmt" + "runtime" + "sync" + "sync/atomic" + "time" +) + +// LoggerInterface определяет интерфейс для логирования +type LoggerInterface interface { + Debug(msg string) + Info(msg string) + Error(msg string) + Warn(msg string) +} + +// EvictionPolicy определяет политику вытеснения +type EvictionPolicy int + +const ( + EvictionNone EvictionPolicy = iota + EvictionLRU + EvictionTTL + EvictionOldest +) + +// RuntimeLimitsManager управляет runtime-ограничениями +type RuntimeLimitsManager struct { + mu sync.RWMutex + globalMaxDocSize int64 + globalMaxCollSize int64 + globalMaxDocsPerColl int64 + globalMaxMemory int64 + collectionOverrides map[string]*CollectionLimits + metrics *LimitMetrics + logger LoggerInterface + enabled bool + evictionPolicy EvictionPolicy + memoryThreshold float64 + evictionChan chan string + stopChan chan struct{} + wg sync.WaitGroup +} + +// CollectionLimits содержит лимиты для конкретной коллекции +type CollectionLimits struct { + MaxDocSize int64 + MaxCollectionSize int64 + MaxDocuments int64 + EvictionPolicy EvictionPolicy + LastUpdated int64 +} + +// LimitMetrics хранит метрики ограничений +type LimitMetrics struct { + RejectedBySize atomic.Uint64 + RejectedByDocCount atomic.Uint64 + RejectedByCollSize atomic.Uint64 + RejectedByMemory atomic.Uint64 + EvictedDocuments atomic.Uint64 + EvictedBytes atomic.Uint64 + LastCheckTime atomic.Int64 + LastEvictionTime atomic.Int64 + SkippedEviction atomic.Uint64 // Пропущено из-за активных транзакций +} + +// RuntimeLimitsConfig содержит конфигурацию ограничений +type RuntimeLimitsConfig struct { + Enabled bool `json:"enabled"` + GlobalMaxDocSizeMB int `json:"global_max_doc_size_mb"` + GlobalMaxCollSizeMB int64 `json:"global_max_coll_size_mb"` + GlobalMaxDocsPerColl int64 `json:"global_max_docs_per_coll"` + GlobalMaxMemoryMB int64 `json:"global_max_memory_mb"` + EvictionPolicy EvictionPolicy `json:"eviction_policy"` + MemoryThreshold float64 `json:"memory_threshold"` +} + +// DefaultRuntimeLimitsConfig возвращает конфигурацию по умолчанию +func DefaultRuntimeLimitsConfig() *RuntimeLimitsConfig { + return &RuntimeLimitsConfig{ + Enabled: true, + GlobalMaxDocSizeMB: 16, + GlobalMaxCollSizeMB: 10240, + GlobalMaxDocsPerColl: 10000000, + GlobalMaxMemoryMB: 0, + EvictionPolicy: EvictionLRU, + MemoryThreshold: 0.85, + } +} + +// NewRuntimeLimitsManager создаёт новый менеджер ограничений +func NewRuntimeLimitsManager(cfg *RuntimeLimitsConfig, logger LoggerInterface) *RuntimeLimitsManager { + if cfg == nil { + cfg = DefaultRuntimeLimitsConfig() + } + + maxMemory := cfg.GlobalMaxMemoryMB * 1024 * 1024 + if maxMemory <= 0 { + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + maxMemory = int64(float64(memStats.Sys) * 0.8) + } + + rlm := &RuntimeLimitsManager{ + globalMaxDocSize: int64(cfg.GlobalMaxDocSizeMB) * 1024 * 1024, + globalMaxCollSize: cfg.GlobalMaxCollSizeMB * 1024 * 1024, + globalMaxDocsPerColl: cfg.GlobalMaxDocsPerColl, + globalMaxMemory: maxMemory, + collectionOverrides: make(map[string]*CollectionLimits), + metrics: &LimitMetrics{}, + logger: logger, + enabled: cfg.Enabled, + evictionPolicy: cfg.EvictionPolicy, + memoryThreshold: cfg.MemoryThreshold, + evictionChan: make(chan string, 100), + stopChan: make(chan struct{}), + } + + rlm.wg.Add(1) + go rlm.memoryMonitorLoop() + + if logger != nil { + logger.Debug(fmt.Sprintf("Runtime limits manager initialized: maxDoc=%dMB, maxColl=%dMB, maxDocs=%d, maxMemory=%dMB, eviction=%d", + cfg.GlobalMaxDocSizeMB, cfg.GlobalMaxCollSizeMB, cfg.GlobalMaxDocsPerColl, maxMemory/(1024*1024), cfg.EvictionPolicy)) + } + + return rlm +} + +func (rlm *RuntimeLimitsManager) memoryMonitorLoop() { + defer rlm.wg.Done() + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if !rlm.enabled { + continue + } + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + currentUsage := int64(memStats.Alloc) + usageRatio := float64(currentUsage) / float64(rlm.globalMaxMemory) + + if usageRatio > rlm.memoryThreshold { + if rlm.logger != nil { + rlm.logger.Warn(fmt.Sprintf("Memory usage %.2f%% exceeds threshold %.2f%%, triggering eviction", + usageRatio*100, rlm.memoryThreshold*100)) + } + rlm.triggerEviction() + } + case <-rlm.stopChan: + return + } + } +} + +func (rlm *RuntimeLimitsManager) triggerEviction() { + select { + case rlm.evictionChan <- "global": + default: + } +} + +// isDocInActiveTransaction проверяет, участвует ли документ в активной транзакции +// Защита от удаления данных активных транзакций +func (rlm *RuntimeLimitsManager) isDocInActiveTransaction(docID, dbName, collName string) bool { + if globalTxManager == nil { + return false + } + + inTx := false + globalTxManager.activeTransactions.Range(func(key, value interface{}) bool { + tx := value.(*Transaction) + tx.mu.RLock() + for _, op := range tx.Operations { + if op.DocumentID == docID && op.Database == dbName && op.Collection == collName { + inTx = true + break + } + } + tx.mu.RUnlock() + if inTx { + return false + } + return true + }) + return inTx +} + +// EvictFromCollection выполняет вытеснение документов из коллекции +// Проверка активных транзакций перед удалением +func (rlm *RuntimeLimitsManager) EvictFromCollection(coll *Collection, targetBytes int64) (int64, error) { + if !rlm.enabled { + return 0, nil + } + + evictedBytes := int64(0) + evictedCount := int64(0) + skippedCount := int64(0) + + docs := coll.GetAllDocumentsIncludingDeleted() + + // Сначала вытесняем удалённые документы + for _, doc := range docs { + if evictedBytes >= targetBytes { + break + } + + if doc.IsDeleted() { + // Проверяем активные транзакции + if rlm.isDocInActiveTransaction(doc.ID, coll.DBName(), coll.Name()) { + skippedCount++ + continue + } + + size := doc.OriginalSize + if size == 0 { + size = 1024 + } + + if err := coll.PermanentDelete(doc.ID); err == nil { + evictedBytes += size + evictedCount++ + } + } + } + + // Затем вытесняем самые старые + if evictedBytes < targetBytes && rlm.evictionPolicy == EvictionLRU { + for _, doc := range docs { + if evictedBytes >= targetBytes { + break + } + + if !doc.IsDeleted() { + // Проверяем активные транзакции + if rlm.isDocInActiveTransaction(doc.ID, coll.DBName(), coll.Name()) { + skippedCount++ + continue + } + + size := doc.OriginalSize + if size == 0 { + size = 1024 + } + + if coll.metadata.Settings.SoftDelete { + if err := coll.Delete(doc.ID); err == nil { + evictedBytes += size + evictedCount++ + } + } else { + if err := coll.PermanentDelete(doc.ID); err == nil { + evictedBytes += size + evictedCount++ + } + } + } + } + } + + rlm.metrics.EvictedDocuments.Add(uint64(evictedCount)) + rlm.metrics.EvictedBytes.Add(uint64(evictedBytes)) + if skippedCount > 0 { + rlm.metrics.SkippedEviction.Add(uint64(skippedCount)) + } + rlm.metrics.LastEvictionTime.Store(time.Now().UnixMilli()) + + if rlm.logger != nil { + rlm.logger.Info(fmt.Sprintf("Evicted %d documents (%d bytes) from collection %s.%s (skipped %d in active transactions)", + evictedCount, evictedBytes, coll.DBName(), coll.Name(), skippedCount)) + } + + return evictedBytes, nil +} + +func (rlm *RuntimeLimitsManager) ValidateDocumentSize(dbName, collName string, docSize int64) error { + if !rlm.enabled { + return nil + } + + limit := rlm.getCollectionLimit(dbName, collName) + maxSize := rlm.globalMaxDocSize + if limit != nil && limit.MaxDocSize > 0 { + maxSize = limit.MaxDocSize + } + + if docSize > maxSize { + rlm.metrics.RejectedBySize.Add(1) + return fmt.Errorf("document size %d bytes exceeds limit %d bytes", docSize, maxSize) + } + return nil +} + +func (rlm *RuntimeLimitsManager) ValidateCollectionSize(coll *Collection, newDocSize int64) error { + if !rlm.enabled { + return nil + } + + limit := rlm.getCollectionLimit(coll.DBName(), coll.Name()) + maxSize := rlm.globalMaxCollSize + if limit != nil && limit.MaxCollectionSize > 0 { + maxSize = limit.MaxCollectionSize + } + + currentSize := coll.Size() + if currentSize+newDocSize > maxSize { + neededBytes := currentSize + newDocSize - maxSize + if evicted, err := rlm.EvictFromCollection(coll, neededBytes); err == nil && evicted >= neededBytes { + return nil + } + + rlm.metrics.RejectedByCollSize.Add(1) + return fmt.Errorf("collection size would exceed limit %d bytes (current: %d, new: %d)", + maxSize, currentSize, newDocSize) + } + return nil +} + +func (rlm *RuntimeLimitsManager) ValidateDocumentCount(coll *Collection) error { + if !rlm.enabled { + return nil + } + + limit := rlm.getCollectionLimit(coll.DBName(), coll.Name()) + maxDocs := rlm.globalMaxDocsPerColl + if limit != nil && limit.MaxDocuments > 0 { + maxDocs = limit.MaxDocuments + } + + currentCount := coll.Count() + if currentCount >= maxDocs { + targetBytes := int64((currentCount - maxDocs + 1) * 1024) + if evicted, err := rlm.EvictFromCollection(coll, targetBytes); err == nil && evicted > 0 { + if coll.Count() < maxDocs { + return nil + } + } + + rlm.metrics.RejectedByDocCount.Add(1) + return fmt.Errorf("collection has reached maximum document count %d", maxDocs) + } + return nil +} + +func (rlm *RuntimeLimitsManager) CheckMemoryUsage() error { + if !rlm.enabled { + return nil + } + + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + + currentUsage := int64(memStats.Alloc) + if currentUsage > rlm.globalMaxMemory { + rlm.metrics.RejectedByMemory.Add(1) + return fmt.Errorf("memory usage %d bytes exceeds limit %d bytes", currentUsage, rlm.globalMaxMemory) + } + return nil +} + +func (rlm *RuntimeLimitsManager) getCollectionLimit(dbName, collName string) *CollectionLimits { + rlm.mu.RLock() + defer rlm.mu.RUnlock() + + key := fmt.Sprintf("%s.%s", dbName, collName) + if limits, ok := rlm.collectionOverrides[key]; ok { + return limits + } + return nil +} + +func (rlm *RuntimeLimitsManager) SetCollectionLimits(dbName, collName string, maxDocSizeMB int, maxCollSizeMB int64, maxDocuments int64) { + rlm.mu.Lock() + defer rlm.mu.Unlock() + + key := fmt.Sprintf("%s.%s", dbName, collName) + rlm.collectionOverrides[key] = &CollectionLimits{ + MaxDocSize: int64(maxDocSizeMB) * 1024 * 1024, + MaxCollectionSize: maxCollSizeMB * 1024 * 1024, + MaxDocuments: maxDocuments, + EvictionPolicy: rlm.evictionPolicy, + LastUpdated: time.Now().UnixMilli(), + } + + if rlm.logger != nil { + rlm.logger.Info(fmt.Sprintf("Set limits for %s: maxDoc=%dMB, maxColl=%dMB, maxDocs=%d", + key, maxDocSizeMB, maxCollSizeMB, maxDocuments)) + } +} + +func (rlm *RuntimeLimitsManager) RemoveCollectionLimits(dbName, collName string) { + rlm.mu.Lock() + defer rlm.mu.Unlock() + + key := fmt.Sprintf("%s.%s", dbName, collName) + delete(rlm.collectionOverrides, key) + + if rlm.logger != nil { + rlm.logger.Info(fmt.Sprintf("Removed limits override for %s", key)) + } +} + +func (rlm *RuntimeLimitsManager) GetMetrics() map[string]interface{} { + return map[string]interface{}{ + "rejected_by_size": rlm.metrics.RejectedBySize.Load(), + "rejected_by_doc_count": rlm.metrics.RejectedByDocCount.Load(), + "rejected_by_coll_size": rlm.metrics.RejectedByCollSize.Load(), + "rejected_by_memory": rlm.metrics.RejectedByMemory.Load(), + "evicted_documents": rlm.metrics.EvictedDocuments.Load(), + "evicted_bytes": rlm.metrics.EvictedBytes.Load(), + "skipped_eviction": rlm.metrics.SkippedEviction.Load(), + "last_eviction_time": rlm.metrics.LastEvictionTime.Load(), + "global_max_doc_size_mb": rlm.globalMaxDocSize / (1024 * 1024), + "global_max_coll_size_mb": rlm.globalMaxCollSize / (1024 * 1024), + "global_max_docs_per_coll": rlm.globalMaxDocsPerColl, + "global_max_memory_mb": rlm.globalMaxMemory / (1024 * 1024), + "eviction_policy": rlm.evictionPolicy, + "memory_threshold": rlm.memoryThreshold, + "enabled": rlm.enabled, + } +} + +func (rlm *RuntimeLimitsManager) Stop() { + close(rlm.stopChan) + rlm.wg.Wait() +} diff --git a/internal/storage/transactions.go b/internal/storage/transactions.go new file mode 100644 index 0000000..3b7bdd7 --- /dev/null +++ b/internal/storage/transactions.go @@ -0,0 +1,4121 @@ +/* + * Copyright 2026 Safronov Grigorii + * + * Licensed under the CDDL, Version 1.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at + * https://opensource.org/licenses/CDDL-1.0 + */ + +// Файл: internal/storage/transactions.go +// Назначение: Реализация транзакций с поддержкой MVCC и WAL без блокировок. +// Для распределённых транзакций используется протокол SAGA. +// ИСПРАВЛЕНО: Deadlock в flushBatch/rotateSegment (wm.mu рекурсивно) +// ИСПРАВЛЕНО: Race между async Write и Sync - добавлен механизм подтверждения +// ИСПРАВЛЕНО: Apply до WAL - теперь сначала WAL, потом применение +// ИСПРАВЛЕНО: Stop теряет записи буфера - flush перед закрытием + +package storage + +import ( + "bufio" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "sync" + "sync/atomic" + "time" + + "futriis/internal/config" +) + +// ============================================================================= +// БАЗОВЫЕ ТИПЫ +// ============================================================================= + +type TransactionID uint64 + +type TransactionState int32 + +const ( + TransactionActive TransactionState = iota + TransactionCommitted + TransactionAborted + TransactionPrepared +) + +type TransactionRecord struct { + ID TransactionID `json:"id"` + State TransactionState `json:"state"` + Timestamp int64 `json:"timestamp"` + Operations []Operation `json:"operations"` + IsDistributed bool `json:"is_distributed"` + Nodes []string `json:"nodes,omitempty"` +} + +type WALRecord struct { + CRC uint32 `json:"crc"` + Length uint32 `json:"length"` + Type byte `json:"type"` + Data []byte `json:"data"` + Timestamp int64 `json:"timestamp"` + LSN uint64 `json:"lsn"` +} + +// ============================================================================= +// КОНСТАНТЫ +// ============================================================================= + +const ( + WALSegmentSize = 64 * 1024 * 1024 + WALSegmentPrefix = "wal_segment_" + WALIndexPrefix = "wal_index_" + + VersionPruneInterval = 5 * time.Minute + + DefaultTxTimeout = 30 * time.Second + DeadlockCheckInterval = 1 * time.Second + MaxSavepointsPerTx = 100 + + AsyncRecoveryBufferSize = 10000 + AsyncRecoveryWorkers = 4 + AsyncRecoveryTimeout = 30 * time.Second + FsyncMaxRetries = 3 + FsyncRetryDelay = 100 * time.Millisecond + + SagaStateDir = "saga_states" + SagaStateFilePrefix = "saga_state_" + SagaStateFileSuffix = ".json" + SagaCheckpointInterval = 30 * time.Second + SagaMaxRetries = 5 + SagaRetryBackoff = 100 * time.Millisecond + + SagaOrphanTimeout = 5 * time.Minute + SagaCleanupInterval = 1 * time.Minute +) + +// ============================================================================= +// CRC32 +// ============================================================================= + +var crc32Table = [256]uint32{ + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, + 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, + 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, + 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, + 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, + 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, + 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, + 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1a4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, + 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, + 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, + 0x40df0b66, 0x37d83bf8, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, + 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, +} + +func crc32(data []byte) uint32 { + crc := uint32(0xFFFFFFFF) + for _, b := range data { + crc = (crc >> 8) ^ crc32Table[(crc^uint32(b))&0xFF] + } + return crc ^ 0xFFFFFFFF +} + +// ============================================================================= +// АУДИТ ТРАНЗАКЦИЙ +// ============================================================================= + +type TransactionAuditEntry struct { + TxID TransactionID `json:"tx_id"` + Action string `json:"action"` + State TransactionState `json:"state"` + Timestamp int64 `json:"timestamp"` + TimestampStr string `json:"timestamp_str"` + Details map[string]interface{} `json:"details"` +} + +type TransactionAuditLogger struct { + entries []TransactionAuditEntry + mu sync.RWMutex + maxSize int + filePath string + fileMu sync.Mutex + enabled bool +} + +var globalTxAuditLogger = &TransactionAuditLogger{ + entries: make([]TransactionAuditEntry, 0), + maxSize: 100000, + enabled: true, +} + +func InitTransactionAuditLogger(filePath string) error { + globalTxAuditLogger.filePath = filePath + if filePath != "" { + if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil { + return err + } + globalTxAuditLogger.loadFromFile() + } + return nil +} + +func (tal *TransactionAuditLogger) loadFromFile() { + if tal.filePath == "" { + return + } + data, err := os.ReadFile(tal.filePath) + if err != nil { + return + } + var entries []TransactionAuditEntry + if err := json.Unmarshal(data, &entries); err != nil { + return + } + tal.mu.Lock() + defer tal.mu.Unlock() + tal.entries = entries +} + +func (tal *TransactionAuditLogger) saveToFile() { + if tal.filePath == "" { + return + } + tal.fileMu.Lock() + defer tal.fileMu.Unlock() + + tal.mu.RLock() + data, err := json.MarshalIndent(tal.entries, "", " ") + tal.mu.RUnlock() + if err != nil { + return + } + os.WriteFile(tal.filePath, data, 0644) +} + +func LogTransactionAudit(txID TransactionID, action string, state TransactionState, details map[string]interface{}) { + if !globalTxAuditLogger.enabled { + return + } + now := time.Now() + entry := TransactionAuditEntry{ + TxID: txID, + Action: action, + State: state, + Timestamp: now.UnixMilli(), + TimestampStr: now.Format("2006-01-02 15:04:05.000"), + Details: details, + } + globalTxAuditLogger.mu.Lock() + globalTxAuditLogger.entries = append(globalTxAuditLogger.entries, entry) + if len(globalTxAuditLogger.entries) > globalTxAuditLogger.maxSize { + globalTxAuditLogger.entries = globalTxAuditLogger.entries[len(globalTxAuditLogger.entries)-globalTxAuditLogger.maxSize:] + } + globalTxAuditLogger.mu.Unlock() + go globalTxAuditLogger.saveToFile() +} + +func GetTransactionAuditLog() []TransactionAuditEntry { + globalTxAuditLogger.mu.RLock() + defer globalTxAuditLogger.mu.RUnlock() + result := make([]TransactionAuditEntry, len(globalTxAuditLogger.entries)) + copy(result, globalTxAuditLogger.entries) + return result +} + +// ============================================================================= +// MVCC MANAGER +// ============================================================================= + +type MVCCManager struct { + versions sync.Map + visibilityMap *VisibilityMap + readCache *ReadTimestampCache + maxVersionsPerDoc int + retentionDuration time.Duration + stats MVCCStats + mu sync.RWMutex + activeReaders sync.Map + readersMu sync.RWMutex + oldestReadTx atomic.Int64 +} + +type MVCCStats struct { + TotalVersionsCreated atomic.Uint64 + TotalVersionsPruned atomic.Uint64 + TotalCacheHits atomic.Uint64 + TotalCacheMisses atomic.Uint64 + TotalVisibilityHits atomic.Uint64 + TotalVisibilityMisses atomic.Uint64 + TotalSkippedPrune atomic.Uint64 +} + +func NewMVCCManager(maxVersionsPerDoc int, retentionDays int) *MVCCManager { + if maxVersionsPerDoc <= 0 { + maxVersionsPerDoc = MaxVersionsPerDoc + } + if retentionDays <= 0 { + retentionDays = VersionRetentionDays + } + m := &MVCCManager{ + maxVersionsPerDoc: maxVersionsPerDoc, + retentionDuration: time.Duration(retentionDays) * 24 * time.Hour, + visibilityMap: NewVisibilityMap(VisibilityMapSize), + readCache: NewReadTimestampCache(10000, 5*time.Minute), + } + m.oldestReadTx.Store(0) + go m.pruneOldVersionsLoop() + return m +} + +func (m *MVCCManager) RegisterReader(txID uint64) { + m.readersMu.Lock() + defer m.readersMu.Unlock() + m.activeReaders.Store(txID, time.Now().UnixMilli()) + m.updateOldestReader() +} + +func (m *MVCCManager) UnregisterReader(txID uint64) { + m.readersMu.Lock() + defer m.readersMu.Unlock() + m.activeReaders.Delete(txID) + m.updateOldestReader() +} + +func (m *MVCCManager) updateOldestReader() { + oldest := int64(^uint64(0) >> 1) + m.activeReaders.Range(func(key, value interface{}) bool { + if ts, ok := value.(int64); ok && ts < oldest { + oldest = ts + } + return true + }) + m.oldestReadTx.Store(oldest) +} + +func (m *MVCCManager) CreateVersion(doc *Document, txID TransactionID) *DocumentVersion { + version := &DocumentVersion{ + Document: doc.Clone(), + Timestamp: time.Now().UnixMilli(), + TxID: txID, + VersionID: fmt.Sprintf("%s_%d_%d", doc.ID, txID, time.Now().UnixNano()), + } + m.mu.Lock() + defer m.mu.Unlock() + + val, _ := m.versions.LoadOrStore(doc.ID, make([]*DocumentVersion, 0)) + versions := val.([]*DocumentVersion) + versions = append(versions, version) + + if len(versions) > m.maxVersionsPerDoc { + oldestReader := m.oldestReadTx.Load() + newVersions := make([]*DocumentVersion, 0, m.maxVersionsPerDoc) + startIdx := len(versions) - m.maxVersionsPerDoc + if startIdx < 0 { + startIdx = 0 + } + for i := startIdx; i < len(versions); i++ { + newVersions = append(newVersions, versions[i]) + } + if oldestReader > 0 { + for i := 0; i < startIdx; i++ { + if versions[i].Timestamp >= oldestReader { + newVersions = append([]*DocumentVersion{versions[i]}, newVersions...) + } + } + } + prunedCount := len(versions) - len(newVersions) + versions = newVersions + m.stats.TotalVersionsPruned.Add(uint64(prunedCount)) + } + m.versions.Store(doc.ID, versions) + m.visibilityMap.MarkVisible(doc.ID, uint64(txID), true) + m.stats.TotalVersionsCreated.Add(1) + + LogTransactionAudit(txID, "VERSION_CREATE", TransactionActive, map[string]interface{}{ + "doc_id": doc.ID, + "version": version.VersionID, + }) + return version +} + +func (m *MVCCManager) GetVersionAt(docID string, timestamp int64) *Document { + if cached := m.readCache.Get(docID, timestamp); cached != nil { + m.stats.TotalCacheHits.Add(1) + return cached + } + m.stats.TotalCacheMisses.Add(1) + + m.mu.RLock() + defer m.mu.RUnlock() + + val, ok := m.versions.Load(docID) + if !ok { + return nil + } + versions := val.([]*DocumentVersion) + + var result *DocumentVersion + for i := len(versions) - 1; i >= 0; i-- { + v := versions[i] + if v.Timestamp <= timestamp && m.visibilityMap.IsVisible(docID, uint64(v.TxID)) { + result = v + break + } + } + if result == nil { + return nil + } + doc := result.Document.Clone() + m.readCache.Set(docID, timestamp, doc) + return doc +} + +func (m *MVCCManager) GetLatestVersion(docID string) *Document { + m.mu.RLock() + defer m.mu.RUnlock() + val, ok := m.versions.Load(docID) + if !ok { + return nil + } + versions := val.([]*DocumentVersion) + if len(versions) == 0 { + return nil + } + return versions[len(versions)-1].Document.Clone() +} + +func (m *MVCCManager) GetAllVersions(docID string) []*DocumentVersion { + m.mu.RLock() + defer m.mu.RUnlock() + val, ok := m.versions.Load(docID) + if !ok { + return nil + } + versions := val.([]*DocumentVersion) + result := make([]*DocumentVersion, len(versions)) + for i, v := range versions { + result[i] = &DocumentVersion{ + Document: v.Document.Clone(), + Timestamp: v.Timestamp, + TxID: v.TxID, + VersionID: v.VersionID, + } + } + return result +} + +func (m *MVCCManager) pruneOldVersionsLoop() { + ticker := time.NewTicker(VersionPruneInterval) + defer ticker.Stop() + for range ticker.C { + m.PruneOldVersions() + } +} + +func (m *MVCCManager) PruneOldVersions() { + cutoffTime := time.Now().Add(-m.retentionDuration).UnixMilli() + oldestReader := m.oldestReadTx.Load() + pruned := int64(0) + + m.mu.Lock() + defer m.mu.Unlock() + m.versions.Range(func(key, value interface{}) bool { + docID := key.(string) + versions := value.([]*DocumentVersion) + newVersions := make([]*DocumentVersion, 0, len(versions)) + for _, v := range versions { + if v.Timestamp >= cutoffTime || (oldestReader > 0 && v.Timestamp >= oldestReader) { + newVersions = append(newVersions, v) + } else { + pruned++ + } + } + if len(newVersions) == 0 && len(versions) > 0 { + newVersions = append(newVersions, versions[len(versions)-1]) + pruned-- + } + if len(newVersions) != len(versions) { + m.versions.Store(docID, newVersions) + } + return true + }) + if pruned > 0 { + m.stats.TotalVersionsPruned.Add(uint64(pruned)) + } +} + +func (m *MVCCManager) GetMVCCStats() map[string]interface{} { + return map[string]interface{}{ + "total_versions_created": m.stats.TotalVersionsCreated.Load(), + "total_versions_pruned": m.stats.TotalVersionsPruned.Load(), + "total_cache_hits": m.stats.TotalCacheHits.Load(), + "total_cache_misses": m.stats.TotalCacheMisses.Load(), + "total_visibility_hits": m.stats.TotalVisibilityHits.Load(), + "total_visibility_misses": m.stats.TotalVisibilityMisses.Load(), + "total_skipped_prune": m.stats.TotalSkippedPrune.Load(), + "max_versions_per_doc": m.maxVersionsPerDoc, + "retention_days": int(m.retentionDuration.Hours() / 24), + "oldest_read_tx": m.oldestReadTx.Load(), + } +} + +// ============================================================================= +// VISIBILITY MAP +// ============================================================================= + +type VisibilityMapEntry struct { + DocID string + VisibleFrom uint64 + VisibleTo uint64 + IsVisible bool + LastAccess int64 +} + +type VisibilityMap struct { + entries sync.Map + maxSize int + hitCount atomic.Uint64 + missCount atomic.Uint64 + mu sync.RWMutex +} + +func NewVisibilityMap(maxSize int) *VisibilityMap { + if maxSize <= 0 { + maxSize = VisibilityMapSize + } + return &VisibilityMap{maxSize: maxSize} +} + +func (vm *VisibilityMap) MarkVisible(docID string, version uint64, visible bool) { + key := fmt.Sprintf("%s@%d", docID, version) + vm.entries.Store(key, &VisibilityMapEntry{ + DocID: docID, + VisibleFrom: version, + VisibleTo: version, + IsVisible: visible, + LastAccess: time.Now().Unix(), + }) +} + +func (vm *VisibilityMap) IsVisible(docID string, version uint64) bool { + key := fmt.Sprintf("%s@%d", docID, version) + if val, ok := vm.entries.Load(key); ok { + entry := val.(*VisibilityMapEntry) + entry.LastAccess = time.Now().Unix() + vm.hitCount.Add(1) + return entry.IsVisible + } + var found bool + vm.entries.Range(func(k, v interface{}) bool { + entry := v.(*VisibilityMapEntry) + if entry.DocID == docID && version >= entry.VisibleFrom && version <= entry.VisibleTo && entry.IsVisible { + found = true + return false + } + return true + }) + if found { + vm.hitCount.Add(1) + return true + } + vm.missCount.Add(1) + return false +} + +func (vm *VisibilityMap) GetStats() map[string]interface{} { + count := 0 + vm.entries.Range(func(_, _ interface{}) bool { + count++ + return true + }) + return map[string]interface{}{ + "hits": vm.hitCount.Load(), + "misses": vm.missCount.Load(), + "entries": count, + "max_size": vm.maxSize, + } +} + +// ============================================================================= +// READ TIMESTAMP CACHE +// ============================================================================= + +type ReadTimestampCache struct { + cache sync.Map + maxSize int + ttl time.Duration + hits atomic.Uint64 + misses atomic.Uint64 + size atomic.Int64 + mu sync.RWMutex +} + +type cachedEntry struct { + doc *Document + cachedAt time.Time + accessCount int64 +} + +func NewReadTimestampCache(maxSize int, ttl time.Duration) *ReadTimestampCache { + if maxSize <= 0 { + maxSize = 10000 + } + if ttl <= 0 { + ttl = 5 * time.Minute + } + c := &ReadTimestampCache{maxSize: maxSize, ttl: ttl} + go c.cleanupLoop() + return c +} + +func (rtc *ReadTimestampCache) Get(docID string, timestamp int64) *Document { + key := fmt.Sprintf("%s@%d", docID, timestamp) + val, ok := rtc.cache.Load(key) + if !ok { + rtc.misses.Add(1) + return nil + } + entry := val.(*cachedEntry) + if time.Since(entry.cachedAt) > rtc.ttl { + rtc.cache.Delete(key) + rtc.size.Add(-1) + rtc.misses.Add(1) + return nil + } + entry.accessCount++ + rtc.hits.Add(1) + return entry.doc +} + +func (rtc *ReadTimestampCache) Set(docID string, timestamp int64, doc *Document) { + key := fmt.Sprintf("%s@%d", docID, timestamp) + if rtc.size.Load() >= int64(rtc.maxSize) { + rtc.evictOldest() + } + rtc.cache.Store(key, &cachedEntry{doc: doc, cachedAt: time.Now(), accessCount: 0}) + rtc.size.Add(1) +} + +func (rtc *ReadTimestampCache) evictOldest() { + var oldestKey interface{} + var oldestTime time.Time + rtc.cache.Range(func(key, value interface{}) bool { + entry := value.(*cachedEntry) + if oldestKey == nil || entry.cachedAt.Before(oldestTime) { + oldestKey = key + oldestTime = entry.cachedAt + } + return true + }) + if oldestKey != nil { + rtc.cache.Delete(oldestKey) + rtc.size.Add(-1) + } +} + +func (rtc *ReadTimestampCache) cleanupLoop() { + ticker := time.NewTicker(rtc.ttl) + defer ticker.Stop() + for range ticker.C { + rtc.cache.Range(func(key, value interface{}) bool { + entry := value.(*cachedEntry) + if time.Since(entry.cachedAt) > rtc.ttl { + rtc.cache.Delete(key) + rtc.size.Add(-1) + } + return true + }) + } +} + +func (rtc *ReadTimestampCache) GetStats() map[string]interface{} { + return map[string]interface{}{ + "hits": rtc.hits.Load(), + "misses": rtc.misses.Load(), + "size": rtc.size.Load(), + "max_size": rtc.maxSize, + "ttl_secs": int(rtc.ttl.Seconds()), + } +} + +// ============================================================================= +// SAGA PERSISTENT STORAGE +// ============================================================================= + +type SagaState struct { + ID string `json:"id"` + Status string `json:"status"` + CurrentStep int `json:"current_step"` + Steps []SagaStepState `json:"steps"` + Data map[string]interface{} `json:"data"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` + CompletedAt int64 `json:"completed_at,omitempty"` + CompensationExecuted bool `json:"compensation_executed"` + NodeID string `json:"node_id"` + CoordinatorID string `json:"coordinator_id"` + Version uint64 `json:"version"` + RetryCount int `json:"retry_count"` + LastError string `json:"last_error,omitempty"` + ExecutionID string `json:"execution_id"` +} + +type SagaStepState struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Data map[string]interface{} `json:"data"` + StartedAt int64 `json:"started_at"` + CompletedAt int64 `json:"completed_at"` + ExecutionID string `json:"execution_id"` + RetryCount int `json:"retry_count"` + LastError string `json:"last_error,omitempty"` + CompensatedAt int64 `json:"compensated_at,omitempty"` +} + +type SagaPersistentStorage struct { + baseDir string + mu sync.RWMutex + cache map[string]*SagaState + maxCache int + logger LoggerInterface + fsyncEnabled bool + fsyncMaxRetries int + fsyncRetryDelay time.Duration +} + +func NewSagaPersistentStorage(baseDir string, logger LoggerInterface) (*SagaPersistentStorage, error) { + return NewSagaPersistentStorageWithConfig(nil, logger) +} + +func NewSagaPersistentStorageWithConfig(cfg *config.SagaConfig, logger LoggerInterface) (*SagaPersistentStorage, error) { + baseDir := SagaStateDir + maxCache := 10000 + fsyncEnabled := true + fsyncMaxRetries := 3 + fsyncRetryDelay := 100 * time.Millisecond + + if cfg != nil { + baseDir = cfg.GetStateDir() + maxCache = cfg.GetMaxCacheSize() + fsyncEnabled = cfg.IsFsyncEnabled() + fsyncMaxRetries = cfg.GetFsyncMaxRetries() + fsyncRetryDelay = cfg.GetFsyncRetryDelay() + } + + fullPath := filepath.Join(baseDir) + if err := os.MkdirAll(fullPath, 0755); err != nil { + return nil, fmt.Errorf("failed to create saga state directory: %v", err) + } + + return &SagaPersistentStorage{ + baseDir: fullPath, + cache: make(map[string]*SagaState), + maxCache: maxCache, + logger: logger, + fsyncEnabled: fsyncEnabled, + fsyncMaxRetries: fsyncMaxRetries, + fsyncRetryDelay: fsyncRetryDelay, + }, nil +} + +func (sps *SagaPersistentStorage) getStatePath(sagaID string) string { + return filepath.Join(sps.baseDir, fmt.Sprintf("%s%s%s", SagaStateFilePrefix, sagaID, SagaStateFileSuffix)) +} + +func (sps *SagaPersistentStorage) Save(state *SagaState) error { + sps.mu.Lock() + defer sps.mu.Unlock() + + state.UpdatedAt = time.Now().UnixMilli() + state.Version++ + + if len(sps.cache) >= sps.maxCache { + var oldestKey string + var oldestTime int64 = time.Now().UnixMilli() + for k, v := range sps.cache { + if v.UpdatedAt < oldestTime { + oldestTime = v.UpdatedAt + oldestKey = k + } + } + if oldestKey != "" { + delete(sps.cache, oldestKey) + } + } + sps.cache[state.ID] = state + + path := sps.getStatePath(state.ID) + data, err := json.MarshalIndent(state, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal saga state: %v", err) + } + + tmpPath := path + ".tmp" + if err := os.WriteFile(tmpPath, data, 0644); err != nil { + return fmt.Errorf("failed to write saga state: %v", err) + } + + if sps.fsyncEnabled { + if f, err := os.OpenFile(tmpPath, os.O_RDWR, 0644); err == nil { + for i := 0; i < sps.fsyncMaxRetries; i++ { + if err := RealFsyncWithRetry(f, sps.fsyncMaxRetries, sps.fsyncRetryDelay); err == nil { + break + } + if i < sps.fsyncMaxRetries-1 { + time.Sleep(sps.fsyncRetryDelay) + } + } + f.Close() + } + } + + if err := os.Rename(tmpPath, path); err != nil { + return fmt.Errorf("failed to rename saga state: %v", err) + } + + if sps.fsyncEnabled { + FsyncDir(sps.baseDir) + } + + if sps.logger != nil { + sps.logger.Debug(fmt.Sprintf("Saved saga state %s (version %d, status %s)", state.ID, state.Version, state.Status)) + } + return nil +} + +func (sps *SagaPersistentStorage) Load(sagaID string) (*SagaState, error) { + sps.mu.RLock() + if state, ok := sps.cache[sagaID]; ok { + sps.mu.RUnlock() + return state, nil + } + sps.mu.RUnlock() + + path := sps.getStatePath(sagaID) + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("failed to read saga state: %v", err) + } + + var state SagaState + if err := json.Unmarshal(data, &state); err != nil { + return nil, fmt.Errorf("failed to unmarshal saga state: %v", err) + } + + sps.mu.Lock() + sps.cache[sagaID] = &state + sps.mu.Unlock() + return &state, nil +} + +func (sps *SagaPersistentStorage) Delete(sagaID string) error { + sps.mu.Lock() + delete(sps.cache, sagaID) + sps.mu.Unlock() + + path := sps.getStatePath(sagaID) + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to delete saga state: %v", err) + } + return nil +} + +func (sps *SagaPersistentStorage) ListAll() ([]*SagaState, error) { + pattern := filepath.Join(sps.baseDir, fmt.Sprintf("%s*%s", SagaStateFilePrefix, SagaStateFileSuffix)) + files, err := filepath.Glob(pattern) + if err != nil { + return nil, err + } + states := make([]*SagaState, 0, len(files)) + for _, file := range files { + data, err := os.ReadFile(file) + if err != nil { + continue + } + var state SagaState + if err := json.Unmarshal(data, &state); err != nil { + continue + } + states = append(states, &state) + } + return states, nil +} + +func (sps *SagaPersistentStorage) ListPending() ([]*SagaState, error) { + all, err := sps.ListAll() + if err != nil { + return nil, err + } + pending := make([]*SagaState, 0) + for _, state := range all { + if state.Status == "pending" || state.Status == "running" || state.Status == "compensating" { + pending = append(pending, state) + } + } + return pending, nil +} + +// ============================================================================= +// SAGA ORCHESTRATOR +// ============================================================================= + +type SagaOrchestrator struct { + storage *SagaPersistentStorage + coordinators []*SagaCoordinator + mu sync.RWMutex + logger LoggerInterface + nodeID string + stopChan chan struct{} + wg sync.WaitGroup + isLeader atomic.Bool + leaderID string + electionMu sync.Mutex + config *config.SagaConfig + metrics *SagaMetrics +} + +type SagaCoordinator struct { + id string + orchestrator *SagaOrchestrator + activeSagas sync.Map + stopChan chan struct{} + wg sync.WaitGroup + isActive bool + config *config.SagaConfig + mu sync.RWMutex +} + +type SagaMetrics struct { + TotalStarted atomic.Uint64 + TotalCompleted atomic.Uint64 + TotalAborted atomic.Uint64 + TotalFailed atomic.Uint64 + TotalCompensated atomic.Uint64 + ActiveCount atomic.Int64 + AvgDurationMs atomic.Uint64 + TotalDurationMs atomic.Uint64 + RecoveryCount atomic.Uint64 + mu sync.RWMutex + latencies []int64 + maxLatency int64 + minLatency int64 +} + +func NewSagaOrchestrator(baseDir string, nodeID string, logger LoggerInterface) (*SagaOrchestrator, error) { + return NewSagaOrchestratorWithConfig(nil, nodeID, logger) +} + +func NewSagaOrchestratorWithConfig(cfg *config.SagaConfig, nodeID string, logger LoggerInterface) (*SagaOrchestrator, error) { + if cfg == nil { + cfg = &config.SagaConfig{ + Enabled: true, + CoordinatorCount: 3, + StateDir: "saga_states", + MaxRetries: 5, + RetryBackoffMs: 100, + SagaTimeoutSec: 300, + StuckCheckIntervalSec: 10, + LeaderElectionIntervalSec: 5, + RecoveryIntervalSec: 30, + MetricsIntervalSec: 60, + CleanupPeriodHours: 24, + MaxCacheSize: 10000, + OperationRetentionDays: 7, + ChannelBufferSize: 10000, + AsyncRecoveryWorkers: 4, + AsyncRecoveryTimeoutSec: 30, + FsyncEnabled: true, + FsyncMaxRetries: 3, + FsyncRetryDelayMs: 100, + } + } + + storage, err := NewSagaPersistentStorageWithConfig(cfg, logger) + if err != nil { + return nil, err + } + + o := &SagaOrchestrator{ + storage: storage, + logger: logger, + nodeID: nodeID, + stopChan: make(chan struct{}, cfg.GetChannelBufferSize()), + metrics: &SagaMetrics{minLatency: -1}, + coordinators: make([]*SagaCoordinator, 0), + config: cfg, + } + + coordinatorCount := cfg.GetCoordinatorCount() + for i := 0; i < coordinatorCount; i++ { + coord := &SagaCoordinator{ + id: fmt.Sprintf("%s-coord-%d", nodeID, i), + orchestrator: o, + stopChan: make(chan struct{}, cfg.GetChannelBufferSize()), + isActive: true, + config: cfg, + } + o.coordinators = append(o.coordinators, coord) + coord.wg.Add(1) + go coord.run() + } + + o.wg.Add(1) + go o.leaderElectionLoop() + o.wg.Add(1) + go o.recoveryLoop() + o.wg.Add(1) + go o.metricsLoop() + o.wg.Add(1) + go o.orphanCleanupLoop() + + if cfg.IsSagaEnabled() && logger != nil { + logger.Info(fmt.Sprintf("Saga orchestrator initialized on node %s with %d coordinators", nodeID, coordinatorCount)) + } + return o, nil +} + +func (o *SagaOrchestrator) leaderElectionLoop() { + defer o.wg.Done() + ticker := time.NewTicker(o.config.GetLeaderElectionInterval()) + defer ticker.Stop() + for { + select { + case <-ticker.C: + o.electLeader() + case <-o.stopChan: + return + } + } +} + +func (o *SagaOrchestrator) electLeader() { + o.electionMu.Lock() + defer o.electionMu.Unlock() + candidates := make([]string, 0) + o.mu.RLock() + for _, coord := range o.coordinators { + if coord.isActive { + candidates = append(candidates, coord.id) + } + } + o.mu.RUnlock() + if len(candidates) == 0 { + o.isLeader.Store(false) + o.leaderID = "" + return + } + sort.Strings(candidates) + leader := candidates[0] + o.leaderID = leader + isLeader := leader == o.coordinators[0].id + o.isLeader.Store(isLeader) +} + +func (o *SagaOrchestrator) IsLeader() bool { return o.isLeader.Load() } +func (o *SagaOrchestrator) GetLeaderID() string { return o.leaderID } + +func (o *SagaOrchestrator) recoveryLoop() { + defer o.wg.Done() + ticker := time.NewTicker(o.config.GetRecoveryInterval()) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if o.IsLeader() { + o.recoverPendingSagas() + } + case <-o.stopChan: + return + } + } +} + +func (o *SagaOrchestrator) orphanCleanupLoop() { + defer o.wg.Done() + ticker := time.NewTicker(SagaCleanupInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if o.IsLeader() { + o.cleanupOrphanSteps() + } + case <-o.stopChan: + return + } + } +} + +func (o *SagaOrchestrator) cleanupOrphanSteps() { + pending, err := o.storage.ListPending() + if err != nil { + if o.logger != nil { + o.logger.Error(fmt.Sprintf("Failed to list pending sagas for cleanup: %v", err)) + } + return + } + now := time.Now().UnixMilli() + for _, state := range pending { + if state.Status == "running" || state.Status == "compensating" { + elapsed := now - state.UpdatedAt + if elapsed > SagaOrphanTimeout.Milliseconds() { + if o.logger != nil { + o.logger.Warn(fmt.Sprintf("Found orphan saga %s (last updated %d ms ago), cleaning up", state.ID, elapsed)) + } + state.Status = "orphaned" + state.LastError = fmt.Sprintf("orphaned after %d ms of inactivity", elapsed) + o.storage.Save(state) + o.compensateOrphanSaga(state) + } + } + } +} + +func (o *SagaOrchestrator) compensateOrphanSaga(state *SagaState) { + if state.CompensationExecuted { + return + } + if o.logger != nil { + o.logger.Info(fmt.Sprintf("Compensating orphan saga %s", state.ID)) + } + saga := &SagaTransaction{ + ID: state.ID, + Status: state.Status, + CurrentStep: state.CurrentStep, + Data: state.Data, + CreatedAt: state.CreatedAt, + UpdatedAt: state.UpdatedAt, + executedOps: make(map[string]bool), + } + for _, stepState := range state.Steps { + step := &SagaStep{ + ID: stepState.ID, + Name: stepState.Name, + Status: stepState.Status, + Data: stepState.Data, + StartedAt: stepState.StartedAt, + CompletedAt: stepState.CompletedAt, + ExecutionID: stepState.ExecutionID, + RetryCount: stepState.RetryCount, + LastError: stepState.LastError, + } + saga.Steps = append(saga.Steps, step) + if stepState.Status == "completed" { + saga.executedOps[stepState.ExecutionID] = true + } + } + if err := o.executeCompensation(saga, len(saga.Steps)-1); err != nil { + if o.logger != nil { + o.logger.Error(fmt.Sprintf("Failed to compensate orphan saga %s: %v", state.ID, err)) + } + } + state.CompensationExecuted = true + state.Status = "orphaned_compensated" + o.storage.Save(state) +} + +func (o *SagaOrchestrator) recoverPendingSagas() { + pending, err := o.storage.ListPending() + if err != nil { + if o.logger != nil { + o.logger.Error(fmt.Sprintf("Failed to list pending sagas: %v", err)) + } + return + } + if len(pending) == 0 { + return + } + if o.logger != nil { + o.logger.Info(fmt.Sprintf("Recovering %d pending sagas", len(pending))) + } + for _, state := range pending { + if state.NodeID != "" && state.NodeID != o.nodeID { + if !o.isNodeAlive(state.NodeID) { + if o.logger != nil { + o.logger.Warn(fmt.Sprintf("Node %s is dead, taking over saga %s", state.NodeID, state.ID)) + } + state.NodeID = o.nodeID + o.storage.Save(state) + } else { + continue + } + } + if err := o.resumeSaga(state); err != nil { + if o.logger != nil { + o.logger.Error(fmt.Sprintf("Failed to resume saga %s: %v", state.ID, err)) + } + } else { + o.metrics.RecoveryCount.Add(1) + } + } +} + +func (o *SagaOrchestrator) isNodeAlive(nodeID string) bool { + return nodeID == o.nodeID +} + +func (o *SagaOrchestrator) resumeSaga(state *SagaState) error { + saga := &SagaTransaction{ + ID: state.ID, + Status: state.Status, + CurrentStep: state.CurrentStep, + Data: state.Data, + CreatedAt: state.CreatedAt, + UpdatedAt: state.UpdatedAt, + compensationExecuted: state.CompensationExecuted, + executedOps: make(map[string]bool), + } + saga.Steps = make([]*SagaStep, len(state.Steps)) + for i, stepState := range state.Steps { + saga.Steps[i] = &SagaStep{ + ID: stepState.ID, + Name: stepState.Name, + Status: stepState.Status, + Data: stepState.Data, + StartedAt: stepState.StartedAt, + CompletedAt: stepState.CompletedAt, + ExecutionID: stepState.ExecutionID, + RetryCount: stepState.RetryCount, + LastError: stepState.LastError, + } + if stepState.Status == "completed" { + saga.executedOps[stepState.ExecutionID] = true + } + } + o.mu.Lock() + for _, coord := range o.coordinators { + coord.activeSagas.Store(state.ID, saga) + } + o.mu.Unlock() + return o.executeSagaInternal(saga) +} + +func (o *SagaOrchestrator) metricsLoop() { + defer o.wg.Done() + ticker := time.NewTicker(o.config.GetMetricsInterval()) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if o.logger != nil { + metrics := o.GetMetrics() + o.logger.Debug(fmt.Sprintf("Saga metrics: started=%d, completed=%d, aborted=%d, active=%d", + metrics["total_started"], metrics["total_completed"], metrics["total_aborted"], metrics["active_count"])) + } + case <-o.stopChan: + return + } + } +} + +func (o *SagaOrchestrator) BeginSaga(id string) (*SagaTransaction, error) { + if !o.IsLeader() { + return nil, fmt.Errorf("current node is not the leader") + } + existing, err := o.storage.Load(id) + if err != nil { + return nil, err + } + if existing != nil { + return nil, fmt.Errorf("saga %s already exists", id) + } + now := time.Now().UnixMilli() + state := &SagaState{ + ID: id, + Status: "pending", + CurrentStep: 0, + Data: make(map[string]interface{}), + CreatedAt: now, + UpdatedAt: now, + NodeID: o.nodeID, + CoordinatorID: o.coordinators[0].id, + Version: 1, + ExecutionID: fmt.Sprintf("%s_%d", id, now), + } + if err := o.storage.Save(state); err != nil { + return nil, err + } + saga := &SagaTransaction{ + ID: id, + Steps: make([]*SagaStep, 0), + CurrentStep: 0, + Status: "pending", + CreatedAt: now, + UpdatedAt: now, + Data: make(map[string]interface{}), + executedOps: make(map[string]bool), + } + o.mu.RLock() + for _, coord := range o.coordinators { + coord.activeSagas.Store(id, saga) + } + o.mu.RUnlock() + o.metrics.TotalStarted.Add(1) + o.metrics.ActiveCount.Add(1) + LogTransactionAudit(TransactionID(0), "SAGA_BEGIN", TransactionActive, map[string]interface{}{ + "saga_id": id, + "node_id": o.nodeID, + }) + if o.logger != nil { + o.logger.Info(fmt.Sprintf("Saga %s started on node %s", id, o.nodeID)) + } + return saga, nil +} + +func (o *SagaOrchestrator) Execute(saga *SagaTransaction) error { + if !o.IsLeader() { + return fmt.Errorf("current node is not the leader") + } + return o.executeSagaInternal(saga) +} + +func (o *SagaOrchestrator) executeSagaInternal(saga *SagaTransaction) error { + saga.mu.Lock() + defer saga.mu.Unlock() + + if saga.Status != "pending" && saga.Status != "running" { + return fmt.Errorf("saga %s is not in pending or running state", saga.ID) + } + saga.Status = "running" + saga.UpdatedAt = time.Now().UnixMilli() + if err := o.saveSagaState(saga); err != nil { + return err + } + startTime := time.Now() + + for i := saga.CurrentStep; i < len(saga.Steps); i++ { + step := saga.Steps[i] + saga.CurrentStep = i + if step.Status == "completed" { + continue + } + step.Status = "running" + step.StartedAt = time.Now().UnixMilli() + + if saga.HasExecuted(step.ExecutionID) { + step.Status = "completed" + step.CompletedAt = time.Now().UnixMilli() + continue + } + + var err error + maxRetries := SagaMaxRetries + if o.config != nil { + maxRetries = o.config.GetMaxRetries() + } + for retry := 0; retry < maxRetries; retry++ { + step.RetryCount = retry + 1 + if err = step.Execute(); err == nil { + break + } + step.LastError = err.Error() + backoff := time.Duration(100*(1<= 0; j-- { + step := saga.Steps[j] + if step.Status == "compensated" || step.Status == "pending" { + continue + } + if step.Status == "completed" { + var err error + maxRetries := SagaMaxRetries + if o.config != nil { + maxRetries = o.config.GetMaxRetries() + } + for retry := 0; retry < maxRetries; retry++ { + if err = step.Compensate(); err == nil { + break + } + backoff := time.Duration(100*(1< o.metrics.maxLatency { + o.metrics.maxLatency = durationMs + } + o.metrics.latencies = append(o.metrics.latencies, durationMs) + if len(o.metrics.latencies) > 1000 { + o.metrics.latencies = o.metrics.latencies[1:] + } +} + +func (o *SagaOrchestrator) GetMetrics() map[string]interface{} { + o.metrics.mu.RLock() + defer o.metrics.mu.RUnlock() + latencies := make([]int64, len(o.metrics.latencies)) + copy(latencies, o.metrics.latencies) + sort.Slice(latencies, func(i, j int) bool { return latencies[i] < latencies[j] }) + p50, p95, p99 := int64(0), int64(0), int64(0) + if len(latencies) > 0 { + p50 = latencies[int(float64(len(latencies))*0.5)] + p95 = latencies[int(float64(len(latencies))*0.95)] + p99 = latencies[int(float64(len(latencies))*0.99)] + } + return map[string]interface{}{ + "total_started": o.metrics.TotalStarted.Load(), + "total_completed": o.metrics.TotalCompleted.Load(), + "total_aborted": o.metrics.TotalAborted.Load(), + "total_failed": o.metrics.TotalFailed.Load(), + "total_compensated": o.metrics.TotalCompensated.Load(), + "active_count": o.metrics.ActiveCount.Load(), + "avg_duration_ms": o.metrics.AvgDurationMs.Load(), + "min_duration_ms": o.metrics.minLatency, + "max_duration_ms": o.metrics.maxLatency, + "p50_duration_ms": p50, + "p95_duration_ms": p95, + "p99_duration_ms": p99, + "recovery_count": o.metrics.RecoveryCount.Load(), + "is_leader": o.IsLeader(), + "leader_id": o.GetLeaderID(), + "node_id": o.nodeID, + } +} + +func (o *SagaOrchestrator) GetSaga(id string) (*SagaTransaction, error) { + o.mu.RLock() + for _, coord := range o.coordinators { + if val, ok := coord.activeSagas.Load(id); ok { + o.mu.RUnlock() + return val.(*SagaTransaction), nil + } + } + o.mu.RUnlock() + state, err := o.storage.Load(id) + if err != nil { + return nil, err + } + if state == nil { + return nil, fmt.Errorf("saga %s not found", id) + } + saga := &SagaTransaction{ + ID: state.ID, + Status: state.Status, + CurrentStep: state.CurrentStep, + Data: state.Data, + CreatedAt: state.CreatedAt, + UpdatedAt: state.UpdatedAt, + CompletedAt: state.CompletedAt, + compensationExecuted: state.CompensationExecuted, + executedOps: make(map[string]bool), + } + for _, stepState := range state.Steps { + step := &SagaStep{ + ID: stepState.ID, + Name: stepState.Name, + Status: stepState.Status, + Data: stepState.Data, + StartedAt: stepState.StartedAt, + CompletedAt: stepState.CompletedAt, + ExecutionID: stepState.ExecutionID, + RetryCount: stepState.RetryCount, + LastError: stepState.LastError, + } + saga.Steps = append(saga.Steps, step) + if stepState.Status == "completed" { + saga.executedOps[stepState.ExecutionID] = true + } + } + return saga, nil +} + +func (o *SagaOrchestrator) Stop() { + close(o.stopChan) + o.wg.Wait() + for _, coord := range o.coordinators { + close(coord.stopChan) + coord.wg.Wait() + } + if o.logger != nil { + o.logger.Info("Saga orchestrator stopped") + } +} + +// ============================================================================= +// SAGA COORDINATOR +// ============================================================================= + +func (c *SagaCoordinator) run() { + defer c.wg.Done() + interval := c.config.GetStuckCheckInterval() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + c.activeSagas.Range(func(key, value interface{}) bool { + saga := value.(*SagaTransaction) + if saga.Status == "running" { + timeout := c.config.GetSagaTimeout() + if time.Since(time.UnixMilli(saga.UpdatedAt)) > timeout { + if err := c.orchestrator.resumeSagaFromStorage(saga.ID); err != nil { + if c.orchestrator.logger != nil { + c.orchestrator.logger.Error(fmt.Sprintf("Failed to recover stuck saga %s: %v", saga.ID, err)) + } + } + } + } + return true + }) + case <-c.stopChan: + return + } + } +} + +func (o *SagaOrchestrator) resumeSagaFromStorage(sagaID string) error { + state, err := o.storage.Load(sagaID) + if err != nil { + return err + } + if state == nil { + return fmt.Errorf("saga %s not found in storage", sagaID) + } + if state.NodeID != "" && state.NodeID != o.nodeID { + if o.isNodeAlive(state.NodeID) { + return fmt.Errorf("saga %s is being executed on node %s", sagaID, state.NodeID) + } + } + return o.resumeSaga(state) +} + +// ============================================================================= +// SAGA TRANSACTION +// ============================================================================= + +type SagaStep struct { + ID string `json:"id"` + Name string `json:"name"` + Execute func() error `json:"-"` + Compensate func() error `json:"-"` + Status string `json:"status"` + Data map[string]interface{} `json:"data"` + StartedAt int64 `json:"started_at"` + CompletedAt int64 `json:"completed_at"` + ExecutionID string `json:"execution_id"` + RetryCount int `json:"retry_count"` + LastError string `json:"last_error,omitempty"` +} + +type SagaTransaction struct { + ID string `json:"id"` + Steps []*SagaStep `json:"steps"` + CurrentStep int `json:"current_step"` + Status string `json:"status"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` + CompletedAt int64 `json:"completed_at,omitempty"` + Data map[string]interface{} `json:"data"` + mu sync.RWMutex + executedOps map[string]bool `json:"-"` + compensationExecuted bool `json:"-"` +} + +func (s *SagaTransaction) AddStep(name string, execute, compensate func() error, data map[string]interface{}) *SagaTransaction { + s.mu.Lock() + defer s.mu.Unlock() + stepID := fmt.Sprintf("%s_step_%d_%d", s.ID, len(s.Steps), time.Now().UnixNano()) + step := &SagaStep{ + ID: stepID, + Name: name, + Execute: execute, + Compensate: compensate, + Status: "pending", + Data: data, + StartedAt: time.Now().UnixMilli(), + ExecutionID: stepID, + RetryCount: 0, + } + s.Steps = append(s.Steps, step) + return s +} + +func (s *SagaTransaction) SetData(key string, value interface{}) { + s.mu.Lock() + defer s.mu.Unlock() + s.Data[key] = value +} + +func (s *SagaTransaction) GetData(key string) (interface{}, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + val, ok := s.Data[key] + return val, ok +} + +func (s *SagaTransaction) HasExecuted(operationID string) bool { + s.mu.RLock() + defer s.mu.RUnlock() + _, ok := s.executedOps[operationID] + return ok +} + +func (s *SagaTransaction) MarkExecuted(operationID string) { + s.mu.Lock() + defer s.mu.Unlock() + s.executedOps[operationID] = true +} + +// ============================================================================= +// ГЛОБАЛЬНЫЕ SAGA +// ============================================================================= + +var globalSagaOrchestrator *SagaOrchestrator +var sagaOrchestratorMu sync.RWMutex + +func SetGlobalSagaOrchestrator(o *SagaOrchestrator) { + sagaOrchestratorMu.Lock() + defer sagaOrchestratorMu.Unlock() + globalSagaOrchestrator = o +} + +func GetGlobalSagaOrchestrator() *SagaOrchestrator { + sagaOrchestratorMu.RLock() + defer sagaOrchestratorMu.RUnlock() + return globalSagaOrchestrator +} + +// ============================================================================= +// SAGA MANAGER - обёртка +// ============================================================================= + +type SagaManager struct { + orchestrator *SagaOrchestrator + sagas sync.Map + logger LoggerInterface + stopChan chan struct{} + wg sync.WaitGroup + mu sync.RWMutex + maxRetries int + executedOps sync.Map +} + +func NewSagaManager(logger LoggerInterface) *SagaManager { + orchestrator, err := NewSagaOrchestrator(SagaStateDir, "default-node", logger) + if err != nil { + if logger != nil { + logger.Error(fmt.Sprintf("Failed to create saga orchestrator: %v", err)) + } + return &SagaManager{ + logger: logger, + stopChan: make(chan struct{}), + maxRetries: 3, + } + } + sm := &SagaManager{ + orchestrator: orchestrator, + logger: logger, + stopChan: make(chan struct{}), + maxRetries: 3, + } + sm.wg.Add(1) + go sm.cleanupExecutedOpsLoop() + return sm +} + +func (sm *SagaManager) BeginSaga(id string) *SagaTransaction { + if sm.orchestrator != nil { + saga, err := sm.orchestrator.BeginSaga(id) + if err != nil { + if sm.logger != nil { + sm.logger.Error(fmt.Sprintf("Failed to begin saga: %v", err)) + } + return &SagaTransaction{ + ID: id, + Status: "pending", + CreatedAt: time.Now().UnixMilli(), + UpdatedAt: time.Now().UnixMilli(), + Data: make(map[string]interface{}), + executedOps: make(map[string]bool), + } + } + sm.sagas.Store(id, saga) + return saga + } + saga := &SagaTransaction{ + ID: id, + Steps: make([]*SagaStep, 0), + CurrentStep: 0, + Status: "pending", + CreatedAt: time.Now().UnixMilli(), + UpdatedAt: time.Now().UnixMilli(), + Data: make(map[string]interface{}), + executedOps: make(map[string]bool), + } + sm.sagas.Store(id, saga) + LogTransactionAudit(TransactionID(0), "SAGA_BEGIN", TransactionActive, map[string]interface{}{ + "saga_id": id, + }) + return saga +} + +func (sm *SagaManager) Execute(saga *SagaTransaction) error { + if sm.orchestrator != nil { + return sm.orchestrator.Execute(saga) + } + saga.mu.Lock() + defer saga.mu.Unlock() + if saga.Status != "pending" { + return fmt.Errorf("saga %s is not in pending state", saga.ID) + } + saga.Status = "running" + saga.UpdatedAt = time.Now().UnixMilli() + for i, step := range saga.Steps { + saga.CurrentStep = i + step.Status = "running" + step.StartedAt = time.Now().UnixMilli() + var err error + for retry := 0; retry < sm.maxRetries; retry++ { + step.RetryCount = retry + 1 + if err = step.Execute(); err == nil { + break + } + step.LastError = err.Error() + time.Sleep(time.Duration(100*(retry+1)) * time.Millisecond) + } + step.CompletedAt = time.Now().UnixMilli() + if err != nil { + step.Status = "failed" + saga.Status = "compensating" + saga.UpdatedAt = time.Now().UnixMilli() + compensationErr := sm.executeCompensation(saga, i) + if compensationErr != nil { + saga.Status = "compensation_failed" + saga.UpdatedAt = time.Now().UnixMilli() + LogTransactionAudit(TransactionID(0), "SAGA_COMPENSATION_FAILED", TransactionAborted, map[string]interface{}{ + "saga_id": saga.ID, + "failed_step": step.Name, + "error": compensationErr.Error(), + }) + return fmt.Errorf("saga %s compensation failed: %v", saga.ID, compensationErr) + } + saga.Status = "aborted" + saga.UpdatedAt = time.Now().UnixMilli() + LogTransactionAudit(TransactionID(0), "SAGA_ABORTED", TransactionAborted, map[string]interface{}{ + "saga_id": saga.ID, + "failed_step": step.Name, + "error": err.Error(), + }) + return fmt.Errorf("saga %s aborted at step %s: %v", saga.ID, step.Name, err) + } + step.Status = "completed" + saga.MarkExecuted(step.ExecutionID) + saga.UpdatedAt = time.Now().UnixMilli() + } + saga.Status = "completed" + saga.UpdatedAt = time.Now().UnixMilli() + saga.CompletedAt = saga.UpdatedAt + LogTransactionAudit(TransactionID(0), "SAGA_COMPLETED", TransactionCommitted, map[string]interface{}{ + "saga_id": saga.ID, + "steps": len(saga.Steps), + }) + return nil +} + +func (sm *SagaManager) executeCompensation(saga *SagaTransaction, failedStep int) error { + if saga.compensationExecuted { + return nil + } + for j := failedStep; j >= 0; j-- { + step := saga.Steps[j] + if step.Status == "compensated" || step.Status == "pending" { + continue + } + var err error + for retry := 0; retry < sm.maxRetries; retry++ { + if err = step.Compensate(); err == nil { + break + } + time.Sleep(time.Duration(100*(retry+1)) * time.Millisecond) + } + if err != nil { + step.Status = "compensation_failed" + return fmt.Errorf("compensation for step %s failed: %v", step.Name, err) + } + step.Status = "compensated" + step.CompletedAt = time.Now().UnixMilli() + } + saga.compensationExecuted = true + return nil +} + +func (sm *SagaManager) cleanupExecutedOpsLoop() { + defer sm.wg.Done() + ticker := time.NewTicker(24 * time.Hour) + defer ticker.Stop() + cutoff := int64(7 * 24 * 3600 * 1000) + for { + select { + case <-ticker.C: + now := time.Now().UnixMilli() + sm.executedOps.Range(func(key, value interface{}) bool { + if ts, ok := value.(int64); ok && now-ts > cutoff { + sm.executedOps.Delete(key) + } + return true + }) + case <-sm.stopChan: + return + } + } +} + +func (sm *SagaManager) GetSaga(id string) (*SagaTransaction, error) { + if sm.orchestrator != nil { + return sm.orchestrator.GetSaga(id) + } + if val, ok := sm.sagas.Load(id); ok { + return val.(*SagaTransaction), nil + } + return nil, fmt.Errorf("saga %s not found", id) +} + +func (sm *SagaManager) GetSagaStatus(id string) (string, error) { + saga, err := sm.GetSaga(id) + if err != nil { + return "", err + } + saga.mu.RLock() + defer saga.mu.RUnlock() + return saga.Status, nil +} + +func (sm *SagaManager) GetActiveSagas() []*SagaTransaction { + result := make([]*SagaTransaction, 0) + sm.sagas.Range(func(key, value interface{}) bool { + saga := value.(*SagaTransaction) + saga.mu.RLock() + status := saga.Status + saga.mu.RUnlock() + if status == "pending" || status == "running" { + result = append(result, saga) + } + return true + }) + return result +} + +func (sm *SagaManager) GetOrchestrator() *SagaOrchestrator { return sm.orchestrator } + +func (sm *SagaManager) Stop() { + close(sm.stopChan) + sm.wg.Wait() + if sm.orchestrator != nil { + sm.orchestrator.Stop() + } +} + +// ============================================================================= +// OPERATION, DocumentVersion +// ============================================================================= + +type Operation struct { + Type string `json:"type"` + Database string `json:"database"` + Collection string `json:"collection"` + DocumentID string `json:"document_id"` + Data map[string]interface{} `json:"data"` + Version uint64 `json:"version"` + OldData map[string]interface{} `json:"old_data"` +} + +type DocumentVersion struct { + Document *Document `json:"document"` + Timestamp int64 `json:"timestamp"` + TxID TransactionID `json:"tx_id"` + VersionID string `json:"version_id"` +} + +// ============================================================================= +// WAL MANAGER (не-сегментированный) +// ============================================================================= + +type WALManager struct { + mu sync.RWMutex + file *os.File + writer *bufio.Writer + path string + currentLSN uint64 + lastSync time.Time + syncInterval time.Duration + bufferSize int + closed bool + writeChan chan *WALRecord + stopChan chan struct{} + wg sync.WaitGroup + batchSize int + fsyncEnabled bool +} + +func NewWALManager(path string, fsyncEnabled bool) (*WALManager, error) { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("failed to create WAL directory: %v", err) + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0644) + if err != nil { + return nil, fmt.Errorf("failed to open WAL file: %v", err) + } + var currentLSN uint64 = 1 + stat, err := file.Stat() + if err == nil && stat.Size() > 0 { + currentLSN = uint64(stat.Size()) / 100 + if currentLSN < 1 { + currentLSN = 1 + } + } + wm := &WALManager{ + file: file, + writer: bufio.NewWriterSize(file, 64*1024), + path: path, + currentLSN: currentLSN, + lastSync: time.Now(), + syncInterval: 5 * time.Second, + bufferSize: 64 * 1024, + writeChan: make(chan *WALRecord, 10000), + stopChan: make(chan struct{}), + batchSize: 100, + fsyncEnabled: fsyncEnabled, + } + wm.wg.Add(1) + go wm.writerLoop() + return wm, nil +} + +func (wm *WALManager) writerLoop() { + defer wm.wg.Done() + batch := make([]*WALRecord, 0, wm.batchSize) + ticker := time.NewTicker(wm.syncInterval) + defer ticker.Stop() + for { + select { + case record, ok := <-wm.writeChan: + if !ok { + if len(batch) > 0 { + wm.flushBatch(batch) + } + return + } + batch = append(batch, record) + if len(batch) >= wm.batchSize { + wm.flushBatch(batch) + batch = batch[:0] + } + case <-ticker.C: + if len(batch) > 0 { + wm.flushBatch(batch) + batch = batch[:0] + } + if time.Since(wm.lastSync) >= wm.syncInterval { + wm.sync() + } + case <-wm.stopChan: + if len(batch) > 0 { + wm.flushBatch(batch) + } + wm.sync() + return + } + } +} + +func (wm *WALManager) flushBatch(batch []*WALRecord) { + wm.mu.Lock() + defer wm.mu.Unlock() + for _, record := range batch { + data, err := json.Marshal(record) + if err != nil { + continue + } + record.CRC = crc32(data) + record.Length = uint32(len(data)) + dataWithCRC, err := json.Marshal(record) + if err != nil { + continue + } + header := make([]byte, 8) + binary.BigEndian.PutUint32(header[0:4], uint32(len(dataWithCRC))) + binary.BigEndian.PutUint32(header[4:8], record.CRC) + if _, err := wm.writer.Write(header); err != nil { + continue + } + if _, err := wm.writer.Write(dataWithCRC); err != nil { + continue + } + record.LSN = wm.currentLSN + wm.currentLSN++ + } +} + +func (wm *WALManager) sync() { + wm.mu.Lock() + defer wm.mu.Unlock() + if err := wm.writer.Flush(); err == nil { + if wm.fsyncEnabled { + if err := RealFsyncWithRetry(wm.file, FsyncMaxRetries, FsyncRetryDelay); err == nil { + wm.lastSync = time.Now() + } + } else { + wm.lastSync = time.Now() + } + } +} + +func (wm *WALManager) Write(record *WALRecord) error { + if wm.closed { + return fmt.Errorf("WAL is closed") + } + record.Timestamp = time.Now().UnixMilli() + data, err := json.Marshal(record.Data) + if err != nil { + return err + } + record.CRC = crc32(data) + select { + case wm.writeChan <- record: + return nil + case <-time.After(100 * time.Millisecond): + return fmt.Errorf("WAL write timeout") + } +} + +func (wm *WALManager) Sync() error { + wm.mu.Lock() + defer wm.mu.Unlock() + if err := wm.writer.Flush(); err != nil { + return err + } + if wm.fsyncEnabled { + return RealFsyncWithRetry(wm.file, FsyncMaxRetries, FsyncRetryDelay) + } + return nil +} + +func (wm *WALManager) ReadAll() ([]*WALRecord, error) { + wm.mu.RLock() + defer wm.mu.RUnlock() + wm.writer.Flush() + file, err := os.Open(wm.path) + if err != nil { + return nil, err + } + defer file.Close() + records := make([]*WALRecord, 0) + reader := bufio.NewReader(file) + headerBuf := make([]byte, 8) + for { + _, err := io.ReadFull(reader, headerBuf) + if err != nil { + break + } + recordLen := binary.BigEndian.Uint32(headerBuf[0:4]) + expectedCRC := binary.BigEndian.Uint32(headerBuf[4:8]) + if recordLen == 0 || recordLen > 100*1024*1024 { + break + } + recordData := make([]byte, recordLen) + _, err = io.ReadFull(reader, recordData) + if err != nil { + break + } + var record WALRecord + if err := json.Unmarshal(recordData, &record); err != nil { + continue + } + data, _ := json.Marshal(record) + calculatedCRC := crc32(data) + if calculatedCRC != expectedCRC && calculatedCRC != record.CRC { + continue + } + records = append(records, &record) + } + return records, nil +} + +// Close - ИСПРАВЛЕНО: flush перед закрытием +func (wm *WALManager) Close() error { + wm.mu.Lock() + if wm.closed { + wm.mu.Unlock() + return nil + } + wm.closed = true + wm.mu.Unlock() + + close(wm.stopChan) + wm.wg.Wait() + close(wm.writeChan) + + wm.mu.Lock() + defer wm.mu.Unlock() + if err := wm.writer.Flush(); err != nil { + return err + } + if wm.fsyncEnabled { + RealFsync(wm.file) + } + return wm.file.Close() +} + +// ============================================================================= +// SEGMENTED WAL MANAGER +// ============================================================================= + +type WALSegment struct { + ID uint32 + File *os.File + Writer *bufio.Writer + Path string + StartLSN uint64 + EndLSN uint64 + Size int64 + mu sync.Mutex +} + +type WALIndexEntry struct { + LSN uint64 + SegmentID uint32 + Offset int64 + Length uint32 + Checksum uint32 +} + +type WALIndexManager struct { + index map[uint64]*WALIndexEntry + segments map[uint32]*WALSegment + mu sync.RWMutex + indexPath string +} + +type SegmentedWALManager struct { + segmentsDir string + segments map[uint32]*WALSegment + currentSegment *WALSegment + currentSegmentID uint32 + index *WALIndexManager + mu sync.RWMutex + writeChan chan *WALRecord + stopCh chan struct{} + ackChan chan uint64 // ИСПРАВЛЕНО: канал для подтверждения записи + wg sync.WaitGroup + batchSize int + logger LoggerInterface + recoveryManager *AsyncRecoveryManager + recoveryComplete atomic.Bool + backupLSN atomic.Uint64 + fsyncEnabled bool +} + +func NewSegmentedWALManager(segmentsDir string, fsyncEnabled bool, logger LoggerInterface) (*SegmentedWALManager, error) { + if err := os.MkdirAll(segmentsDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create segments dir: %v", err) + } + wm := &SegmentedWALManager{ + segmentsDir: segmentsDir, + segments: make(map[uint32]*WALSegment), + index: &WALIndexManager{ + index: make(map[uint64]*WALIndexEntry), + segments: make(map[uint32]*WALSegment), + indexPath: filepath.Join(segmentsDir, WALIndexPrefix+"index.json"), + }, + writeChan: make(chan *WALRecord, 10000), + stopCh: make(chan struct{}), + ackChan: make(chan uint64, 10000), + batchSize: 100, + logger: logger, + fsyncEnabled: fsyncEnabled, + } + if err := wm.loadExistingSegments(); err != nil { + return nil, err + } + if err := wm.index.load(); err != nil { + if logger != nil { + logger.Warn(fmt.Sprintf("Failed to load WAL index: %v", err)) + } + } + if err := wm.validateSegments(); err != nil { + if logger != nil { + logger.Warn(fmt.Sprintf("WAL segment validation failed: %v", err)) + } + } + if wm.currentSegment == nil { + if err := wm.rotateSegmentInternal(); err != nil { + return nil, err + } + } + wm.wg.Add(1) + go wm.writerLoop() + return wm, nil +} + +func (wm *SegmentedWALManager) validateSegments() error { + wm.mu.Lock() + defer wm.mu.Unlock() + for id, segment := range wm.segments { + if segment.File == nil { + file, err := os.OpenFile(segment.Path, os.O_RDWR, 0644) + if err != nil { + if wm.logger != nil { + wm.logger.Warn(fmt.Sprintf("WAL segment %d file missing: %s", id, segment.Path)) + } + delete(wm.segments, id) + continue + } + segment.File = file + segment.Writer = bufio.NewWriterSize(file, 64*1024) + } + } + return nil +} + +func (wm *SegmentedWALManager) GetBackupLSN() uint64 { return wm.backupLSN.Load() } +func (wm *SegmentedWALManager) SetBackupLSN(lsn uint64) { wm.backupLSN.Store(lsn) } + +func (wm *SegmentedWALManager) loadExistingSegments() error { + files, err := filepath.Glob(filepath.Join(wm.segmentsDir, WALSegmentPrefix+"*")) + if err != nil { + return err + } + for _, filePath := range files { + var segmentID uint32 + if _, err := fmt.Sscanf(filepath.Base(filePath), WALSegmentPrefix+"%d.log", &segmentID); err != nil { + continue + } + file, err := os.OpenFile(filePath, os.O_RDWR, 0644) + if err != nil { + continue + } + stat, _ := file.Stat() + segment := &WALSegment{ + ID: segmentID, + File: file, + Writer: bufio.NewWriterSize(file, 64*1024), + Path: filePath, + Size: stat.Size(), + StartLSN: uint64(segmentID) * WALSegmentSize / 100, + } + wm.segments[segmentID] = segment + if segmentID > wm.currentSegmentID { + wm.currentSegmentID = segmentID + wm.currentSegment = segment + } + } + return nil +} + +// rotateSegmentInternal - ИСПРАВЛЕНО: без захвата wm.mu (вызывается из flushBatch с уже захваченным mu) +func (wm *SegmentedWALManager) rotateSegmentInternal() error { + newSegmentID := wm.currentSegmentID + 1 + segmentPath := filepath.Join(wm.segmentsDir, fmt.Sprintf(WALSegmentPrefix+"%d.log", newSegmentID)) + file, err := os.OpenFile(segmentPath, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0644) + if err != nil { + return fmt.Errorf("failed to create segment: %v", err) + } + newSegment := &WALSegment{ + ID: newSegmentID, + File: file, + Writer: bufio.NewWriterSize(file, 64*1024), + Path: segmentPath, + StartLSN: wm.getCurrentLSNLocked(), + } + if wm.currentSegment != nil { + if err := wm.currentSegment.Writer.Flush(); err != nil { + if wm.logger != nil { + wm.logger.Error(fmt.Sprintf("Failed to flush old segment: %v", err)) + } + } + if wm.fsyncEnabled { + if err := RealFsyncWithRetry(wm.currentSegment.File, FsyncMaxRetries, FsyncRetryDelay); err != nil { + if wm.logger != nil { + wm.logger.Error(fmt.Sprintf("Failed to fsync old segment: %v", err)) + } + } + } + wm.currentSegment.EndLSN = wm.getCurrentLSNLocked() + wm.currentSegment.File.Close() + FsyncDir(wm.segmentsDir) + } + wm.currentSegment = newSegment + wm.currentSegmentID = newSegmentID + wm.segments[newSegmentID] = newSegment + if wm.logger != nil { + wm.logger.Info(fmt.Sprintf("Created new WAL segment: %d", newSegmentID)) + } + return nil +} + +// rotateSegment - публичный метод (захватывает mu) +func (wm *SegmentedWALManager) rotateSegment() error { + wm.mu.Lock() + defer wm.mu.Unlock() + return wm.rotateSegmentInternal() +} + +func (wm *SegmentedWALManager) getCurrentLSNLocked() uint64 { + if wm.currentSegment == nil { + return 1 + } + return wm.currentSegment.StartLSN + uint64(wm.currentSegment.Size/100) +} + +func (wm *SegmentedWALManager) getCurrentLSN() uint64 { + wm.mu.RLock() + defer wm.mu.RUnlock() + return wm.getCurrentLSNLocked() +} + +func (wm *SegmentedWALManager) Write(record *WALRecord) error { + record.Timestamp = time.Now().UnixMilli() + wm.writeChan <- record + return nil +} + +// WriteSync - ИСПРАВЛЕНО: синхронная запись с подтверждением через ackChan +func (wm *SegmentedWALManager) WriteSync(record *WALRecord) error { + record.Timestamp = time.Now().UnixMilli() + wm.writeChan <- record + + // Ждём подтверждения записи (по LSN или таймауту) + select { + case <-wm.ackChan: + return nil + case <-time.After(5 * time.Second): + return fmt.Errorf("WAL write sync timeout") + } +} + +func (wm *SegmentedWALManager) writerLoop() { + defer wm.wg.Done() + batch := make([]*WALRecord, 0, wm.batchSize) + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case record, ok := <-wm.writeChan: + if !ok { + if len(batch) > 0 { + wm.flushBatch(batch) + } + return + } + batch = append(batch, record) + if len(batch) >= wm.batchSize { + wm.flushBatch(batch) + batch = batch[:0] + } + case <-ticker.C: + if len(batch) > 0 { + wm.flushBatch(batch) + batch = batch[:0] + } + case <-wm.stopCh: + // ИСПРАВЛЕНО: flush оставшихся записей перед выходом + if len(batch) > 0 { + wm.flushBatch(batch) + } + return + } + } +} + +// flushBatch - ИСПРАВЛЕНО: не освобождает wm.mu при ротации (вызывает rotateSegmentInternal) +func (wm *SegmentedWALManager) flushBatch(batch []*WALRecord) { + wm.mu.Lock() + defer wm.mu.Unlock() + for _, record := range batch { + if wm.currentSegment.Size >= WALSegmentSize { + // ИСПРАВЛЕНО: вызываем внутренний метод без повторного захвата mu + if err := wm.rotateSegmentInternal(); err != nil { + if wm.logger != nil { + wm.logger.Error(fmt.Sprintf("Failed to rotate segment: %v", err)) + } + continue + } + } + data, err := json.Marshal(record) + if err != nil { + continue + } + lsnBytes := make([]byte, 8) + binary.BigEndian.PutUint64(lsnBytes, record.LSN) + crcData := append(lsnBytes, data...) + record.CRC = crc32(crcData) + header := make([]byte, 8) + binary.BigEndian.PutUint32(header[0:4], uint32(len(data))) + binary.BigEndian.PutUint32(header[4:8], record.CRC) + if _, err := wm.currentSegment.Writer.Write(header); err != nil { + continue + } + if _, err := wm.currentSegment.Writer.Write(data); err != nil { + continue + } + wm.index.addEntry(&WALIndexEntry{ + LSN: record.LSN, + SegmentID: wm.currentSegment.ID, + Offset: wm.currentSegment.Size, + Length: uint32(len(data)), + Checksum: record.CRC, + }) + wm.currentSegment.Size += int64(8 + len(data)) + wm.currentSegment.EndLSN = record.LSN + } + wm.currentSegment.Writer.Flush() + if wm.fsyncEnabled { + RealFsyncWithRetry(wm.currentSegment.File, FsyncMaxRetries, FsyncRetryDelay) + } + wm.index.save() + + // ИСПРАВЛЕНО: отправляем подтверждение для всех записей в batch + for range batch { + select { + case wm.ackChan <- 1: + default: + } + } +} + +func (wm *SegmentedWALManager) Sync() error { + wm.mu.Lock() + defer wm.mu.Unlock() + if wm.currentSegment == nil { + return nil + } + if err := wm.currentSegment.Writer.Flush(); err != nil { + return err + } + if wm.fsyncEnabled { + return RealFsyncWithRetry(wm.currentSegment.File, FsyncMaxRetries, FsyncRetryDelay) + } + return nil +} + +func (wm *SegmentedWALManager) ReadAll() ([]*WALRecord, error) { + wm.mu.RLock() + segments := make([]*WALSegment, 0, len(wm.segments)) + for _, seg := range wm.segments { + segments = append(segments, seg) + } + wm.mu.RUnlock() + sort.Slice(segments, func(i, j int) bool { + return segments[i].ID < segments[j].ID + }) + records := make([]*WALRecord, 0) + for _, seg := range segments { + segRecords, err := wm.readSegmentRecords(seg) + if err != nil { + if wm.logger != nil { + wm.logger.Warn(fmt.Sprintf("Failed to read segment %d: %v", seg.ID, err)) + } + continue + } + records = append(records, segRecords...) + } + return records, nil +} + +func (wm *SegmentedWALManager) ReadSince(lsn uint64) ([]*WALRecord, error) { + allRecords, err := wm.ReadAll() + if err != nil { + return nil, err + } + result := make([]*WALRecord, 0) + for _, record := range allRecords { + if record.LSN > lsn { + result = append(result, record) + } + } + return result, nil +} + +func (wm *SegmentedWALManager) GetCurrentLSN() uint64 { + wm.mu.RLock() + defer wm.mu.RUnlock() + if wm.currentSegment == nil { + return 1 + } + return wm.currentSegment.EndLSN +} + +func (wm *SegmentedWALManager) readSegmentRecords(seg *WALSegment) ([]*WALRecord, error) { + seg.mu.Lock() + defer seg.mu.Unlock() + if seg.File == nil { + return nil, nil + } + seg.Writer.Flush() + seg.File.Seek(0, 0) + records := make([]*WALRecord, 0) + reader := bufio.NewReader(seg.File) + headerBuf := make([]byte, 8) + for { + _, err := io.ReadFull(reader, headerBuf) + if err != nil { + break + } + recordLen := binary.BigEndian.Uint32(headerBuf[0:4]) + expectedCRC := binary.BigEndian.Uint32(headerBuf[4:8]) + if recordLen == 0 || recordLen > 100*1024*1024 { + break + } + recordData := make([]byte, recordLen) + _, err = io.ReadFull(reader, recordData) + if err != nil { + break + } + var record WALRecord + if err := json.Unmarshal(recordData, &record); err != nil { + continue + } + lsnBytes := make([]byte, 8) + binary.BigEndian.PutUint64(lsnBytes, record.LSN) + crcData := append(lsnBytes, recordData...) + calculatedCRC := crc32(crcData) + if calculatedCRC != expectedCRC && calculatedCRC != record.CRC { + continue + } + records = append(records, &record) + } + return records, nil +} + +// Close - ИСПРАВЛЕНО: flush оставшихся записей перед закрытием +func (wm *SegmentedWALManager) Close() error { + close(wm.stopCh) + wm.wg.Wait() + close(wm.writeChan) + + wm.mu.Lock() + defer wm.mu.Unlock() + if wm.currentSegment != nil { + wm.currentSegment.Writer.Flush() + if wm.fsyncEnabled { + RealFsyncWithRetry(wm.currentSegment.File, FsyncMaxRetries, FsyncRetryDelay) + } + wm.currentSegment.File.Close() + } + wm.index.save() + return nil +} + +func (im *WALIndexManager) addEntry(entry *WALIndexEntry) { + im.mu.Lock() + defer im.mu.Unlock() + im.index[entry.LSN] = entry +} + +func (im *WALIndexManager) save() error { + im.mu.RLock() + defer im.mu.RUnlock() + data, err := json.Marshal(im.index) + if err != nil { + return err + } + return os.WriteFile(im.indexPath, data, 0644) +} + +func (im *WALIndexManager) load() error { + data, err := os.ReadFile(im.indexPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if len(data) == 0 { + return nil + } + return json.Unmarshal(data, &im.index) +} + +// ============================================================================= +// ASYNC RECOVERY MANAGER +// ============================================================================= + +type AsyncRecoveryManager struct { + recordChan chan *WALRecord + errChan chan error + doneChan chan struct{} + wg sync.WaitGroup + callback func(*WALRecord) error + mu sync.RWMutex + isRunning bool + recoveredCnt atomic.Uint64 + errorCnt atomic.Uint64 + startTime time.Time +} + +func NewAsyncRecoveryManager(callback func(*WALRecord) error, workers int) *AsyncRecoveryManager { + arm := &AsyncRecoveryManager{ + recordChan: make(chan *WALRecord, AsyncRecoveryBufferSize), + errChan: make(chan error, workers), + doneChan: make(chan struct{}), + callback: callback, + startTime: time.Now(), + isRunning: true, + } + for i := 0; i < workers; i++ { + arm.wg.Add(1) + go arm.worker() + } + go arm.errorMonitor() + return arm +} + +func (arm *AsyncRecoveryManager) worker() { + defer arm.wg.Done() + for record := range arm.recordChan { + var err error + if arm.callback != nil { + err = arm.callback(record) + } + if err != nil { + select { + case arm.errChan <- err: + default: + } + arm.errorCnt.Add(1) + } else { + arm.recoveredCnt.Add(1) + } + } +} + +func (arm *AsyncRecoveryManager) errorMonitor() { + criticalErrors := 0 + for range arm.errChan { + criticalErrors++ + if criticalErrors > 10 { + arm.Stop() + return + } + } +} + +func (arm *AsyncRecoveryManager) Push(record *WALRecord) bool { + arm.mu.RLock() + if !arm.isRunning { + arm.mu.RUnlock() + return false + } + arm.mu.RUnlock() + select { + case arm.recordChan <- record: + return true + case <-time.After(100 * time.Millisecond): + return false + } +} + +func (arm *AsyncRecoveryManager) Wait() { + close(arm.recordChan) + arm.wg.Wait() + close(arm.doneChan) +} + +// Stop - ИСПРАВЛЕНО: ждём обработки буфера перед закрытием +func (arm *AsyncRecoveryManager) Stop() { + arm.mu.Lock() + if !arm.isRunning { + arm.mu.Unlock() + return + } + arm.isRunning = false + arm.mu.Unlock() + + // ИСПРАВЛЕНО: ждём обработки оставшихся записей + close(arm.recordChan) + arm.wg.Wait() + close(arm.errChan) +} + +func (arm *AsyncRecoveryManager) GetStats() map[string]interface{} { + return map[string]interface{}{ + "recovered": arm.recoveredCnt.Load(), + "errors": arm.errorCnt.Load(), + "is_running": arm.isRunning, + "elapsed_ms": time.Since(arm.startTime).Milliseconds(), + } +} + +// ============================================================================= +// DEADLOCK DETECTOR +// ============================================================================= + +type DeadlockDetector struct { + waitForGraph sync.Map + checkInterval time.Duration + timeout time.Duration + mu sync.RWMutex + stopChan chan struct{} + wg sync.WaitGroup + logger LoggerInterface +} + +func NewDeadlockDetector(checkInterval, timeout time.Duration) *DeadlockDetector { + if checkInterval <= 0 { + checkInterval = DeadlockCheckInterval + } + if timeout <= 0 { + timeout = DefaultTxTimeout + } + d := &DeadlockDetector{ + checkInterval: checkInterval, + timeout: timeout, + stopChan: make(chan struct{}), + } + d.wg.Add(1) + go d.detectLoop() + return d +} + +func (dd *DeadlockDetector) SetLogger(logger LoggerInterface) { dd.logger = logger } + +func (dd *DeadlockDetector) Stop() { + close(dd.stopChan) + dd.wg.Wait() +} + +func (dd *DeadlockDetector) AddWaiting(waiting, waitingFor TransactionID) { + var list []TransactionID + if val, ok := dd.waitForGraph.Load(waiting); ok { + list = val.([]TransactionID) + } + list = append(list, waitingFor) + dd.waitForGraph.Store(waiting, list) +} + +func (dd *DeadlockDetector) RemoveWaiting(txID TransactionID) { + dd.waitForGraph.Delete(txID) +} + +func (dd *DeadlockDetector) detectLoop() { + defer dd.wg.Done() + ticker := time.NewTicker(dd.checkInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + dd.detect() + case <-dd.stopChan: + return + } + } +} + +func (dd *DeadlockDetector) detect() { + visited := make(map[TransactionID]bool) + stack := make(map[TransactionID]bool) + var dfs func(txID TransactionID) bool + dfs = func(txID TransactionID) bool { + visited[txID] = true + stack[txID] = true + val, ok := dd.waitForGraph.Load(txID) + if ok { + for _, next := range val.([]TransactionID) { + if !visited[next] { + if dfs(next) { + return true + } + } else if stack[next] { + dd.resolveDeadlock(txID, next) + return true + } + } + } + stack[txID] = false + return false + } + dd.waitForGraph.Range(func(key, value interface{}) bool { + txID := key.(TransactionID) + if !visited[txID] { + dfs(txID) + } + return true + }) +} + +func (dd *DeadlockDetector) resolveDeadlock(txID1, txID2 TransactionID) { + if globalTxManager != nil { + if val, ok := globalTxManager.activeTransactions.Load(txID1); ok { + tx := val.(*Transaction) + if tx.IsDistributed { + if dd.logger != nil { + dd.logger.Warn(fmt.Sprintf("Distributed transaction %d involved in deadlock with %d, aborting", txID1, txID2)) + } + AbortDistributedTransaction(txID1) + } else { + tx.State.Store(int32(TransactionAborted)) + globalTxManager.activeTransactions.Delete(txID1) + globalTxManager.stats.TotalDeadlocks.Add(1) + globalTxManager.stats.ActiveCount.Add(^uint64(0)) + if dd.logger != nil { + dd.logger.Warn(fmt.Sprintf("Deadlock resolved: aborted transaction %d due to conflict with %d", txID1, txID2)) + } + } + } + } +} + +// ============================================================================= +// DISTRIBUTED TX COORDINATOR +// ============================================================================= + +type TxState int32 + +const ( + TxActive TxState = iota + TxCommitted + TxAborted + TxTimeout +) + +type DistributedTxInfo struct { + TxID TransactionID + Nodes []string + Status TxState + StartTime int64 + Timeout time.Duration +} + +type DistributedTransactionCoordinator struct { + pendingTxs sync.Map + timeout time.Duration + mu sync.RWMutex +} + +func NewDistributedTransactionCoordinator(timeout time.Duration) *DistributedTransactionCoordinator { + if timeout <= 0 { + timeout = 30 * time.Second + } + return &DistributedTransactionCoordinator{timeout: timeout} +} + +func (dtc *DistributedTransactionCoordinator) Prepare(txID TransactionID, nodes []string) error { + info := &DistributedTxInfo{ + TxID: txID, + Nodes: nodes, + Status: TxActive, + StartTime: time.Now().UnixMilli(), + Timeout: dtc.timeout, + } + dtc.pendingTxs.Store(txID, info) + return nil +} + +func (dtc *DistributedTransactionCoordinator) Commit(txID TransactionID) error { + val, ok := dtc.pendingTxs.Load(txID) + if !ok { + return fmt.Errorf("transaction not found: %d", txID) + } + info := val.(*DistributedTxInfo) + info.Status = TxCommitted + return nil +} + +func (dtc *DistributedTransactionCoordinator) Abort(txID TransactionID) error { + dtc.pendingTxs.Delete(txID) + return nil +} + +// ============================================================================= +// Transaction +// ============================================================================= + +type Transaction struct { + ID TransactionID + State atomic.Int32 + Operations []Operation + StartTime int64 + Version uint64 + mu sync.RWMutex + IsDistributed bool + Nodes []string + savepoints []*Savepoint + timeout time.Duration + timeoutTimer *time.Timer +} + +type Savepoint struct { + Name string + Timestamp int64 + OpCount int + Snapshot *Document +} + +type TransactionOptions struct { + Timeout time.Duration + IsDistributed bool + Nodes []string + IsolationLevel string +} + +type TransactionInfo struct { + ID string `json:"id"` + Status string `json:"status"` + StartTime int64 `json:"start_time"` + OperationCount int `json:"operation_count"` + Operations []OperationInfo `json:"operations,omitempty"` + Savepoints []string `json:"savepoints,omitempty"` + Nodes []string `json:"nodes,omitempty"` +} + +type OperationInfo struct { + Type string `json:"type"` + Database string `json:"database"` + Collection string `json:"collection"` + DocumentID string `json:"document_id"` +} + +type TransactionStats struct { + TotalStarted atomic.Uint64 + TotalCommitted atomic.Uint64 + TotalAborted atomic.Uint64 + TotalTimedOut atomic.Uint64 + TotalDeadlocks atomic.Uint64 + ActiveCount atomic.Uint64 + PeakActiveCount atomic.Uint64 + MaxOpsPerTx atomic.Uint64 + AvgOpsPerTx atomic.Uint64 + TotalOps atomic.Uint64 + StartTime time.Time +} + +type TransactionManager struct { + activeTransactions sync.Map + nextTxID atomic.Uint64 + wal *SegmentedWALManager + logger LoggerInterface + mu sync.RWMutex + walPath string + checkpointInterval int64 + lastCheckpoint int64 + checkpointFile *os.File + documentVersions sync.Map + maxVersions int + visibilityMap *VisibilityMap + readCache *ReadTimestampCache + distCoord *DistributedTransactionCoordinator + deadlockDetector *DeadlockDetector + recoveryManager *AsyncRecoveryManager + recoveryComplete atomic.Bool + backupLock sync.RWMutex + backupInProgress atomic.Bool + stats *TransactionStats + mvccManager *MVCCManager +} + +// ============================================================================= +// ГЛОБАЛЬНЫЕ ПЕРЕМЕННЫЕ +// ============================================================================= + +var ( + globalTxManager *TransactionManager + txManagerOnce sync.Once + currentTx atomic.Value + globalStorage *Storage +) + +func InitTransactionManager(walPath string) error { + return InitTransactionManagerWithConfig(walPath, nil) +} + +func InitTransactionManagerWithConfig(walPath string, config map[string]interface{}) error { + var err error + txManagerOnce.Do(func() { + maxVersions := 10 + if config != nil { + if v, ok := config["max_versions"].(int); ok && v > 0 { + maxVersions = v + } + } + globalTxManager = &TransactionManager{ + nextTxID: atomic.Uint64{}, + walPath: walPath, + checkpointInterval: 300, + lastCheckpoint: time.Now().Unix(), + maxVersions: maxVersions, + visibilityMap: NewVisibilityMap(VisibilityMapSize), + readCache: NewReadTimestampCache(10000, 5*time.Minute), + distCoord: NewDistributedTransactionCoordinator(30 * time.Second), + deadlockDetector: NewDeadlockDetector(DeadlockCheckInterval, DefaultTxTimeout), + stats: &TransactionStats{StartTime: time.Now()}, + mvccManager: NewMVCCManager(MaxVersionsPerDoc, VersionRetentionDays), + } + globalTxManager.nextTxID.Store(1) + var walErr error + fsyncEnabled := true + if config != nil { + if v, ok := config["fsync_enabled"].(bool); ok { + fsyncEnabled = v + } + } + globalTxManager.wal, walErr = NewSegmentedWALManager(filepath.Dir(walPath), fsyncEnabled, nil) + if walErr != nil { + err = walErr + return + } + globalTxManager.startAsyncRecovery() + go globalTxManager.checkpointLoop() + go globalTxManager.versionCleanupLoop() + go globalTxManager.statsMonitor() + }) + return err +} + +func GetTransactionManager() *TransactionManager { return globalTxManager } + +func SetTransactionLogger(logger LoggerInterface) { + if globalTxManager != nil { + globalTxManager.logger = logger + if globalTxManager.wal != nil { + globalTxManager.wal.logger = logger + } + if globalTxManager.deadlockDetector != nil { + globalTxManager.deadlockDetector.SetLogger(logger) + } + } +} + +func SetGlobalStorage(s *Storage) { globalStorage = s } +func GetGlobalStorage() *Storage { return globalStorage } + +func BeginTransaction() *Transaction { + if globalTxManager == nil { + InitTransactionManager("futriis.wal") + } + return BeginTransactionWithOptions(&TransactionOptions{ + Timeout: DefaultTxTimeout, + }) +} + +func BeginTransactionWithOptions(options *TransactionOptions) *Transaction { + if globalTxManager == nil { + InitTransactionManager("futriis.wal") + } + if options == nil { + options = &TransactionOptions{Timeout: DefaultTxTimeout} + } + tx := &Transaction{ + ID: TransactionID(globalTxManager.nextTxID.Add(1) - 1), + StartTime: time.Now().UnixMilli(), + Operations: make([]Operation, 0, 100), + Version: 1, + savepoints: make([]*Savepoint, 0), + timeout: options.Timeout, + IsDistributed: options.IsDistributed, + Nodes: options.Nodes, + } + tx.State.Store(int32(TransactionActive)) + globalTxManager.activeTransactions.Store(tx.ID, tx) + currentTx.Store(tx) + globalTxManager.stats.TotalStarted.Add(1) + globalTxManager.stats.ActiveCount.Add(1) + LogTransactionAudit(tx.ID, "START", TransactionActive, map[string]interface{}{ + "start_time": tx.StartTime, + "timeout_ms": options.Timeout.Milliseconds(), + "distributed": options.IsDistributed, + }) + if options.Timeout > 0 { + tx.timeoutTimer = time.AfterFunc(options.Timeout, func() { + if TransactionState(tx.State.Load()) == TransactionActive { + tx.State.Store(int32(TransactionAborted)) + globalTxManager.activeTransactions.Delete(tx.ID) + globalTxManager.stats.TotalTimedOut.Add(1) + globalTxManager.stats.ActiveCount.Add(^uint64(0)) + LogTransactionAudit(tx.ID, "TIMEOUT", TransactionAborted, map[string]interface{}{ + "timeout_ms": options.Timeout.Milliseconds(), + }) + } + }) + } + return tx +} + +func BeginTransactionWithTimeout(timeout time.Duration) *Transaction { + return BeginTransactionWithOptions(&TransactionOptions{Timeout: timeout}) +} + +func BeginDistributedTransaction(nodes []string) (*Transaction, error) { + if globalTxManager == nil { + if err := InitTransactionManager("futriis.wal"); err != nil { + return nil, err + } + } + options := &TransactionOptions{ + Timeout: 30 * time.Second, + IsDistributed: true, + Nodes: nodes, + IsolationLevel: "READ_COMMITTED", + } + tx := BeginTransactionWithOptions(options) + if tx == nil { + return nil, fmt.Errorf("failed to create transaction") + } + if err := globalTxManager.distCoord.Prepare(tx.ID, nodes); err != nil { + return nil, err + } + if globalTxManager.logger != nil { + globalTxManager.logger.Info(fmt.Sprintf("Distributed transaction %d started on nodes: %v", tx.ID, nodes)) + } + return tx, nil +} + +// CommitCurrentTransaction - ИСПРАВЛЕНО: сначала WAL, потом применение +func CommitCurrentTransaction() error { + txVal := currentTx.Load() + if txVal == nil { + return fmt.Errorf("no active transaction") + } + tx := txVal.(*Transaction) + if TransactionState(tx.State.Load()) != TransactionActive { + return fmt.Errorf("transaction is not active") + } + if tx.IsDistributed { + return fmt.Errorf("distributed transaction must use CommitDistributedTransaction") + } + if tx.timeout > 0 && time.Since(time.UnixMilli(tx.StartTime)) > tx.timeout { + AbortCurrentTransaction() + return fmt.Errorf("transaction timeout exceeded") + } + + // ИСПРАВЛЕНО: Сначала записываем транзакцию в WAL (prepared state) + txRecord := &TransactionRecord{ + ID: tx.ID, + State: TransactionCommitted, + Timestamp: time.Now().UnixMilli(), + Operations: tx.Operations, + } + data, err := json.Marshal(txRecord) + if err != nil { + return fmt.Errorf("failed to marshal transaction: %v", err) + } + walRecord := &WALRecord{Type: 1, Data: data} + // ИСПРАВЛЕНО: используем WriteSync для гарантии записи в WAL до применения + if err := globalTxManager.wal.WriteSync(walRecord); err != nil { + return fmt.Errorf("failed to write transaction to WAL: %v", err) + } + + // Теперь применяем операции + for _, op := range tx.Operations { + if err := applyOperation(op); err != nil { + AbortCurrentTransaction() + return fmt.Errorf("transaction commit failed at operation %s: %v", op.Type, err) + } + if globalTxManager != nil && op.DocumentID != "" { + if globalStorage != nil { + db, _ := globalStorage.GetDatabase(op.Database) + if db != nil { + coll, _ := db.GetCollection(op.Collection) + if coll != nil { + if doc, err := coll.Find(op.DocumentID); err == nil { + if globalTxManager.mvccManager != nil { + globalTxManager.mvccManager.CreateVersion(doc, tx.ID) + } + globalTxManager.AddDocumentVersion(op.DocumentID, &DocumentVersion{ + Document: doc.Clone(), + Timestamp: time.Now().UnixMilli(), + TxID: tx.ID, + }) + } + } + } + } + } + } + + tx.State.Store(int32(TransactionCommitted)) + LogTransactionAudit(tx.ID, "COMMIT", TransactionCommitted, map[string]interface{}{ + "operations": len(tx.Operations), + }) + globalTxManager.stats.TotalCommitted.Add(1) + globalTxManager.stats.ActiveCount.Add(^uint64(0)) + globalTxManager.stats.TotalOps.Add(uint64(len(tx.Operations))) + if uint64(len(tx.Operations)) > globalTxManager.stats.MaxOpsPerTx.Load() { + globalTxManager.stats.MaxOpsPerTx.Store(uint64(len(tx.Operations))) + } + if tx.timeoutTimer != nil { + tx.timeoutTimer.Stop() + } + currentTx.Store(nil) + globalTxManager.activeTransactions.Delete(tx.ID) + return nil +} + +func AbortCurrentTransaction() error { + txVal := currentTx.Load() + if txVal == nil { + return fmt.Errorf("no active transaction") + } + tx := txVal.(*Transaction) + tx.State.Store(int32(TransactionAborted)) + LogTransactionAudit(tx.ID, "ABORT", TransactionAborted, map[string]interface{}{ + "operations": len(tx.Operations), + }) + globalTxManager.stats.TotalAborted.Add(1) + globalTxManager.stats.ActiveCount.Add(^uint64(0)) + if tx.timeoutTimer != nil { + tx.timeoutTimer.Stop() + } + currentTx.Store(nil) + globalTxManager.activeTransactions.Delete(tx.ID) + return nil +} + +func CommitDistributedTransaction(txID TransactionID) error { + if globalTxManager == nil { + return fmt.Errorf("transaction manager not initialized") + } + val, ok := globalTxManager.activeTransactions.Load(txID) + if !ok { + return fmt.Errorf("transaction not found: %d", txID) + } + tx := val.(*Transaction) + if TransactionState(tx.State.Load()) != TransactionActive { + return fmt.Errorf("transaction is not active") + } + + // ИСПРАВЛЕНО: Сначала WAL + txRecord := &TransactionRecord{ + ID: tx.ID, + State: TransactionCommitted, + Timestamp: time.Now().UnixMilli(), + Operations: tx.Operations, + IsDistributed: true, + Nodes: tx.Nodes, + } + data, err := json.Marshal(txRecord) + if err != nil { + return fmt.Errorf("failed to marshal: %v", err) + } + if err := globalTxManager.wal.WriteSync(&WALRecord{Type: 1, Data: data}); err != nil { + return fmt.Errorf("failed to write to WAL: %v", err) + } + + if err := globalTxManager.distCoord.Commit(txID); err != nil { + return err + } + for _, op := range tx.Operations { + if err := applyOperation(op); err != nil { + return fmt.Errorf("failed to apply operation: %v", err) + } + } + tx.State.Store(int32(TransactionCommitted)) + globalTxManager.activeTransactions.Delete(txID) + globalTxManager.stats.TotalCommitted.Add(1) + globalTxManager.stats.ActiveCount.Add(^uint64(0)) + if tx.timeoutTimer != nil { + tx.timeoutTimer.Stop() + } + LogTransactionAudit(txID, "COMMIT_DISTRIBUTED", TransactionCommitted, map[string]interface{}{ + "nodes": tx.Nodes, + }) + return nil +} + +func AbortDistributedTransaction(txID TransactionID) error { + if globalTxManager == nil { + return fmt.Errorf("transaction manager not initialized") + } + val, ok := globalTxManager.activeTransactions.Load(txID) + if !ok { + return fmt.Errorf("transaction not found: %d", txID) + } + tx := val.(*Transaction) + if err := globalTxManager.distCoord.Abort(txID); err != nil { + return err + } + tx.State.Store(int32(TransactionAborted)) + globalTxManager.activeTransactions.Delete(txID) + globalTxManager.stats.TotalAborted.Add(1) + globalTxManager.stats.ActiveCount.Add(^uint64(0)) + if tx.timeoutTimer != nil { + tx.timeoutTimer.Stop() + } + LogTransactionAudit(txID, "ABORT_DISTRIBUTED", TransactionAborted, map[string]interface{}{ + "nodes": tx.Nodes, + }) + return nil +} + +func applyOperation(op Operation) error { + if globalStorage == nil { + return fmt.Errorf("storage not initialized") + } + db, err := globalStorage.GetDatabase(op.Database) + if err != nil { + return fmt.Errorf("database not found: %s", op.Database) + } + coll, err := db.GetCollection(op.Collection) + if err != nil { + return fmt.Errorf("collection not found: %s", op.Collection) + } + switch op.Type { + case "insert": + doc := NewDocumentWithID(op.DocumentID) + for k, v := range op.Data { + doc.SetField(k, v) + } + doc.Version = op.Version + if globalTxManager != nil && globalTxManager.mvccManager != nil { + globalTxManager.mvccManager.CreateVersion(doc, 0) + } + return coll.Insert(doc) + case "update": + if err := coll.Update(op.DocumentID, op.Data); err != nil { + return err + } + if globalTxManager != nil && globalTxManager.mvccManager != nil { + if doc, err := coll.Find(op.DocumentID); err == nil { + globalTxManager.mvccManager.CreateVersion(doc, 0) + } + } + return nil + case "delete": + return coll.Delete(op.DocumentID) + case "restore": + return coll.RestoreDeleted(op.DocumentID) + } + return nil +} + +func (tx *Transaction) CreateSavepoint(name string) error { + if TransactionState(tx.State.Load()) != TransactionActive { + return fmt.Errorf("transaction is not active") + } + if len(tx.savepoints) >= MaxSavepointsPerTx { + return fmt.Errorf("too many savepoints (max %d)", MaxSavepointsPerTx) + } + for _, sp := range tx.savepoints { + if sp.Name == name { + return fmt.Errorf("savepoint '%s' already exists", name) + } + } + savepoint := &Savepoint{ + Name: name, + Timestamp: time.Now().UnixMilli(), + OpCount: len(tx.Operations), + } + if len(tx.Operations) > 0 { + lastOp := tx.Operations[len(tx.Operations)-1] + if lastOp.DocumentID != "" && globalStorage != nil { + db, _ := globalStorage.GetDatabase(lastOp.Database) + if db != nil { + coll, _ := db.GetCollection(lastOp.Collection) + if coll != nil { + if doc, err := coll.Find(lastOp.DocumentID); err == nil { + savepoint.Snapshot = doc.Clone() + } + } + } + } + } + tx.mu.Lock() + tx.savepoints = append(tx.savepoints, savepoint) + tx.mu.Unlock() + LogTransactionAudit(tx.ID, "SAVEPOINT", TransactionActive, map[string]interface{}{ + "savepoint": name, + "op_count": savepoint.OpCount, + }) + return nil +} + +func (tx *Transaction) RollbackToSavepoint(name string) error { + if TransactionState(tx.State.Load()) != TransactionActive { + return fmt.Errorf("transaction is not active") + } + tx.mu.Lock() + defer tx.mu.Unlock() + var targetIdx int = -1 + for i, sp := range tx.savepoints { + if sp.Name == name { + targetIdx = i + break + } + } + if targetIdx == -1 { + return fmt.Errorf("savepoint '%s' not found", name) + } + if len(tx.Operations) > tx.savepoints[targetIdx].OpCount { + tx.Operations = tx.Operations[:tx.savepoints[targetIdx].OpCount] + } + tx.savepoints = tx.savepoints[:targetIdx+1] + LogTransactionAudit(tx.ID, "ROLLBACK_TO_SAVEPOINT", TransactionActive, map[string]interface{}{ + "savepoint": name, + }) + return nil +} + +func (tx *Transaction) ReleaseSavepoint(name string) error { + tx.mu.Lock() + defer tx.mu.Unlock() + for i, sp := range tx.savepoints { + if sp.Name == name { + tx.savepoints = append(tx.savepoints[:i], tx.savepoints[i+1:]...) + LogTransactionAudit(tx.ID, "RELEASE_SAVEPOINT", TransactionActive, map[string]interface{}{ + "savepoint": name, + }) + return nil + } + } + return fmt.Errorf("savepoint '%s' not found", name) +} + +func (tx *Transaction) GetSavepoints() []string { + tx.mu.RLock() + defer tx.mu.RUnlock() + names := make([]string, len(tx.savepoints)) + for i, sp := range tx.savepoints { + names[i] = sp.Name + } + return names +} + +func (tm *TransactionManager) AddDocumentVersion(docID string, version *DocumentVersion) { + val, _ := tm.documentVersions.LoadOrStore(docID, make([]*DocumentVersion, 0)) + versions := val.([]*DocumentVersion) + versions = append(versions, version) + if len(versions) > tm.maxVersions && tm.maxVersions > 0 { + versions = versions[len(versions)-tm.maxVersions:] + } + tm.documentVersions.Store(docID, versions) + if tm.visibilityMap != nil { + tm.visibilityMap.MarkVisible(docID, uint64(version.TxID), true) + } +} + +func (tm *TransactionManager) GetDocumentVersion(docID string, timestamp int64) *Document { + if tm.readCache != nil { + if cached := tm.readCache.Get(docID, timestamp); cached != nil { + return cached + } + } + val, ok := tm.documentVersions.Load(docID) + if !ok { + return nil + } + versions := val.([]*DocumentVersion) + for i := len(versions) - 1; i >= 0; i-- { + if versions[i].Timestamp <= timestamp { + doc := versions[i].Document.Clone() + if tm.readCache != nil { + tm.readCache.Set(docID, timestamp, doc) + } + return doc + } + } + return nil +} + +func (tm *TransactionManager) checkpointLoop() { + ticker := time.NewTicker(time.Duration(tm.checkpointInterval) * time.Second) + defer ticker.Stop() + for range ticker.C { + tm.createCheckpoint() + } +} + +// createCheckpoint - ИСПРАВЛЕНО: fsync для чекпоинта +func (tm *TransactionManager) createCheckpoint() { + if tm.wal == nil { + return + } + now := time.Now().Unix() + if now-tm.lastCheckpoint < tm.checkpointInterval { + return + } + checkpointPath := fmt.Sprintf("%s.checkpoint.%d", tm.walPath, now) + checkpoint := make(map[string]interface{}) + checkpoint["timestamp"] = now + checkpoint["backup_lsn"] = tm.wal.GetBackupLSN() + data, err := json.Marshal(checkpoint) + if err != nil { + if tm.logger != nil { + tm.logger.Error(fmt.Sprintf("Failed to marshal checkpoint: %v", err)) + } + return + } + // ИСПРАВЛЕНО: используем writeFileSync с fsync + if err := writeFileSync(checkpointPath, data, 0644); err != nil { + if tm.logger != nil { + tm.logger.Error(fmt.Sprintf("Failed to write checkpoint: %v", err)) + } + return + } + tm.lastCheckpoint = now + if tm.logger != nil { + tm.logger.Info(fmt.Sprintf("Checkpoint created: %s", checkpointPath)) + } +} + +func (tm *TransactionManager) versionCleanupLoop() { + if tm.maxVersions <= 0 { + return + } + ticker := time.NewTicker(VersionPruneInterval) + defer ticker.Stop() + for range ticker.C { + cutoffTime := time.Now().AddDate(0, 0, -VersionRetentionDays).UnixMilli() + tm.documentVersions.Range(func(key, value interface{}) bool { + versions := value.([]*DocumentVersion) + if len(versions) <= tm.maxVersions { + return true + } + newVersions := make([]*DocumentVersion, 0, tm.maxVersions) + for _, v := range versions { + if v.Timestamp >= cutoffTime && len(newVersions) < tm.maxVersions { + newVersions = append(newVersions, v) + } + } + if len(newVersions) < len(versions) { + tm.documentVersions.Store(key, newVersions) + } + return true + }) + } +} + +func (tm *TransactionManager) statsMonitor() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for range ticker.C { + active := tm.stats.ActiveCount.Load() + if active > tm.stats.PeakActiveCount.Load() { + tm.stats.PeakActiveCount.Store(active) + } + } +} + +func (tm *TransactionManager) startAsyncRecovery() { + if tm.wal == nil { + tm.recoveryComplete.Store(true) + return + } + if tm.logger != nil { + tm.logger.Info("Starting asynchronous WAL recovery...") + } + records, err := tm.wal.ReadAll() + if err != nil { + if tm.logger != nil { + tm.logger.Error(fmt.Sprintf("Failed to read WAL: %v", err)) + } + tm.recoveryComplete.Store(true) + return + } + if len(records) == 0 { + if tm.logger != nil { + tm.logger.Info("No records to recover") + } + tm.recoveryComplete.Store(true) + return + } + tm.recoveryManager = NewAsyncRecoveryManager(func(record *WALRecord) error { + if record.Type == 1 { + var txRecord TransactionRecord + if err := json.Unmarshal(record.Data, &txRecord); err != nil { + return err + } + if txRecord.State == TransactionCommitted { + for _, op := range txRecord.Operations { + if err := applyOperation(op); err != nil { + return err + } + } + } + } + return nil + }, AsyncRecoveryWorkers) + for _, record := range records { + if !tm.recoveryManager.Push(record) { + if tm.logger != nil { + tm.logger.Warn("Recovery buffer full, some records may be delayed") + } + } + } + go func() { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + stats := tm.recoveryManager.GetStats() + if tm.logger != nil { + tm.logger.Debug(fmt.Sprintf("Recovery progress: %d records recovered", stats["recovered"])) + } + case <-tm.recoveryManager.doneChan: + stats := tm.recoveryManager.GetStats() + if tm.logger != nil { + tm.logger.Info(fmt.Sprintf("WAL recovery completed: %d records recovered, %d errors", + stats["recovered"], stats["errors"])) + } + tm.recoveryComplete.Store(true) + return + } + } + }() +} + +func (tm *TransactionManager) IsRecoveryComplete() bool { return tm.recoveryComplete.Load() } + +func (tm *TransactionManager) GetRecoveryProgress() map[string]interface{} { + if tm.recoveryManager == nil { + return map[string]interface{}{ + "is_recovering": false, + "recovered": 0, + "complete": true, + } + } + stats := tm.recoveryManager.GetStats() + return map[string]interface{}{ + "is_recovering": !tm.recoveryComplete.Load(), + "recovered": stats["recovered"], + "complete": tm.recoveryComplete.Load(), + "elapsed_ms": stats["elapsed_ms"], + } +} + +func (tm *TransactionManager) LockForBackup() { + tm.backupLock.Lock() + tm.backupInProgress.Store(true) +} + +func (tm *TransactionManager) UnlockForBackup() { + tm.backupInProgress.Store(false) + tm.backupLock.Unlock() +} + +func (tm *TransactionManager) IsBackupInProgress() bool { return tm.backupInProgress.Load() } + +func GetTransactionStats() map[string]interface{} { + if globalTxManager == nil { + return map[string]interface{}{"error": "transaction manager not initialized"} + } + stats := globalTxManager.stats + active := stats.ActiveCount.Load() + totalStarted := stats.TotalStarted.Load() + totalCommitted := stats.TotalCommitted.Load() + totalAborted := stats.TotalAborted.Load() + return map[string]interface{}{ + "total_started": totalStarted, + "total_committed": totalCommitted, + "total_aborted": totalAborted, + "total_timed_out": stats.TotalTimedOut.Load(), + "total_deadlocks": stats.TotalDeadlocks.Load(), + "active_count": active, + "peak_active_count": stats.PeakActiveCount.Load(), + "max_ops_per_tx": stats.MaxOpsPerTx.Load(), + "avg_ops_per_tx": stats.AvgOpsPerTx.Load(), + "total_ops": stats.TotalOps.Load(), + "commit_rate": float64(totalCommitted) / float64(totalStarted+1) * 100, + "abort_rate": float64(totalAborted) / float64(totalStarted+1) * 100, + "uptime_seconds": time.Since(stats.StartTime).Seconds(), + "is_recovery_complete": globalTxManager.IsRecoveryComplete(), + "backup_in_progress": globalTxManager.IsBackupInProgress(), + } +} + +func StopTransactionManager() error { + if globalTxManager == nil { + return nil + } + if globalTxManager.deadlockDetector != nil { + globalTxManager.deadlockDetector.Stop() + } + if globalTxManager.wal != nil { + globalTxManager.wal.Sync() + return globalTxManager.wal.Close() + } + return nil +} + +func HasActiveTransaction() bool { return currentTx.Load() != nil } + +func GetCurrentTransactionID() string { + txVal := currentTx.Load() + if txVal == nil { + return "" + } + tx := txVal.(*Transaction) + return fmt.Sprintf("%d", tx.ID) +} + +func GetActiveTransactions() []TransactionInfo { + if globalTxManager == nil { + return []TransactionInfo{} + } + transactions := make([]TransactionInfo, 0) + globalTxManager.activeTransactions.Range(func(key, value interface{}) bool { + tx := value.(*Transaction) + status := "active" + state := TransactionState(tx.State.Load()) + switch state { + case TransactionCommitted: + status = "committed" + case TransactionAborted: + status = "aborted" + } + tx.mu.RLock() + opCount := len(tx.Operations) + operations := make([]OperationInfo, 0, opCount) + for _, op := range tx.Operations { + operations = append(operations, OperationInfo{ + Type: op.Type, + Database: op.Database, + Collection: op.Collection, + DocumentID: op.DocumentID, + }) + } + savepoints := tx.GetSavepoints() + tx.mu.RUnlock() + info := TransactionInfo{ + ID: fmt.Sprintf("%d", tx.ID), + Status: status, + StartTime: tx.StartTime, + OperationCount: opCount, + Operations: operations, + Savepoints: savepoints, + } + if tx.IsDistributed { + info.Status = "distributed_" + status + info.Nodes = tx.Nodes + } + transactions = append(transactions, info) + return true + }) + return transactions +} + +func GetTransactionByID(id string) (*Transaction, error) { + if globalTxManager == nil { + return nil, fmt.Errorf("transaction manager not initialized") + } + var txID TransactionID + fmt.Sscanf(id, "%d", &txID) + if val, ok := globalTxManager.activeTransactions.Load(txID); ok { + return val.(*Transaction), nil + } + return nil, fmt.Errorf("transaction not found") +} + +func AddToTransaction(coll *Collection, opType string, doc *Document) error { + txVal := currentTx.Load() + if txVal == nil { + return fmt.Errorf("no active transaction") + } + tx := txVal.(*Transaction) + if TransactionState(tx.State.Load()) != TransactionActive { + return fmt.Errorf("transaction is not active") + } + op := Operation{ + Type: opType, + Database: coll.dbName, + Collection: coll.name, + DocumentID: doc.ID, + Data: doc.GetFields(), + Version: doc.Version, + } + tx.mu.Lock() + tx.Operations = append(tx.Operations, op) + tx.mu.Unlock() + LogTransactionAudit(tx.ID, "ADD_OPERATION", TransactionActive, map[string]interface{}{ + "operation": opType, + "document": doc.ID, + }) + return nil +} + +func FindInTransaction(coll *Collection, id string) (*Document, error) { + txVal := currentTx.Load() + if txVal == nil { + return coll.Find(id) + } + tx := txVal.(*Transaction) + tx.mu.RLock() + defer tx.mu.RUnlock() + for i := len(tx.Operations) - 1; i >= 0; i-- { + op := tx.Operations[i] + if op.DocumentID == id { + if op.Type == "delete" { + return nil, fmt.Errorf("document deleted in transaction") + } + if op.Type == "insert" || op.Type == "update" { + doc := NewDocumentWithID(op.DocumentID) + for k, v := range op.Data { + doc.SetField(k, v) + } + doc.Version = op.Version + return doc, nil + } + } + } + if globalTxManager != nil { + if versionDoc := globalTxManager.GetDocumentVersion(id, tx.StartTime); versionDoc != nil { + return versionDoc, nil + } + } + return coll.Find(id) +} + +func MVCCSnapshot() uint64 { return uint64(time.Now().UnixNano()) } + +func CreateDocumentVersion(doc *Document, txID TransactionID) *DocumentVersion { + return &DocumentVersion{ + Document: doc.Clone(), + Timestamp: time.Now().UnixMilli(), + TxID: txID, + } +} + +func BeginTransactionOnCollection(coll *Collection) error { + if globalTxManager == nil { + if err := InitTransactionManager("futriis.wal"); err != nil { + return err + } + } + tx := BeginTransaction() + if tx == nil { + return fmt.Errorf("failed to create transaction") + } + if globalTxManager.logger != nil { + globalTxManager.logger.Debug(fmt.Sprintf("Transaction %d started on collection %s.%s", tx.ID, coll.dbName, coll.name)) + } + return nil +} + +func CheckTransactionTimeout(txID TransactionID) error { + if globalTxManager == nil { + return fmt.Errorf("transaction manager not initialized") + } + val, ok := globalTxManager.activeTransactions.Load(txID) + if !ok { + return fmt.Errorf("transaction not found: %d", txID) + } + tx := val.(*Transaction) + if TransactionState(tx.State.Load()) != TransactionActive { + return nil + } + if tx.timeout > 0 && time.Since(time.UnixMilli(tx.StartTime)) > tx.timeout { + tx.State.Store(int32(TransactionAborted)) + globalTxManager.activeTransactions.Delete(tx.ID) + globalTxManager.stats.TotalTimedOut.Add(1) + globalTxManager.stats.ActiveCount.Add(^uint64(0)) + if tx.timeoutTimer != nil { + tx.timeoutTimer.Stop() + } + LogTransactionAudit(txID, "TIMEOUT_CHECK", TransactionAborted, map[string]interface{}{ + "timeout_ms": tx.timeout.Milliseconds(), + "elapsed_ms": time.Since(time.UnixMilli(tx.StartTime)).Milliseconds(), + }) + return fmt.Errorf("transaction %d timed out", txID) + } + return nil +} + +// ============================================================================= +// ФУНКЦИИ МИГРАЦИИ +// ============================================================================= + +func GetDocumentAtTimestamp(docID string, timestamp int64) (*Document, error) { + if globalTxManager == nil { + return nil, fmt.Errorf("transaction manager not initialized") + } + doc := globalTxManager.GetDocumentVersion(docID, timestamp) + if doc == nil { + return nil, fmt.Errorf("document %s not found at timestamp %d", docID, timestamp) + } + return doc, nil +} + +func GetDocumentVersionsSince(docID string, since int64) ([]*DocumentVersion, error) { + if globalTxManager == nil { + return nil, fmt.Errorf("transaction manager not initialized") + } + val, ok := globalTxManager.documentVersions.Load(docID) + if !ok { + return nil, fmt.Errorf("no versions found for document %s", docID) + } + versions := val.([]*DocumentVersion) + result := make([]*DocumentVersion, 0) + for _, v := range versions { + if v.Timestamp > since { + result = append(result, v) + } + } + return result, nil +} + +func CreateSnapshot(database, collection string) (map[string][]*DocumentVersion, error) { + if globalStorage == nil { + return nil, fmt.Errorf("storage not initialized") + } + db, err := globalStorage.GetDatabase(database) + if err != nil { + return nil, err + } + coll, err := db.GetCollection(collection) + if err != nil { + return nil, err + } + docs := coll.GetAllDocuments() + result := make(map[string][]*DocumentVersion) + for _, doc := range docs { + versions, err := GetDocumentVersionsSince(doc.ID, 0) + if err != nil { + continue + } + if len(versions) > 0 { + result[doc.ID] = versions + } else if globalTxManager != nil { + version := &DocumentVersion{ + Document: doc.Clone(), + Timestamp: time.Now().UnixMilli(), + TxID: TransactionID(0), + VersionID: fmt.Sprintf("%s_%d", doc.ID, time.Now().UnixNano()), + } + globalTxManager.AddDocumentVersion(doc.ID, version) + result[doc.ID] = []*DocumentVersion{version} + } + } + return result, nil +} + +func ApplySnapshot(database, collection string, snapshot map[string][]*DocumentVersion) error { + if globalStorage == nil { + return fmt.Errorf("storage not initialized") + } + db, err := globalStorage.GetDatabase(database) + if err != nil { + return err + } + coll, err := db.GetCollection(collection) + if err != nil { + return err + } + for docID, versions := range snapshot { + if len(versions) == 0 { + continue + } + latest := versions[len(versions)-1] + doc := latest.Document + existing, err := coll.Find(docID) + if err == nil && existing != nil { + updates := doc.GetFields() + if err := coll.Update(docID, updates); err != nil { + return fmt.Errorf("failed to update document %s: %v", docID, err) + } + } else { + if err := coll.Insert(doc); err != nil { + return fmt.Errorf("failed to insert document %s: %v", docID, err) + } + } + if globalTxManager != nil { + for _, v := range versions { + globalTxManager.AddDocumentVersion(docID, v) + } + } + } + return nil +} + +func GetMVCCManager() *MVCCManager { + if globalTxManager == nil { + return nil + } + return globalTxManager.mvccManager +} + +func GetMVCCStats() map[string]interface{} { + if globalTxManager == nil || globalTxManager.mvccManager == nil { + return map[string]interface{}{"error": "MVCC manager not initialized"} + } + return globalTxManager.mvccManager.GetMVCCStats() +} + +func GetVisibilityMapStats() map[string]interface{} { + if globalTxManager == nil || globalTxManager.visibilityMap == nil { + return map[string]interface{}{"error": "Visibility map not initialized"} + } + return globalTxManager.visibilityMap.GetStats() +} + +func GetReadCacheStats() map[string]interface{} { + if globalTxManager == nil || globalTxManager.readCache == nil { + return map[string]interface{}{"error": "Read cache not initialized"} + } + return globalTxManager.readCache.GetStats() +} + +// ============================================================================= +// ВСПОМОГАТЕЛЬНЫЕ ФУНКЦИИ +// ============================================================================= + +func maxUint64(a, b uint64) uint64 { + if a > b { + return a + } + return b +}