first commit
This commit is contained in:
brusnitsyn
156
app/config.py
Normal file
156
app/config.py
Normal file
@@ -0,0 +1,156 @@
|
||||
import os
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional
|
||||
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
except ImportError:
|
||||
def load_dotenv(path: str = '.env'):
|
||||
"""Минимальный fallback для локального .env, если python-dotenv еще не установлен."""
|
||||
if not os.path.exists(path):
|
||||
return False
|
||||
with open(path, encoding='utf-8') as env_file:
|
||||
for raw_line in env_file:
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith('#') or '=' not in line:
|
||||
continue
|
||||
key, value = line.split('=', 1)
|
||||
key = key.strip()
|
||||
value = value.strip().strip('"').strip("'")
|
||||
os.environ.setdefault(key, value)
|
||||
return True
|
||||
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
@dataclass
|
||||
class TableMigrationConfig:
|
||||
"""Настройки миграции одной таблицы."""
|
||||
source_table: str
|
||||
target_table: Optional[str] = None
|
||||
mode: str = "full"
|
||||
initial_load_mode: str = "full_then_incremental"
|
||||
life_table: Optional[str] = None
|
||||
datetime_column: str = "x_DateTime"
|
||||
sequence_column: Optional[str] = None
|
||||
order_columns: List[str] = field(default_factory=list)
|
||||
operation_column: Optional[str] = None
|
||||
delete_operations: List[str] = field(default_factory=lambda: ['d'])
|
||||
upsert_operations: List[str] = field(default_factory=lambda: ['i', 'u'])
|
||||
primary_key: List[str] = field(default_factory=list)
|
||||
exclude_columns: List[str] = field(default_factory=list)
|
||||
timescale: bool = False
|
||||
timescale_time_column: Optional[str] = None
|
||||
enabled: bool = True
|
||||
|
||||
@property
|
||||
def pg_table(self) -> str:
|
||||
return (self.target_table or self.source_table).lower()
|
||||
|
||||
@property
|
||||
def read_table(self) -> str:
|
||||
return self.life_table or self.source_table
|
||||
|
||||
@property
|
||||
def incremental_order_columns(self) -> List[str]:
|
||||
if self.order_columns:
|
||||
return self.order_columns
|
||||
columns = [self.datetime_column]
|
||||
if self.sequence_column:
|
||||
columns.append(self.sequence_column)
|
||||
return columns
|
||||
|
||||
# ============================================================================
|
||||
# КОНФИГУРАЦИЯ
|
||||
# ============================================================================
|
||||
class Config:
|
||||
"""Конфигурация приложения"""
|
||||
# Настройки MSSQL
|
||||
MSSQL_CONNECTION_STRING = os.getenv('MSSQL_CONNECTION_STRING')
|
||||
MSSQL_CHARSET = os.getenv('MSSQL_CHARSET', 'cp1251')
|
||||
MSSQL_POOL_RECYCLE = int(os.getenv('MSSQL_POOL_RECYCLE', '1800'))
|
||||
MSSQL_CONNECT_TIMEOUT = int(os.getenv('MSSQL_CONNECT_TIMEOUT', '30'))
|
||||
MSSQL_TABLE_RETRIES = int(os.getenv('MSSQL_TABLE_RETRIES', '2'))
|
||||
MSSQL_POOL_SIZE = int(os.getenv('MSSQL_POOL_SIZE', '1'))
|
||||
MSSQL_MAX_OVERFLOW = int(os.getenv('MSSQL_MAX_OVERFLOW', '0'))
|
||||
|
||||
# Настройки PostgreSQL
|
||||
POSTGRES_CONNECTION_STRING = os.getenv('POSTGRES_CONNECTION_STRING')
|
||||
|
||||
# Настройки email
|
||||
EMAIL_HOST = os.getenv('EMAIL_HOST')
|
||||
EMAIL_PORT = int(os.getenv('EMAIL_PORT', '465'))
|
||||
EMAIL_USER = os.getenv('EMAIL_USER')
|
||||
EMAIL_PASSWORD = os.getenv('EMAIL_PASSWORD')
|
||||
EMAIL_FROM = os.getenv('EMAIL_FROM')
|
||||
EMAIL_TO = [
|
||||
email.strip()
|
||||
for email in os.getenv('EMAIL_TO', '').split(',')
|
||||
if email.strip()
|
||||
]
|
||||
EMAIL_SUBJECT = os.getenv('EMAIL_SUBJECT', 'Результат миграции данных MSSQL → PostgreSQL')
|
||||
|
||||
# Настройки логирования
|
||||
LOG_DIR = 'logs'
|
||||
LOG_FILE = 'migration_log_{timestamp}.log'
|
||||
LOG_LEVEL = logging.INFO
|
||||
|
||||
# Настройки миграции
|
||||
CHUNK_SIZE = int(os.getenv('CHUNK_SIZE', '5000'))
|
||||
WRITE_CHUNK_SIZE = int(os.getenv('WRITE_CHUNK_SIZE', str(CHUNK_SIZE)))
|
||||
BATCH_SIZE = 10 # Через сколько чанков выводить прогресс
|
||||
REPLICATOR_SCHEMA = os.getenv('REPLICATOR_SCHEMA', 'replicator')
|
||||
STATE_TABLE = 'migration_state'
|
||||
TABLE_CONFIGS_TABLE = 'migration_tables'
|
||||
ENABLE_TIMESCALE = os.getenv('ENABLE_TIMESCALE', 'false').lower() == 'true'
|
||||
DRY_RUN = os.getenv('DRY_RUN', 'false').lower() == 'true'
|
||||
READ_LIMIT = int(os.getenv('READ_LIMIT', '0')) or None
|
||||
QUEUE_POLL_SECONDS = float(os.getenv('QUEUE_POLL_SECONDS', '1'))
|
||||
SCHEDULE_GRACE_SECONDS = int(os.getenv('SCHEDULE_GRACE_SECONDS', '60'))
|
||||
START_API_WORKER = os.getenv('START_API_WORKER', 'true').lower() == 'true'
|
||||
CREATE_FOREIGN_KEYS = os.getenv('CREATE_FOREIGN_KEYS', 'true').lower() == 'true'
|
||||
|
||||
# Настройки таблиц. Для инкрементальной миграции заполните life_table
|
||||
# primary_key и exclude_columns, чтобы запись можно было делать идемпотентно.
|
||||
DEFAULT_TABLE_MIGRATIONS = [
|
||||
TableMigrationConfig(
|
||||
source_table='Oms_LPU',
|
||||
target_table='Oms_LPU',
|
||||
mode='incremental',
|
||||
initial_load_mode='full_then_incremental',
|
||||
life_table='Life_oms_LPU',
|
||||
datetime_column='x_DateTime',
|
||||
sequence_column='LPULifeID',
|
||||
order_columns=['x_DateTime', 'LPULifeID'],
|
||||
operation_column='x_Operation',
|
||||
primary_key=['LPUID'],
|
||||
exclude_columns=[
|
||||
'LPULifeID',
|
||||
'x_Operation',
|
||||
'x_DateTime',
|
||||
'x_Seance',
|
||||
'x_User',
|
||||
],
|
||||
timescale=False,
|
||||
enabled=True,
|
||||
),
|
||||
# TableMigrationConfig(
|
||||
# source_table='stt_MigrationPatient',
|
||||
# target_table='stt_MigrationPatient',
|
||||
# mode='incremental',
|
||||
# initial_load_mode='full_then_incremental',
|
||||
# life_table='Life_stt_MigrationPatient',
|
||||
# datetime_column='x_DateTime',
|
||||
# sequence_column='MigrationPatientLifeID',
|
||||
# order_columns=['x_DateTime', 'MigrationPatientLifeID'],
|
||||
# operation_column='x_Operation',
|
||||
# primary_key=['Id'],
|
||||
# exclude_columns=['MigrationPatientLifeID', 'x_Operation', 'x_DateTime', 'x_Seance', 'x_User'],
|
||||
# timescale=True,
|
||||
# timescale_time_column='x_DateTime',
|
||||
# ),
|
||||
]
|
||||
TABLE_MIGRATIONS = list(DEFAULT_TABLE_MIGRATIONS)
|
||||
TABLES_TO_COPY = [table.source_table for table in TABLE_MIGRATIONS]
|
||||
Reference in New Issue
Block a user