diff --git a/dags/pogasheniya.py b/dags/pogasheniya.py index cba3e9c..dddfa98 100644 --- a/dags/pogasheniya.py +++ b/dags/pogasheniya.py @@ -30,62 +30,98 @@ def get_db_engine(): pool_size=10, max_overflow=20 ) - -def pogasheniya(**kwargs): + +def update_pogasheniya_structure_with_detailed_errors(**kwargs): + """ + Версия с подробными сообщениями об ошибках + """ engine = get_db_engine() - query = text(""" - SELECT DISTINCT - osv.schet - , osv.uid_subkonto2 as uid_dogovor - , osv.subkonto2 as name - , osv.nomer - , osv.date_begin - , osv.date_end - FROM oborotno_salbdovaya_vedomostb osv - LEFT JOIN pogasheniya p on p.uid_dogovor = osv.uid_subkonto2 and p.schet = osv.schet - WHERE p.uid_dogovor is null and (osv.schet::text like '%03%' OR osv.schet::text like '%01%') - UNION ALL - SELECT - '76.07.1' as schet - , '00000000-0000-0000-0000-000000000000' as uid_dogovor - , 'Лизинг' as name - , osv.agreement_num as nomer - , osv.agreement_date as date_begin - , osv.redemption_date as date_end - FROM lizingi_garantii osv - LEFT JOIN pogasheniya p on p.nomer = osv.agreement_num - WHERE p.uid_dogovor is null - """) - df = pd.read_sql(query, engine) - with engine.begin() as conn: - if not df.empty: - conn.execute("CREATE TEMP TABLE temp_pogasheniya (schet text null, uid_dogovor text null, name text null, nomer text null, date_begin text null, date_end text null)") - df.to_sql('temp_pogasheniya', con=conn, if_exists='append', index=False, method='multi') - conn.execute(""" - INSERT INTO public.pogasheniya (schet, uid_dogovor, name, nomer, date_begin, date_end) - SELECT * FROM temp_pogasheniya - ON CONFLICT (schet, uid_dogovor, nomer) - DO UPDATE SET - name = EXCLUDED.name, - nomer = EXCLUDED.nomer, - date_begin = EXCLUDED.date_begin, - date_end = EXCLUDED.date_end - """) - # conn.execute(""" - # UPDATE public.pogasheniya fp - # SET id = subquery.new_id - # FROM ( - # SELECT - # id, - # ROW_NUMBER() OVER (ORDER BY uid_dogovor, schet, nomer) as new_id - # FROM public.pogasheniya - # ) AS subquery - # WHERE fp.id = subquery.id; - # """ - # ) - return 'Список обновлен.' - else: - return 'Обновлять нечего.' + + logger.info("Запуск обновления структуры pogasheniya с проверками") + + # Шаг 1: Проверяем существование функции get_years() + try: + with engine.connect() as conn: + # Проверяем, существует ли функция + func_exists = conn.execute(""" + SELECT EXISTS( + SELECT 1 FROM pg_proc + JOIN pg_namespace ON pg_proc.pronamespace = pg_namespace.oid + WHERE proname = 'get_years' + AND nspname = 'public' + ) + """).scalar() + + if not func_exists: + return "ERROR: Функция get_years() не существует в схеме public" + + # Проверяем существование таблицы years_for_pogasheniya + table_exists = conn.execute(""" + SELECT EXISTS( + SELECT 1 FROM information_schema.tables + WHERE table_name = 'years_for_pogasheniya' + AND table_schema = 'public' + ) + """).scalar() + + if not table_exists: + return "ERROR: Таблица years_for_pogasheniya не существует" + + # Проверяем, есть ли данные в таблице + has_data = conn.execute(""" + SELECT EXISTS(SELECT 1 FROM years_for_pogasheniya LIMIT 1) + """).scalar() + + if not has_data: + return "ERROR: Таблица years_for_pogasheniya пустая" + + # Вызываем функцию get_years() + result = conn.execute("SELECT * FROM get_years()").fetchone() + + if result is None: + return "ERROR: Функция get_years() вернула NULL. Проверьте данные в таблице" + + start_year, end_year = result + + # Детальная проверка значений + error_messages = [] + + if start_year is None: + error_messages.append("start_year is NULL") + if end_year is None: + error_messages.append("end_year is NULL") + + if error_messages: + return f"ERROR: {', '.join(error_messages)}" + + # Преобразуем в числа с проверкой + try: + start_year_int = int(start_year) + end_year_int = int(end_year) + except ValueError: + return f"ERROR: Невозможно преобразовать годы в числа: start_year='{start_year}', end_year='{end_year}'" + + if start_year_int > end_year_int: + return f"ERROR: Неправильный порядок лет: start_year ({start_year}) > end_year ({end_year})" + + logger.info(f"Валидные настройки лет: {start_year}-{end_year}") + + except Exception as e: + return f"ERROR: Ошибка при проверке данных: {str(e)}" + + # Шаг 2: Обновляем структуру + try: + with engine.begin() as conn: + update_result = conn.execute( + "SELECT update_years_and_rebuild(%s, %s)", + (str(start_year), str(end_year)) + ).scalar() + + return f"SUCCESS: {update_result}" + + except Exception as e: + return f"ERROR: Ошибка при пересоздании таблицы: {str(e)}" + with DAG( dag_id='update_pogasheniya', @@ -97,8 +133,8 @@ with DAG( ) as dag: pogasheniya_task = PythonOperator( - task_id='pogasheniya', - python_callable=pogasheniya, + task_id='update_pogasheniya_structure_with_detailed_errors', + python_callable=update_pogasheniya_structure_with_detailed_errors, provide_context=True )