#!/usr/bin/env python3 """Test complex merged cell normalization from user example.""" from uuid import uuid4 from src.extraction_v2.html_table_extractor import HtmlTableExtractor from src.extraction_v2.llm_header_detector import LlmHeaderDetector from src.extraction_v2.record_builder import RecordBuilder def test_complex_merged_cells(): """Test complex table with extensive merged cells from user example.""" print("=" * 80) print("COMPLEX MERGED CELL TEST (User Example)") print("=" * 80) # User's example HTML html = """
条件自動更新可否
SO工事直請区分直請設定（SO工事） 情報設定値 *1記事欄 （お客様情報）ＳＯ工事予定日発注納期チェック
'直営''直営''支店設定文言あり' *2－－'否'
'支店設定文言なし' *2null以外ＳＯ工事予定日≧システム日付+Ａ日 *3'可'
ＳＯ工事予定日＜システム日付+Ａ日 *3'否'
上記以外－'否'
'請負'－－'否'
上記以外'否'
'請負''直営'－－－'否'
'請負''支店設定文言あり' *2－－'否'
'支店設定文言なし' *2null以外ＳＯ工事予定日≧システム日付+Ａ日 *3'可'
ＳＯ工事予定日＜システム日付+Ａ日 *3'否'
上記以外－'否'
上記以外－－'否'
上記以外－－－－'否'
""" print("\n1. Extracting tables from HTML...") extractor = HtmlTableExtractor() tables = extractor.extract_tables(html) if not tables: print("ERROR: No tables found") return table = tables[0] print(f" Table extracted: {table.row_count} rows x {table.col_count} cols") print("\n2. Raw 2D grid (first 10 rows, showing merged cell expansion):") for i, row in enumerate(table.rows[:10]): print(f" Row {i:2d}: {row[:10]}... (showing first 10 cols)") print(f"\n3. Detecting headers with LLM...") detector = LlmHeaderDetector() headers = detector.detect_headers(table.html) print(f" Detected {len(headers.header_rows)} header row(s)") print(f" Header rows: {headers.header_rows}") print(f" Number of columns: {len(headers.headers)}") flat_headers = headers.get_flat_headers() print(f"\n4. Flat headers (first 10):") for i, h in enumerate(flat_headers[:10]): print(f" {i}: '{h}'") if len(flat_headers) > 10: print(f" ... and {len(flat_headers) - 10} more") print(f"\n5. Building records with merged cell normalization...") builder = RecordBuilder() records = builder.build_records( table=table, headers=headers, document_id=uuid4(), sheet_name="Test" ) print(f" Built {len(records)} records") print(f"\n6. Examining records (checking for empty values):") for i, record in enumerate(records[:5]): # Show first 5 records print(f"\n Record {i + 1}:") print(f" Content keys: {list(record['content'].keys())}") print(f" Content values (first 5):") for key, value in list(record['content'].items())[:5]: print(f" '{key}': '{value}'") # Check for empty strings empty_count = sum(1 for v in record['content'].values() if v == '') non_empty_count = len(record['content']) if empty_count > 0: print(f" ⚠️ WARNING: Found {empty_count} empty string values!") else: print(f" ✓ Clean: {non_empty_count} non-empty values, no empty strings") if len(records) > 5: print(f"\n ... and {len(records) - 5} more records") print("\n" + "=" * 80) print("TEST COMPLETE") print("=" * 80) # Summary print(f"\nSummary:") print(f" - Table size: {table.row_count} rows x {table.col_count} cols") print(f" - Header rows: {headers.header_rows}") print(f" - Columns detected: {len(headers.headers)}") print(f" - Records extracted: {len(records)}") # Count empty strings across all records total_empty = sum( sum(1 for v in record['content'].values() if v == '') for record in records ) total_values = sum(len(record['content']) for record in records) print(f" - Total values stored: {total_values}") print(f" - Empty strings: {total_empty}") if total_empty == 0: print(f"\n✓ SUCCESS: No empty strings in any records!") else: print(f"\n⚠️ WARNING: Found {total_empty} empty strings across all records") if __name__ == "__main__": test_complex_merged_cells()