基于文档智能 Layout 模型合并跨页表格的启发式方法

01

def check_paragraph_presence(paragraphs, start, end):
    """
    Checks if there is a paragraph within the specified range that is not a page header, page footer, or page number. If this were the case, the table would not be a merge table candidate.


    Args:
        paragraphs (list): List of paragraphs to check.
        start (int): Start offset of the range.
        end (int): End offset of the range.


    Returns:
        bool: True if a paragraph is found within the range that meets the conditions, False otherwise.
    """
    for paragraph in paragraphs:
        for span in paragraph.spans:
            if span.offset > start and span.offset < end:
                # The logic role of a parapgaph is used to idenfiy if it's page header, page footer, page number, title, section heading, etc. Learn more: https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/concept-layout?view=doc-intel-4.0.0#document-layout-analysis
                if not hasattr(paragraph, 'role'):
                    return True
                elif hasattr(paragraph, 'role') and paragraph.role not in ["pageHeader", "pageFooter", "pageNumber"]:
                    return True
    return False

def merge_vertical_tables(md_table_1, md_table_2) :
    """
    Merge two consecutive vertical markdown tables into one markdown table.


    Args:
        md_table_1: markdown table 1
        md_table_2: markdown table 2
    
    Returns:
        string: merged markdown table
    """
    table2_without_header = remove_header_from_markdown_table(md_table_2)
    rows1 = md_table_1.strip().splitlines()
    rows2 = table2_without_header.strip().splitlines()


    num_columns1 = len(rows1[0].split(BORDER_SYMBOL)) - 2
    num_columns2 = len(rows2[0].split(BORDER_SYMBOL)) - 2


    if num_columns1 != num_columns2:
        raise ValueError("Different count of columns")


    merged_rows = rows1 + rows2
    merged_table = '\n'.join(merged_rows)


    return merged_table

def merge_horizontal_tables(md_table_1, md_table_2):
    """
    Merge two consecutive horizontal markdown tables into one markdown table.


    Args:
        md_table_1: markdown table 1
        md_table_2: markdown table 2
    
    Returns:
        string: merged markdown table
    """
    rows1 = md_table_1.strip().splitlines()
    rows2 = md_table_2.strip().splitlines()


    merged_rows = []
    for row1, row2 in zip(rows1, rows2):
        merged_row = (
            (row1[:-1] if row1.endswith(BORDER_SYMBOL) else row1)
            + BORDER_SYMBOL
            + (row2[1:] if row2.startswith(BORDER_SYMBOL) else row2)
        )
        merged_rows.append(merged_row)


    merged_table = "\n".join(merged_rows)
    return merged_table

02

该方案具有以下优势，让用户能够在多种场景下轻松处理跨页表：

1.保留表格原始语义：通过获取表的页码、跨度偏移等信息，采用各种复杂技术来确保合并后的表保留数据的原始语义和结构。

2.增强LLM表格理解：Markdown 格式提供了一种创建和格式化表格的简单方法，使LLM更易于阅读和理解表格中的数据。该解决方案以 Markdown 输出的形式合并了表格，进一步增强了 LLM 处理表格数据的能力。

3.简化数据分析处理：无论您需要处理大量文档还是对数据处理有很高的要求，通过该解决方案，分析和处理数据的过程都会被简化。

03

04

# Initiate Azure AI Document Intelligence to load the document. You can either specify file_path or url_path to load the document.
file_path = "<path to your file>"
with open(file_path, 'r', encoding='utf-8') as file:
    markdown_text = file.read()
 
# Split the document into chunks base on markdown headers.
headers_to_split_on = [
    ("#", "Header 1"),
    ("##", "Header 2"),
    ("###", "Header 3"),
]
text_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
 
docs_string = markdown_text
splits = text_splitter.split_text(docs_string)
 
print("Length of splits: " + str(len(splits)))