python Automatic-Corpus-Generation sgml文件解析为csv

最近需要用到文本纠错的数据集，但是是sgml格式的，类似xml格式的，我这里顺手用beautifulsoup解析了一下，核心代码还是来自​​https://github.com/wdimmy/Automatic-Corpus-Generation​​，但是我修改了一下，分享一下代码：

import logging
import os
import codecs
from tqdm import tqdm
from bs4 import BeautifulSoup 
import pandas as pd

def replace_char(string,char,index):
    string = list(string)
    string[index] = char
    return ''.join(string)

def read_langs(file_name):

    logging.info(("Reading lines from {}".format(file_name)))
    total_data=[]

    with codecs.open(file_name, "r", "utf-8") as file:

        data = file.read()
        soup = BeautifulSoup(data, 'html.parser')
        results = soup.find_all('sentence')
        for item in tqdm(results):

            text = item.find("text").text.strip()
            correct_text=text
            mistakes = item.find_all("mistake")

            locations = []
            for mistake in mistakes:
                location = mistake.find("location").text.strip()
                wrong =  mistake.find("wrong").text.strip()
                correction_word=mistake.find('correction').text.strip()
                correct_text=replace_char(correct_text,correction_word,int(location)-1)
                locations.append(int(location))
                if text[int(location)-1] != wrong:
                    print("The character of the given location does not equal to the real character")

            sen = list(text)
            tags = ["0" for _ in range(len(sen))]

            for i in locations:
                tags[i - 1] = "1"
            total_data.append([correct_text,text, " ".join(tags)])

    return total_data

file_name='train.sgml'
total_data=read_langs(file_name)




column_name = ['origin_text','random_text','label']
csv_name='Automatic-Corpus-Generation.csv'
xml_df = pd.DataFrame(total_data, columns=column_name)
xml_df.to_csv(csv_name, index=None)

把它的train.sgml下载下来，就可以开开心心的跑代码了，哈哈哈。

参考文献

[1].A Hybrid Approach to Automatic Corpus Generation for Chinese Spelling Checking (EMNLP2018). ​​https://github.com/wdimmy/Automatic-Corpus-Generation​​