数据预处理

#!/usr/bin/env python
# coding=utf-8

from string import punctuation
import re
import codecs

# 英文标点符号+中文标点符号
# 未去除空格、换行符等（正则表达式以"/s"表示）
punc = punctuation + u'.,;《》？！“”‘’@#￥%…&×（）——+【】{};；●，。&～、|:：'


fr = codecs.open("……\YWP_EventExtraction_CRF_Elephant\01 Preprocessing\20220107 test data.txt", "r", encoding='utf-8')
fw = codecs.open("……\YWP_EventExtraction_CRF_Elephant\01 Preprocessing\20220107 test data_result.txt", "w", encoding='utf-8')

# 利用正则表达式替换为一个空格
for line in fr:
    line = re.sub(r"[{}]+".format(punc)," ",line)
    fw.write(line+' ')

fr.close()
fw.close()

#!/usr/bin/env python
# coding=utf-