最近处理一个文本的去重问题,下面是代码
# -*- coding: utf-8 -*- """ Created on Sat Mar 24 13:38:25 2018 @author: Administrator """ import pandas as pd inputfile = 'E:/pythondata/meidi_jd1.txt' #评论文件 outputfile = 'E:/pythonresult/meidi_jd_process_1.txt' #评论处理后保存路径 data = pd.read_csv(inputfile,encoding='utf-8',header = None) l1 = len(data) data = pd.DataFrame(data[0].unique()) l2 = len(data) data.to_csv(outputfile, index = False, header = False,encoding='utf-8') print(u'删除了%s条评论。' %(l1 - l2))# -*- coding: utf-8 -*- "