sent_compress.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import random\n",
    "import re\n",
    "import codecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn\n",
    "import scipy.stats\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "import sklearn_crfsuite\n",
    "from sklearn_crfsuite import scorers\n",
    "from sklearn_crfsuite import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyltp import Segmentor\n",
    "from pyltp import Postagger\n",
    "from pyltp import SementicRoleLabeller\n",
    "from pyltp import NamedEntityRecognizer\n",
    "from pyltp import Parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with codecs.open('data/orig.org_order.txt',encoding='utf-8') as f:\n",
    "    orig=[]\n",
    "    for line in f:\n",
    "        orig.append(line.strip())\n",
    "with codecs.open('data/manual.del.star.6584.txt',encoding='utf-8') as f:\n",
    "    manual_del=[]\n",
    "    for line in f:\n",
    "        manual_del.append(line.strip())\n",
    "with codecs.open('data/answers.6584.txt',encoding='utf-8') as f:\n",
    "    answer_tp=[]\n",
    "    for line in f:\n",
    "        answer_tp.append(line.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6584 6584 6584\n"
     ]
    }
   ],
   "source": [
    "print(orig.__len__(),manual_del.__len__(),answer_tp.__len__())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6584\n"
     ]
    }
   ],
   "source": [
    "sent_data=[]\n",
    "for i in range(len(orig)):\n",
    "    row=[orig[i],manual_del[i],answer_tp[i]]\n",
    "    sent_data.append(row)\n",
    "print(sent_data.__len__())\n",
    "sent_data_pd=pd.DataFrame(sent_data,columns=['orig','manual_del','ans_tp'],index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>orig</th>\n",
       "      <th>manual_del</th>\n",
       "      <th>ans_tp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>不仅外观上略显累赘</td>\n",
       "      <td>外观 累赘\\t*****\\t不仅\\t*****\\t上略显\\t*****</td>\n",
       "      <td>不仅&lt;Target&gt;外观&lt;/Target&gt;上略显&lt;P&gt;累赘&lt;/P&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>所以在其它相机上常见的紫边现像已经很少了</td>\n",
       "      <td>紫边现像 很少了\\t*****\\t所以在其它相机上常见的\\t*****\\t已经\\t*****</td>\n",
       "      <td>所以在其它相机上常见的&lt;Target&gt;紫边&lt;/Target&gt;现像已经很&lt;P&gt;少&lt;/P&gt;了</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>精良的做工</td>\n",
       "      <td>精 良 的 做 工 \\t*****</td>\n",
       "      <td>&lt;P&gt;精良&lt;/P&gt;的&lt;Target&gt;做工&lt;/Target&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>而且配置了接近完美的手动功能人像</td>\n",
       "      <td>配置了接近完美的手动功能人像\\t*****\\t而且\\t*****</td>\n",
       "      <td>而且配置了接近&lt;P&gt;完美&lt;/P&gt;的&lt;Target&gt;手动功能&lt;/Target&gt;人像</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>快门延时很长</td>\n",
       "      <td>快 门 延 时 很 长 \\t*****</td>\n",
       "      <td>&lt;Target&gt;快门延时&lt;/Target&gt;很&lt;P&gt;长&lt;/P&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>使得拍照的抓拍能力不是很强</td>\n",
       "      <td>拍照的抓拍能力不是很强\\t*****\\t使得\\t*****</td>\n",
       "      <td>使得&lt;Target&gt;拍照的抓拍能力&lt;/Target&gt;不是很&lt;P&gt;强&lt;/P&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>非常清晰的取景器图像</td>\n",
       "      <td>非 常 清 晰 的 取 景 器 图 像 \\t*****</td>\n",
       "      <td>非常&lt;P&gt;清晰&lt;/P&gt;的&lt;Target&gt;取景器图像&lt;/Target&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>白平衡控制方面也不出众</td>\n",
       "      <td>白平衡控制 也不出众\\t*****\\t方面\\t*****</td>\n",
       "      <td>&lt;Target&gt;白平衡&lt;/Target&gt;控制方面也不&lt;P&gt;出众&lt;/P&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>同时小巧的高速快门</td>\n",
       "      <td>小巧的高速快门\\t*****\\t同时\\t*****</td>\n",
       "      <td>同时小巧的&lt;P&gt;高速&lt;/P&gt;&lt;Target&gt;快门&lt;/Target&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>机顶闪光灯位置相当合理</td>\n",
       "      <td>机 顶 闪 光 灯 位 置 相 当 合 理 \\t*****</td>\n",
       "      <td>机顶&lt;Target&gt;闪光灯&lt;/Target&gt;位置相当&lt;P&gt;合理&lt;/P&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    orig                                      manual_del  \\\n",
       "24             不仅外观上略显累赘             外观 累赘\\t*****\\t不仅\\t*****\\t上略显\\t*****   \n",
       "25  所以在其它相机上常见的紫边现像已经很少了  紫边现像 很少了\\t*****\\t所以在其它相机上常见的\\t*****\\t已经\\t*****   \n",
       "26                 精良的做工                               精 良 的 做 工 \\t*****   \n",
       "27      而且配置了接近完美的手动功能人像                配置了接近完美的手动功能人像\\t*****\\t而且\\t*****   \n",
       "28                快门延时很长                             快 门 延 时 很 长 \\t*****   \n",
       "29         使得拍照的抓拍能力不是很强                   拍照的抓拍能力不是很强\\t*****\\t使得\\t*****   \n",
       "30            非常清晰的取景器图像                     非 常 清 晰 的 取 景 器 图 像 \\t*****   \n",
       "31           白平衡控制方面也不出众                    白平衡控制 也不出众\\t*****\\t方面\\t*****   \n",
       "32             同时小巧的高速快门                       小巧的高速快门\\t*****\\t同时\\t*****   \n",
       "33           机顶闪光灯位置相当合理                   机 顶 闪 光 灯 位 置 相 当 合 理 \\t*****   \n",
       "\n",
       "                                          ans_tp  \n",
       "24             不仅<Target>外观</Target>上略显<P>累赘</P>  \n",
       "25  所以在其它相机上常见的<Target>紫边</Target>现像已经很<P>少</P>了  \n",
       "26                 <P>精良</P>的<Target>做工</Target>  \n",
       "27      而且配置了接近<P>完美</P>的<Target>手动功能</Target>人像  \n",
       "28                <Target>快门延时</Target>很<P>长</P>  \n",
       "29         使得<Target>拍照的抓拍能力</Target>不是很<P>强</P>  \n",
       "30            非常<P>清晰</P>的<Target>取景器图像</Target>  \n",
       "31           <Target>白平衡</Target>控制方面也不<P>出众</P>  \n",
       "32             同时小巧的<P>高速</P><Target>快门</Target>  \n",
       "33           机顶<Target>闪光灯</Target>位置相当<P>合理</P>  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent_data_pd[24:34]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "segmentor = Segmentor()  # 初始化实例\n",
    "segmentor.load('../ltp_data/cws.model')  # 加载模型\n",
    "postagger = Postagger() # 初始化实例\n",
    "postagger.load('../ltp_data/pos.model')  # 加载模型\n",
    "parser = Parser() # 初始化实例\n",
    "parser.load('../ltp_data/parser.model')  # 加载模型\n",
    "\n",
    "# print (\"\\t\".join(\"%d:%s\" % (arc.head, arc.relation) for arc in arcs))\n",
    "# parser.release()  # 释放模型\n",
    "def segment_pos(sentence):\n",
    "    words = segmentor.segment(sentence)  # 分词\n",
    "    words_list = list(words)\n",
    "    postags = postagger.postag(words)  # 词性标注\n",
    "    word_pos=[]\n",
    "    arcs = parser.parse(words, postags)  # 句法分析\n",
    "    for it in zip(words_list,postags,arcs):\n",
    "        word_pos.append((it[0],it[1],it[2].head,it[2].relation))\n",
    "#     print (\"\\t\".join(\"%d:%s\" % (arc.head, arc.relation) for arc in arcs))\n",
    "    \n",
    "    # postagger.release()  # 释放模型\n",
    "    # segmentor.release()  # 释放模型\n",
    "    return word_pos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('所以', 'c', 12, 'ADV'),\n",
       " ('在', 'p', 6, 'ADV'),\n",
       " ('其它', 'r', 4, 'ATT'),\n",
       " ('相机', 'n', 5, 'ATT'),\n",
       " ('上', 'nd', 2, 'POB'),\n",
       " ('常见', 'a', 8, 'ATT'),\n",
       " ('的', 'u', 6, 'RAD'),\n",
       " ('紫边', 'n', 9, 'ATT'),\n",
       " ('现像', 'n', 12, 'SBV'),\n",
       " ('已经', 'd', 12, 'ADV'),\n",
       " ('很', 'd', 12, 'ADV'),\n",
       " ('少', 'a', 0, 'HED'),\n",
       " ('了', 'u', 12, 'RAD')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "segment_pos('所以在其它相机上常见的紫边现像已经很少了')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "orig_seg=sent_data_pd['orig'].apply(lambda x: segment_pos(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_data_pd['orig_seg']=orig_seg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_labels(row):\n",
    "    items=row['manual_del'].replace('\\t','').split('*****')[:-1]\n",
    "    orig_seg=row['orig_seg']\n",
    "    new_orig_seg=[]\n",
    "#     word=[it[0] for it in orig_seg]\n",
    "    if len(items)>1:\n",
    "        will_del='\\t'.join(items[1:])\n",
    "        for word in orig_seg:\n",
    "            if word[0] in will_del:\n",
    "                new_orig_seg.append((word[0],word[1],word[2],word[3],'Y'))\n",
    "            else:\n",
    "                new_orig_seg.append((word[0],word[1],word[2],word[3],'N'))\n",
    "    else:\n",
    "        for word in orig_seg:\n",
    "                new_orig_seg.append((word[0],word[1],word[2],word[3],'N'))\n",
    "#     print(items,new_orig_seg)\n",
    "    return new_orig_seg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels=[]\n",
    "for i in range(sent_data_pd.shape[0]):\n",
    "    labels.append(get_labels(sent_data_pd.loc[i]))\n",
    "sent_data_pd['labels']=labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with codecs.open('resource/perception.txt',encoding='utf-8') as f:\n",
    "    percept_words=[line.strip() for line in f]\n",
    "percept_words=set(percept_words)\n",
    "def perception(word):\n",
    "    if word in percept_words:\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with codecs.open('resource/polarity_words_1.txt',encoding='utf-8') as f:\n",
    "    polar_words1=[line.strip() for line in f]\n",
    "with codecs.open('resource/polarity_words_2.txt',encoding='utf-8') as f:\n",
    "    polar_words2=[line.strip() for line in f]\n",
    "polar_words1.extend(polar_words2)\n",
    "polar_word=set(polar_words1)\n",
    "def polarword(word):\n",
    "    if word in polar_word:\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('resource/word2cluster.json') as f:\n",
    "    word2cluster=json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_cluster(word):\n",
    "    try:\n",
    "        cluster=word2cluster[word]\n",
    "    except KeyError:\n",
    "        cluster='0000'\n",
    "    return cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "with codecs.open('resource/baike-50.vec.txt',encoding='utf-8') as f:\n",
    "    word2emb={}\n",
    "    for line in f:\n",
    "        items=line.strip().split()\n",
    "        emb=[float(it) for it in items[1:]]\n",
    "        try:\n",
    "            word2emb[items[0]]=emb\n",
    "        except:\n",
    "            pass       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_embed(word):\n",
    "    embed={}\n",
    "    try:\n",
    "        for ind,it in enumerate(word2emb[word]):\n",
    "            embed[str(ind)]=it\n",
    "    except:\n",
    "        pass\n",
    "    return embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word2feature(sent,i):\n",
    "    sent_len=len(sent)\n",
    "    word=sent[i][0]\n",
    "    postag=sent[i][1]\n",
    "    features={\n",
    "        'word':word,\n",
    "        'pref:word':word[0],\n",
    "        'suff:word':word[-1],\n",
    "        'postag': postag,\n",
    "        'perception':perception(word),\n",
    "        'polar_word':polarword(word),\n",
    "        'cluster':word_cluster(word),\n",
    "\n",
    "        # negative effect    \n",
    "#         'word_head':sent[i][2],\n",
    "#         'word_relation':sent[i][3],\n",
    "    }\n",
    "    if i>0 and i<sent_len-1:\n",
    "        # about word\n",
    "        word1=sent[i-1][0]\n",
    "        word2=sent[i+1][0]\n",
    "        features.update({\n",
    "            '-1:word':word1,\n",
    "            '+1:word':word2,\n",
    "            '-1+word':word1+'|'+word,\n",
    "            '+1+word':word+'|'+word2,\n",
    "        })\n",
    "        # about tag\n",
    "        postag1=sent[i-1][1]\n",
    "        postag2=sent[i+1][1]\n",
    "        features.update({\n",
    "            '-1:postag': postag1,\n",
    "            '+1:postag': postag2,\n",
    "            '-1+postag':postag1+'|'+postag,\n",
    "            '+1+posttag':postag+'|'+postag2,\n",
    "        })\n",
    "    elif i==0 and len(sent)>1:\n",
    "        # about word\n",
    "        word2=sent[i+1][0]\n",
    "        features.update({\n",
    "            '+1:word':word2,\n",
    "            '+1+word':word+'|'+word2,\n",
    "        })\n",
    "        # about tag\n",
    "        postag2=sent[i+1][1]\n",
    "        features.update({\n",
    "            '+1:postag': postag2,\n",
    "            '+1+posttag':postag+'|'+postag2,\n",
    "        })\n",
    "        features['EOS']=True\n",
    "    elif len(sent)>1:\n",
    "        # about word\n",
    "        word1=sent[i-1][0]\n",
    "        features.update({\n",
    "            '-1:word':word1,\n",
    "            '-1+word':word1+'|'+word,\n",
    "        })\n",
    "        # about tag\n",
    "        postag1=sent[i-1][1]\n",
    "        features.update({\n",
    "            '-1:postag': postag1,\n",
    "            '-1+postag':postag1+'|'+postag,\n",
    "        })\n",
    "        features['END']=True\n",
    "    else:\n",
    "        features['EOS']=True\n",
    "        features['END']=True\n",
    "    features.update(add_embed(word))\n",
    "    return features\n",
    "def sent2features(sent):\n",
    "    return [word2feature(sent,i) for i in range(len(sent))]\n",
    "def sent2labels(sent):\n",
    "    return [it[4] for it in sent]\n",
    "def sent2token(sent):\n",
    "    return [it[0] for it in sent]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "percent=0.9\n",
    "data_num=sent_data_pd.shape[0]\n",
    "train_num=int(data_num*percent)\n",
    "random.shuffle(labels)\n",
    "train_sents=labels[:train_num]\n",
    "test_sents=labels[train_num:]\n",
    "x_train=[sent2features(s) for s in train_sents]\n",
    "y_train=[sent2labels(s) for s in train_sents]\n",
    "x_test=[sent2features(s) for s in test_sents]\n",
    "y_test=[sent2labels(s) for s in test_sents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 5.17 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "crf=sklearn_crfsuite.CRF(\n",
    "    algorithm='lbfgs',\n",
    "#     c1=0.20,\n",
    "#     c2=0.14,\n",
    "    c1=1,\n",
    "    c2=2,\n",
    "    max_iterations=100,\n",
    "    all_possible_transitions=True\n",
    "                        )\n",
    "crf.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          N      0.899     0.963     0.930      3688\n",
      "          Y      0.745     0.500     0.598       794\n",
      "\n",
      "avg / total      0.872     0.881     0.871      4482\n",
      "\n"
     ]
    }
   ],
   "source": [
    "label_list=list(crf.classes_)\n",
    "y_pred=crf.predict(x_test)\n",
    "print(metrics.flat_classification_report(\n",
    "    y_test, y_pred, label_list, digits=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "with open('model/CRF_745.mdl','wb') as f:\n",
    "    joblib.dump(crf,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('按键', 'n', 3, 'ATT', 'N'), ('反应', 'v', 3, 'ATT', 'N'), ('速度', 'n', 4, 'SBV', 'N'), ('有', 'v', 0, 'HED', 'N'), ('一点', 'm', 6, 'ADV', 'N'), ('慢', 'a', 4, 'VOB', 'N')]\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'tag'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-73-15e1ee51e2f2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m#     crf=joblib.load(f)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mcrf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict_single\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mE:\\Anaconda3\\lib\\site-packages\\sklearn_crfsuite\\estimator.py\u001b[0m in \u001b[0;36mpredict_single\u001b[1;34m(self, xseq)\u001b[0m\n\u001b[0;32m    365\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    366\u001b[0m         \"\"\"\n\u001b[1;32m--> 367\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtagger_\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtag\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mxseq\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    368\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    369\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpredict_marginals\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'tag'"
     ]
    }
   ],
   "source": [
    "# with open('model/CRF.mdl','rb') as f:\n",
    "#     crf=joblib.load(f)\n",
    "print(labels[0])\n",
    "crf.predict_single(x_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # fine tuning param\n",
    "# # %%time\n",
    "# crf=sklearn_crfsuite.CRF(algorithm='lbfgs',\n",
    "#                         max_iterations=100,\n",
    "#                         all_possible_transitions=True)\n",
    "# params_space={\n",
    "#     'c1':np.linspace(0,10,10),\n",
    "#     'c2':np.linspace(0,10,10),\n",
    "# }\n",
    "# f1_score=make_scorer(metrics.flat_f1_score,average='weighted',labels=label_list)\n",
    "# rs=GridSearchCV(crf,\n",
    "#                      params_space,\n",
    "#                      cv=5,\n",
    "#                      verbose=1,\n",
    "#                      n_jobs=-1,\n",
    "# #                      n_iter=100,\n",
    "# #                      scoring=f1_score\n",
    "#                )\n",
    "# rs.fit(x_train,y_train)\n",
    "# print('best params:', rs.best_params_)\n",
    "# print('best CV score:', rs.best_score_)\n",
    "# print('model size: {:0.2f}M'.format(rs.best_estimator_.size_ / 1000000))\n",
    "# crf = rs.best_estimator_\n",
    "# y_pred = crf.predict(x_test)\n",
    "# print(metrics.flat_classification_report(\n",
    "#     y_test, y_pred, label_list, digits=3\n",
    "# ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "taiyi_train=pd.read_excel('../data/taiyi/train.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "with codecs.open('../data/taiyi/valid.txt',encoding='utf-8') as f:\n",
    "    taiyi_test=[]\n",
    "    for l in f:\n",
    "        items=l.strip().split(',')\n",
    "        taiyi_test.append([items[0],'，'.join(items[1:])])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "for l in taiyi_test:\n",
    "    if len(l)>2:\n",
    "        print(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "taiyi_test=pd.DataFrame(taiyi_test,columns=['row_id','content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>比黑人碳头牙刷好用?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>比之前坏的要好些，亮一些。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>质量很好 用着很舒服 感觉有点小贵啊</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>发货速度很快，东西收到后基本上和描述的没什么差别，质量还行，比较实用，就是感觉有点渣脚，上面...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1、每星期至少两次掉线，一天重启几次也正常；2、5G已坏。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>从中学一直在用的，质量好</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>此产品太差劲了，没有任何内包装，直接放在盒子里就过来了。铅笔也是打开过多次，已经很久了的样子...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>色粉笔也太贵了吧，而且包装也有些旧了，后悔买啊，早知到上实品店买</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>买了跟老公一人一个，用下来还不错，比较静音，充满电能用半个月。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>亚马逊自营的东西是正品，一贯都不错，东西好坏是产品本身的事情。</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  row_id                                            content\n",
       "0      1                                         比黑人碳头牙刷好用?\n",
       "1      2                                      比之前坏的要好些，亮一些。\n",
       "2      3                                 质量很好 用着很舒服 感觉有点小贵啊\n",
       "3      4  发货速度很快，东西收到后基本上和描述的没什么差别，质量还行，比较实用，就是感觉有点渣脚，上面...\n",
       "4      5                      1、每星期至少两次掉线，一天重启几次也正常；2、5G已坏。\n",
       "5      6                                       从中学一直在用的，质量好\n",
       "6      7  此产品太差劲了，没有任何内包装，直接放在盒子里就过来了。铅笔也是打开过多次，已经很久了的样子...\n",
       "7      8                   色粉笔也太贵了吧，而且包装也有些旧了，后悔买啊，早知到上实品店买\n",
       "8      9                    买了跟老公一人一个，用下来还不错，比较静音，充满电能用半个月。\n",
       "9     10                    亚马逊自营的东西是正品，一贯都不错，东西好坏是产品本身的事情。"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "taiyi_test[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>content-评论内容</th>\n",
       "      <th>theme-主题</th>\n",
       "      <th>sentiment_word-情感关键词</th>\n",
       "      <th>sentiment_anls-情感正负面</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>收到了，太实惠了，买了一大箱，以后继续购买，送货速度快服务也好</td>\n",
       "      <td>NULL;送货速度;服务;</td>\n",
       "      <td>实惠;快;也好;</td>\n",
       "      <td>1;1;1;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>热水器加热时间太长，安装费太贵，预留太阳能口摆设，根本用不到，没有水位指示器，加满热水的指示...</td>\n",
       "      <td>加热时间;安装费;用户;</td>\n",
       "      <td>太长;太贵;不方便;</td>\n",
       "      <td>-1;-1;-1;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>真**差，我都无语了。眼瞎才买了</td>\n",
       "      <td>NULL;NULL;</td>\n",
       "      <td>差;无语;</td>\n",
       "      <td>-1;-1;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>很满意，因为出差，还没有用，东西装上我看了，非常好</td>\n",
       "      <td>NULL;NULL;</td>\n",
       "      <td>满意;好;</td>\n",
       "      <td>1;1;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>给奶奶买的，效果不错，厂家服务也好，给好评[追评]</td>\n",
       "      <td>效果;服务;NULL;</td>\n",
       "      <td>不错;也好;好评;</td>\n",
       "      <td>1;1;1;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>还没用，等用了再来追评吧</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>这个中奖有用吗？且不说中奖，这个是我买了这么多一加3t钢化膜中最高清最好看的，没有之一。</td>\n",
       "      <td>钢化膜;NULL;</td>\n",
       "      <td>高清;最好;</td>\n",
       "      <td>1;1;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>不错，家用刚刚好，制冷也不错</td>\n",
       "      <td>NULL;制冷;NULL;</td>\n",
       "      <td>不错;不错;好;</td>\n",
       "      <td>1;1;1;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>电流声真心受不了！而且电脑开机的时候会有爆破声。。。</td>\n",
       "      <td>电流;</td>\n",
       "      <td>受不了;</td>\n",
       "      <td>-1;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>电视挺好，都挺满意</td>\n",
       "      <td>电视;NULL;</td>\n",
       "      <td>挺好;满意;</td>\n",
       "      <td>1;1;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   row_id                                       content-评论内容       theme-主题  \\\n",
       "0       1                    收到了，太实惠了，买了一大箱，以后继续购买，送货速度快服务也好  NULL;送货速度;服务;   \n",
       "1       2  热水器加热时间太长，安装费太贵，预留太阳能口摆设，根本用不到，没有水位指示器，加满热水的指示...   加热时间;安装费;用户;   \n",
       "2       3                                   真**差，我都无语了。眼瞎才买了     NULL;NULL;   \n",
       "3       4                          很满意，因为出差，还没有用，东西装上我看了，非常好     NULL;NULL;   \n",
       "4       5                          给奶奶买的，效果不错，厂家服务也好，给好评[追评]    效果;服务;NULL;   \n",
       "5       6                                       还没用，等用了再来追评吧            NaN   \n",
       "6       7       这个中奖有用吗？且不说中奖，这个是我买了这么多一加3t钢化膜中最高清最好看的，没有之一。      钢化膜;NULL;   \n",
       "7       8                                     不错，家用刚刚好，制冷也不错  NULL;制冷;NULL;   \n",
       "8       9                         电流声真心受不了！而且电脑开机的时候会有爆破声。。。            电流;   \n",
       "9      10                                          电视挺好，都挺满意       电视;NULL;   \n",
       "\n",
       "  sentiment_word-情感关键词 sentiment_anls-情感正负面  \n",
       "0             实惠;快;也好;               1;1;1;  \n",
       "1           太长;太贵;不方便;            -1;-1;-1;  \n",
       "2                差;无语;               -1;-1;  \n",
       "3                满意;好;                 1;1;  \n",
       "4            不错;也好;好评;               1;1;1;  \n",
       "5                  NaN                  NaN  \n",
       "6               高清;最好;                 1;1;  \n",
       "7             不错;不错;好;               1;1;1;  \n",
       "8                 受不了;                  -1;  \n",
       "9               挺好;满意;                 1;1;  "
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "taiyi_train[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sent_list=re.split('，|。|？|！',sent)\n",
    "def sent_split(sent):\n",
    "    sent_list=re.split('，|。|？|！',sent)\n",
    "    sent_list=[it  for it in sent_list if len(it)>1]\n",
    "    return sent_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['清洁功能挺好', '以前就算刚洗完头发不久就有粉粉的头屑', '用这个洗就一点都没有了']"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent_list=sent_split(taiyi_train.loc[24][1])\n",
    "sent_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sent_compress(sentence):\n",
    "    sent_list=sent_split(sentence)\n",
    "    sent_compress_list=[]\n",
    "    sent_no_compress=[]\n",
    "    sent_label=[]\n",
    "    for sent in sent_list:\n",
    "#         print(sent_compress(sent))\n",
    "        sent_comp=[]\n",
    "        sent_pos=segment_pos(sent)\n",
    "        sent_no_compress.append([it[0]for it in sent_pos])\n",
    "        sent_feature=sent2features(sent_pos)\n",
    "        label=crf.predict_single(sent_feature)\n",
    "        sent_label.append(label)\n",
    "        for ind,lb in enumerate(label):\n",
    "            if lb == 'N':\n",
    "                sent_comp.append(sent_pos[ind][0])\n",
    "        sent_compress_list.append(sent_comp)\n",
    "    return (sent_compress_list,sent_no_compress,sent_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sent_compress('第一次用华为手机，特意用了一段时间来评价，包装很严谨，手机速度不错，外观很薄，屏幕大小刚好，黑色很大气，真是高大上，很多功能都不错。下次还会来光顾哦。谢谢客服暮思的耐心讲解')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_press=[]\n",
    "sent_no_press=[]\n",
    "sent_label=[]\n",
    "for ind,row in enumerate(taiyi_train[taiyi_train.columns[1]]):\n",
    "    try:\n",
    "        result=sent_compress(row)\n",
    "        sent_press.append(result[0])\n",
    "        sent_no_press.append(result[1])\n",
    "        sent_label.append(result[2])\n",
    "    except:\n",
    "        sent_press.append([])\n",
    "        sent_no_press.append([])\n",
    "        sent_label.append([])\n",
    "taiyi_train['sent_compress']=sent_press\n",
    "taiyi_train['sent_no_press']=sent_no_press\n",
    "taiyi_train['sent_label']=sent_label\n",
    "# taiyi_train.drop('row_id',axis=1,inplace=True)\n",
    "taiyi_train.to_csv('taiyi_train_compress.csv',index=None,encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_press=[]\n",
    "sent_no_press=[]\n",
    "sent_label=[]\n",
    "for ind,row in enumerate(taiyi_test[taiyi_test.columns[1]]):\n",
    "    try:\n",
    "        result=sent_compress(row)\n",
    "        sent_press.append(result[0])\n",
    "        sent_no_press.append(result[1])\n",
    "        sent_label.append(result[2])\n",
    "    except:\n",
    "        sent_press.append([])\n",
    "        sent_no_press.append([])\n",
    "        sent_label.append([])\n",
    "taiyi_test['sent_compress']=sent_press\n",
    "taiyi_test['sent_no_press']=sent_no_press\n",
    "taiyi_test['sent_label']=sent_label\n",
    "# taiyi_test.drop('row_id',axis=1,inplace=True)\n",
    "taiyi_test.to_csv('taiyi_test_compress.csv',index=None,encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}