단어 벡터를 문장 벡터로 바꾸는 텍스트 유사도 계산

# coding: utf-8

# In[2]:
### 
from gensim.models import word2vec
w2v=word2vec.Word2Vec.load('d:/chat_data/corpus_vector.model')


## 
import jieba
import re
raw_data = []
w = open('******','r',encoding= 'utf-8')
for line in w.readlines():
    newline = line.strip()
    newline = re.sub(' ','',newline)
    newline = jieba.cut(newline)
    raw_data.append(list(newline))    
w.close()



import numpy as np


# In[72]:
### 
def sent2vec(s):
    words = s
    M = []
    for w in words:
        try:
            M.append(w2v.wv[w])
        except:
            continue
    M = np.array(M)
    v = M.sum(axis=0)
    return v / np.sqrt((v ** 2).sum())

newdata = []
newdata_dict = {}
seed = 0  
for word in raw_data:
    try:
        newline = sent2vec(word)
        if len(newline)<300:
            continue
        newdata.append(newline)
        times += 1
        newdata_dict[tuple(newline)] = ''.join(word)

    except:
        continue



#### 
ws = open('******','r',encoding='gbk')
times = 0
import re
import jieba  
standard_data = []
from zhon.hanzi import punctuation
for i in ws.readlines():
    times += 1
    if times == 1:
        continue
    newline = i.strip().split(',')
    newline = re.sub("[A-Za-z0-9\[\`\~\!\@\#\$\^\&\*\(\)\=\|\{\}\'\:\;\'\,\[\]\.\\/\?\~\！\@\#\\\&\*\%\-\_]", "", newline[0])
    newline = re.sub(' ','',newline)
    newline = re.sub("[%s]+" %punctuation, "", newline)
    standard_data.append(list(jieba.cut(newline)))
ws.close()





standard_newdata = []
new_standard_dict = {}
for word in standard_data:
    try:
        newline = sent2vec(word)
        if len(newline)<300:
            continue
        standard_newdata.append(newline)
        new_standard_dict[tuple(newline)] = ''.join(word)

    except:
        continue





# In[45]:

from gensim import corpora, models, similarities


# In[114]:

#### 
import math

def cos_dist(a, b):
    if len(a) != len(b):
        return None
    part_up = 0.0
    a_sq = 0.0
    b_sq = 0.0
    for a1, b1 in zip(a,b):
        part_up += a1*b1
        a_sq += a1**2
        b_sq += b1**2
    part_down = math.sqrt(a_sq*b_sq)
    if part_down == 0.0:
        return None
    else:
        return part_up / part_down


# In[ ]:
#### 
result_data = {}
for i in standard_newdata:
    for j in newdata:
                result_data[new_standard_dict[tuple(i)]+'\t'+newdata_dict[tuple(j)]] = cos_dist(i,j)