# 安裝所需套件
pip install scikit-learn pandas

# 安裝jieba斷詞器台灣繁體特化版本
pip install git+https://github.com/APCLab/jieba-tw.git

import jieba
from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer
import pandas as pd

# Step1: 斷詞
# docs = ['我喜歡喝綠茶', '我喜歡喝奶茶', '我不喜歡喝紅茶但喜歡喝奶茶']
docs = [
    '新莊的輔仁大學在新莊的中正路上', 
    '台北的臺灣師範大學在台北的和平東路上', 
    '新竹的交通大學在新竹的大學路上', 
    '嘉義的嘉義大學在嘉義的學府路上', 
    '台北的政治大學在台北的指南路上', 
    '新竹的清華大學在新竹的光復路上'
]
docs_seg = []
for doc in docs:
    seg = jieba.cut(doc)
    docs_seg.append(' '.join(seg))


# Step2: 建立計算每個term在doc有多少個
# 加入tokenizer參數，覆寫原有的tokenize程序，否則單一個中文字會被自動審略掉
# 參考 https://github.com/scikit-learn/scikit-learn/issues/7251#issuecomment-242897897
vectorizer = CountVectorizer(tokenizer=lambda x: x.split()) 
text_count_vector = vectorizer.fit_transform(docs_seg)

print('============全部字詞=============')
print(vectorizer.get_feature_names())
print()
print('==========各文件字詞統計==============')
print(text_count_vector.toarray())
print()


# Step3: 計算TFIDF
tfidf_transfomer = TfidfTransformer()
docs_tfidf = tfidf_transfomer.fit_transform(text_count_vector)

print('==============IDF============')
print(tfidf_transfomer.idf_)
print()

df = pd.DataFrame(docs_tfidf.T.toarray(), index=vectorizer.get_feature_names())
print('============TFIDF============')
print(df)
print()


# Step4: 關鍵字萃取(印出各文件tfidf數值top 3)
print('============關鍵字萃取============')
for i, vector in enumerate(docs_tfidf.toarray()):
    doc_tfidf = list(zip(vectorizer.get_feature_names(), vector))
    doc_tfidf = sorted(doc_tfidf, key=lambda x: x[1], reverse=True)
    top3_keywords = [x[0] for x in doc_tfidf[:3]]
    print('doc' + str(i+1) + ": " + '、'.join(top3_keywords))

import jieba
from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import pandas as pd

# Step1: 斷詞
# docs = ['我喜歡喝綠茶', '我喜歡喝奶茶', '我不喜歡喝紅茶但喜歡喝奶茶']
docs = [
    '新莊的輔仁大學在新莊的中正路上', 
    '台北的臺灣師範大學在台北的和平東路上', 
    '新竹的交通大學在新竹的大學路上', 
    '嘉義的嘉義大學在嘉義的學府路上', 
    '台北的政治大學在台北的指南路上', 
    '新竹的清華大學在新竹的光復路上'
]
docs_seg = []
for doc in docs:
    seg = jieba.cut(doc)
    docs_seg.append(' '.join(seg))


# Step2: 建立計算每個term在doc有多少個
# 加入tokenizer參數，覆寫原有的tokenize程序，否則單一個中文字會被自動審略掉
# 參考 https://github.com/scikit-learn/scikit-learn/issues/7251#issuecomment-242897897
vectorizer = CountVectorizer(tokenizer=lambda x: x.split()) 
text_count_vector = vectorizer.fit_transform(docs_seg)

print('============全部字詞=============')
print(vectorizer.get_feature_names())
print()
print('==========各文件字詞統計==============')
print(text_count_vector.toarray())
print()


# Step3: 計算TFIDF
tfidf_transfomer = TfidfTransformer()
docs_tfidf = tfidf_transfomer.fit_transform(text_count_vector)

print('==============IDF============')
print(tfidf_transfomer.idf_)
print()

df = pd.DataFrame(docs_tfidf.T.toarray(), index=vectorizer.get_feature_names())
print('============TFIDF============')
print(df)
print()


# Step4: 計算query與文件們的相似度，並列出前三相似之文件
query = '台北'
query_seg = ' '.join(jieba.cut(query))
query_count_vector = vectorizer.transform([query_seg])
query_tfidf = tfidf_transfomer.transform(query_count_vector)
similarities = cosine_similarity(query_tfidf, docs_tfidf).flatten()
results = zip(docs, similarities)
results = sorted(results, key=lambda x: x[1], reverse=True)

print('============計算相似度============')
print('query:', query)
print()
for i in range(3):
    print('doc' + str(i+1) + ': ', results[i][1])
    print(results[i][0])
    print()