"""
詞頻統計－不含詞性標記，使用jieba斷詞，載入自定義辭典，自訂停用字
"""
import jieba
import jieba.analyse
import csv
import collections

def readFile(fname):
    my_dict = dict()    # 建立空字典
    with open(fname, encoding="UTF-8") as file:
        rows = csv.reader(file)    # 讀取所有內容產生串列
        for r in rows:             # 建立字典
            my_dict[r[0]] = r[1]
    return my_dict

# 步驟一：讀取日記，建立字典、停用字
diary = readFile('Data/MackayFull.csv')
stop_word = []
# 從檔案讀取停用字
with open("Data/stopword.utf8.txt", encoding="UTF-8") as stop:
    rows = stop.readlines()
    for r in rows:
        stop_word.append(r.strip())
jieba.load_userdict('Data/wordsMackay.utf8.txt')    # 載入自定義辭典
# 步驟二：詞頻統計
word_count = dict()    # 統計字典初值設定
for key, value in diary.items():
    words = jieba.cut(value)    # 對每一篇日記斷詞
    for w in words:         # 處理當天的斷詞結果
        if (w in stop_word or w == ' ' or w == '　' or w.isdigit()):    # 移除停用字
            continue
        if (word_count.get(w) == None):  # 詞不在字典裡
            word_count[w] = 1    # 建立字典項目, 詞頻為1
        else:                    # 詞在字典項目裡
            word_count[w] += 1   # 詞頻加1
# 步驟三：重新排序：根據頻率高低
sorted_w = sorted(word_count.items(), key=lambda kv: kv[1], reverse=True)
# 步驟四：寫出檔案
with open('output_notag.csv', 'w', encoding="UTF-8", newline='') as csvfile:    
    writer = csv.writer(csvfile) # 建立 CSV 檔寫入器    
    writer.writerow(['詞彙', '頻率'])  # 寫入標題列
    # 寫入詞頻統計資料
    for item in sorted_w:
        writer.writerow([item[0], item[1]])

### 使用外部模組jieba進行中文斷詞, jieba.posseg標記詞性
import jieba
import jieba.analyse
import jieba.posseg as posseg
import csv
import collections

def readFile(fname):
    my_dict = dict()    # 建立空字典
    with open(fname, encoding="UTF-8") as file:
        rows = csv.reader(file)    # 讀取所有內容產生串列
        for r in rows:             # 建立字典 
            my_dict[r[0]] = r[1]
    return my_dict
# 步驟一：讀取日記，建立字典
diary = readFile('Data/MackayDemo.csv')
stop_word = []
with open("Data/stopword.utf8.txt", encoding="UTF-8") as stop:
    rows = stop.readlines()
    for r in rows:
        stop_word.append(r.strip())
jieba.load_userdict('Data/wordsMackay.utf8.txt')
# 欲省略的詞性列表
pos = {'u': '語助詞', 'r': '代名詞', 'd': '副詞', 'uj': '語助詞', 'c': '連接詞',
       'm': '數詞', 'p': '介詞', 'j': '簡稱', 'y': '語氣詞'}

# 步驟二：詞頻統計
word_count = dict()    # 詞頻統計字典
for key, value in diary.items():
    words = posseg.cut(value)   # 對每一篇日記斷詞，標記詞性
    for w in words:         # 處理每天的斷詞結果
        word = w.word
        if (word in stop_word or word == ' ' or word == '　' or word.isdigit()):    # 移除停用字
            continue
        if (w.flag in pos):  # 忽略的詞性
            continue
        if (word_count.get(w) == None):  # 詞不在字典裡
            word_count[w] = 1    # 建立字典, 詞頻為1
        else:                    # 詞在字典裡
            word_count[w] += 1   # 詞頻加1
# 步驟三：重新排序（根據頻率高低）
sorted_w = sorted(word_count.items(), key=lambda kv: kv[1], reverse=True)  # 排序

# 步驟四：輸出詞頻到檔案
with open('output.csv', 'w', encoding="UTF-8", newline='') as csvfile:
    writer = csv.writer(csvfile)    # 建立 CSV 檔寫入器
    writer.writerow(['詞彙', '頻率', '詞性'])  # 寫入標題列
    for item in sorted_w:    # 寫入統計資料列
        writer.writerow([item[0].word, item[1], item[0].flag])

...
# 步驟一：讀取日記，設定停用字、自定義辭典
diary = readFile('Data/MackayFull.csv')
...

# 共同出現於同一天日記
import jieba
import jieba.analyse
import csv
import collections

def readFile(fname):
    my_dict = dict()    # 建立空字典
    with open(fname, encoding="UTF-8") as file:
        rows = csv.reader(file)    # 讀取所有內容產生串列
        for r in rows:         # 建立字典
            my_dict[r[0]] = r[1]
    return my_dict
# 步驟一：讀取日記，建立字典
diary = readFile('Data/MackayDemo.csv')
stop_word = []       # 1-1 停用字
with open("Data/stopword.utf8.txt", encoding="UTF-8") as stop:
    rows = stop.readlines()
    for r in rows:
        stop_word.append(r.strip())
jieba.load_userdict('Data/wordsMackay.utf8.txt')    # 1-2 自定義辭典
# 步驟二：共同出現統計
word_count = dict()    # 統計字典初值設定
co_occu = dict()
for key, value in diary.items():
    words = jieba.cut(value)    # 對每一篇日記斷詞
    for w1 in words:         # 處理當天的斷詞結果
        if (w1 in stop_word or w1 == ' ' or w1 == '　' or w1.isdigit()):    # 移除停用字
            continue
        for w2 in words:
            if (w2 in stop_word or w2 == ' ' or w2 == '　' or w2.isdigit()):    # 移除停用字
                continue
            if (w1 == w2):
                continue
            w = (w1 + ' ' + w2) if (w1 < w2) else (w2 + ' ' + w1)
            if (word_count.get(w) == None):  # 詞不在字典裡
                word_count[w] = 1    # 建立字典項目, 詞頻為1
            else:                    # 詞在字典項目裡
                word_count[w] += 1   # 詞頻加1
# 步驟三重新排序：根據頻率高低
sorted_w = sorted(word_count.items(), key=lambda kv: kv[1], reverse=True)
# 步驟四：輸出
with open('output_co_occu.csv', 'w', encoding="UTF-8", newline='') as csvfile:
    writer = csv.writer(csvfile)  # 建立 CSV 檔寫入器
    writer.writerow(['詞彙', '頻率'])  # 寫入標題列
    # 寫入詞頻統計資料
    for item in sorted_w:
        writer.writerow([item[0], item[1]])