n-gram、共起ネットワーク（１）

#環境設定

!pip install janome

!pip install wikipedia

!pip install japanize-matplotlib

!pip install nlplot

#Wikipediaからのテキスト読み込み

from janome.tokenizer import Tokenizer

t = Tokenizer()

import wikipedia

keyword = "\u5728\u5B85\u533B\u7642"#@param{type:"string"}

wikipedia.set_lang("ja")

search_response = wikipedia.search(keyword)

for sr in search_response:

print(search_response.index(sr),sr)

keyword_index=0#@param{type:"number"}

print('読み込みページ：',search_response[keyword_index])

page_data = wikipedia.page(search_response[keyword_index])

docs=page_data.content

docT=docs.replace('\n','').split('。')

#読み込んだ文章の分かち書き（名詞のみ）

from collections import Counter

import collections

import re

word=[]

for d in docT:

dd=""

for token in t.tokenize(d):

if re.match('名詞', token.part_of_speech):

dd+=" "+token.surface

word.append(dd)

import pandas as pd

df=pd.DataFrame(word,columns={"text"})

print(df)

#n-gramの表示

import nlplot

npt = nlplot.NLPlot(df, target_col='text')

ngram=1#@param{type:"integer"}

npt.bar_ngram(

ngram=ngram,

top_n=20,

width=800,

height=600,)

#ストップワード（出現頻度の高すぎる単語の除去）の設定

top_n=0#@param{type:"integer"}

stopwords = npt.get_stopword(top_n=top_n, min_freq=0)

print(stopwords)

#共起語の算出設定

min_edge_frequency=2#@param{type:"integer"}

npt.build_graph(stopwords=stopwords,min_edge_frequency=min_edge_frequency)

display(

npt.node_df.head(npt.node_df.shape[0]), npt.node_df.shape,

npt.edge_df.head(npt.edge_df.shape[0]), npt.edge_df.shape

)

#共起ネットワークの描画

npt.co_network(

title='Co-occurrence network',

width=800,

height=600,)