import argparse
import json
import re
import unicodedata
from collections import Counter
from itertools import combinations
from pathlib import Path
import pandas as pd
from janome.tokenizer import Tokenizer
from sklearn.feature_extraction.text import TfidfVectorizer
from gp_common import Run, load_reviews

# ここを変えてから、このファイル全体を再実行する。古いtokensを使い回さない。
STOPWORDS = {"する", "ある", "いる", "なる", "こと", "もの"}
KEEPWORDS = {"ガチャ", "課金"}
AUTO_MIN_REVIEWS, AUTO_RATIO = 5, 0.8
FONT_PATH = "C:/Windows/Fonts/meiryo.ttc"
NORMALIZATION = "NFKC; URL removal; nouns/adjectives/verbs; base form; not numeric"

def tokenize(text, tokenizer):
    text = re.sub(r"https?://\S+", " ", unicodedata.normalize("NFKC", str(text)))
    words = []
    for token in tokenizer.tokenize(text):
        word = token.base_form if token.base_form != "*" else token.surface
        if token.part_of_speech.split(",")[0] in {"名詞", "形容詞", "動詞"} and not word.isnumeric() and word.strip():
            words.append(word)
    return words

def prepare(path, stopwords=STOPWORDS, keepwords=KEEPWORDS, auto_ratio=AUTO_RATIO):
    df = load_reviews(path)
    tokenizer = Tokenizer()
    df["tokens_raw"] = [tokenize(t, tokenizer) for t in df["content_raw"]]
    nonempty = int(df["content_raw"].str.strip().ne("").sum())
    documents = Counter(w for tokens in df["tokens_raw"] for w in set(tokens))
    auto = {w for w, n in documents.items() if nonempty >= AUTO_MIN_REVIEWS and n / nonempty >= auto_ratio} - keepwords
    excluded = (set(stopwords) | auto) - keepwords
    df["tokens"] = [[w for w in tokens if w not in excluded] for tokens in df["tokens_raw"]]
    df["doc"] = df["tokens"].map(" ".join)
    settings = dict(normalization=NORMALIZATION, stopwords=sorted(stopwords), keepwords=sorted(keepwords),
                    auto_min_reviews=AUTO_MIN_REVIEWS, auto_ratio=auto_ratio, auto_stopwords=sorted(auto),
                    auto_denominator=nonempty, excluded=sorted(excluded))
    return df, settings

def text_analysis(path="outputs/google_play_reviews.csv", root="outputs/gp_runs", stopwords=STOPWORDS):
    df, settings = prepare(path, stopwords=stopwords)
    run = Run("text", path, settings, root)
    audit = df.copy()
    for name in ["tokens_raw", "tokens"]:
        audit[name] = audit[name].map(lambda x: json.dumps(x, ensure_ascii=False))
    run.csv("review_tokens.csv", audit)
    occurrence = Counter(w for tokens in df["tokens"] for w in tokens)
    documents = Counter(w for tokens in df["tokens"] for w in set(tokens))
    terms = pd.DataFrame([(w, n, documents[w]) for w, n in occurrence.most_common()], columns=["term", "occurrences", "review_count"])
    run.csv("term_frequency.csv", terms)
    run.csv("cooccurrence_nodes.csv", terms)
    edges = Counter(pair for tokens in df["tokens"] for pair in combinations(sorted(set(tokens)), 2))
    run.csv("cooccurrence_edges.csv", pd.DataFrame([(a, b, n) for (a, b), n in edges.items()], columns=["source", "target", "review_cooccurrence_count"]))
    if occurrence:
        import math
        import plotly.graph_objects as go
        nodes = [w for w, _ in occurrence.most_common(20)]
        positions = {w: (math.cos(2*math.pi*i/len(nodes)), math.sin(2*math.pi*i/len(nodes))) for i, w in enumerate(nodes)}
        fig = go.Figure()
        for (a, b), n in edges.items():
            if a in positions and b in positions:
                fig.add_trace(go.Scatter(x=[positions[a][0], positions[b][0]], y=[positions[a][1], positions[b][1]],
                    mode="lines", line=dict(width=min(5, n)), hovertemplate=f"{a} / {b}: {n}レビュー<extra></extra>", showlegend=False))
        fig.add_trace(go.Scatter(x=[positions[w][0] for w in nodes], y=[positions[w][1] for w in nodes],
            mode="markers+text", text=nodes, textposition="top center", hovertext=[f"{w}: 延べ{occurrence[w]}回 / {documents[w]}レビュー" for w in nodes], showlegend=False))
        fig.update_layout(title="共起：上位20語（円形の配置に距離の意味はありません）", xaxis=dict(visible=False), yaxis=dict(visible=False))
        run.figure("cooccurrence.html", fig)
    else:
        run.meta["skipped"]["cooccurrence.html"] = "語彙なし"
    rows, statuses = [], []
    for score in range(1, 6):
        group = df.loc[df["score_valid"].eq(score) & df["doc"].ne(""), "doc"].tolist()
        n = len(group); min_df = 1 if n < 5 else 2; max_df = 1.0 if n < 5 else 0.95
        status = "SKIPPED_NO_TOKENS"
        if n:
            try:
                v = TfidfVectorizer(tokenizer=str.split, token_pattern=None, lowercase=False, min_df=min_df, max_df=max_df)
                matrix = v.fit_transform(group)
                means = matrix.mean(axis=0).A1
                for rank, i in enumerate(means.argsort()[::-1][:20], 1):
                    rows.append((score, v.get_feature_names_out()[i], rank, float(means[i]), n))
                status = "OK_EXPLORATORY_SMALL" if n < 5 else "OK"
            except ValueError as exc:
                if not any(t in str(exc).lower() for t in ["empty vocabulary", "no terms remain", "max_df corresponds"]):
                    raise
                status = "SKIPPED_EMPTY_VOCABULARY"
        statuses.append((score, n, min_df, max_df, status))
    run.csv("tfidf.csv", pd.DataFrame(rows, columns=["score", "term", "rank", "mean_tfidf", "document_count"]))
    run.csv("tfidf_status.csv", pd.DataFrame(statuses, columns=["score", "document_count", "min_df", "max_df", "status"]))
    if occurrence and Path(FONT_PATH).is_file():
        try:
            from wordcloud import WordCloud
            WordCloud(font_path=FONT_PATH, width=900, height=500, background_color="white").generate_from_frequencies(occurrence).to_file(str(run.path / "wordcloud.png"))
            run.record("wordcloud.png")
        except ImportError:
            run.meta["skipped"]["wordcloud.png"] = "wordcloud未導入。必要ならpip install wordcloud。CSVは保存済み。"
    else:
        run.meta["skipped"]["wordcloud.png"] = "語彙なし、または日本語フォント未検出。FONT_PATHを実在する日本語フォントへ設定。"
    return run.finish(input_rows=len(df), empty_text=int(df["content_raw"].str.strip().eq("").sum()))

if __name__ == "__main__":
    p = argparse.ArgumentParser(); p.add_argument("input", nargs="?", default="outputs/google_play_reviews.csv")
    text_analysis(p.parse_args().input)