#!/usr/bin/env python3
"""Generate a self-contained, narrative market-opportunity report for one
JTBD pipeline run.

Reads a project run's output CSVs (e.g. reddit_overwatering_comments_*),
synthesizes them into a readable HTML report oriented at the question
"is there a market here, what's the opportunity, should I build?", and writes
it to reports/<prefix>_market_report.html.

Usage:
    python3 generate_market_report.py [prefix]   # default: first discovered run
    python3 generate_market_report.py --all      # one report per discovered run
"""
import csv
import glob
import html
import json
import os
import re
import sys
from collections import Counter, defaultdict

SUFFIX = "_jtbd_comments__archetype_summary.csv"
BASE_DIR = os.path.dirname(os.path.abspath(__file__))


# ----------------------------------------------------------------------
# CSV reading
# ----------------------------------------------------------------------
def read_rows(path):
    with open(path, newline="", encoding="utf-8") as f:
        return list(csv.DictReader(f))


def discover_datasets():
    prefixes = set()
    for path in glob.glob(os.path.join(BASE_DIR, "*" + SUFFIX)):
        name = os.path.basename(path)
        prefix = name[: -len(SUFFIX)]
        if prefix:
            prefixes.add(prefix)
    return sorted(prefixes)


def load(prefix):
    base = f"{prefix}_jtbd_comments"
    paths = {
        "raw": f"{prefix}.csv",
        "summary": f"{base}__archetype_summary.csv",
        "opp": f"{base}__jtbd_archetypes_value_opportunity_map.csv",
        "polarity": f"{base}__jtbd_archetypes_value_opportunity_polarity_map.csv",
        "readiness": f"{base}__jtbd_archetypes_opportunity_change_readiness_summary.csv",
        "cross": f"{base}__cross_archetype_comparison.csv",
        "outcomes": f"{base}__jtbd_archetypes_value_opportunity_outcomes.csv",
        "struggles": f"{base}__jtbd_archetypes_value_opportunity_struggles.csv",
        "transition": f"{base}__transition_probs_situation_struggle_outcome.csv",
        "arch_rows": f"{base}__jtbd_archetypes.csv",
        "pain_clusters": f"{prefix}_pain_points_painpoint_clusters.csv",
        "pain_points": f"{prefix}_pain_points.csv",
        "approach_clusters": f"{base}__current_approach_clusters.csv",
        "approach_themes": f"{base}__current_approach_themes.csv",
        "hesitation_clusters": f"{base}__hesitation_clusters.csv",
        "hesitation_themes": f"{base}__hesitation_themes.csv",
        "vo_full": f"{base}__jtbd_archetypes_jtbd_value_opportunity_full.csv",
        "trigger_verbs": f"{base}__trigger_verbs.csv",
        "trigger_stages": f"{base}__trigger_stages.csv",
        "audience_forces": f"{prefix}_audience_forces.csv",
    }
    data = {k: read_rows(os.path.join(BASE_DIR, p)) for k, p in paths.items() if os.path.exists(os.path.join(BASE_DIR, p))}
    data["prefix"] = prefix
    return data


# ----------------------------------------------------------------------
# Helpers
# ----------------------------------------------------------------------
def fnum(x):
    try:
        return float(x)
    except (TypeError, ValueError):
        return None


def subreddit_from_url(url):
    m = re.search(r"reddit\.com/(r/[^/]+)", url or "")
    return m.group(1) if m else ""


def esc(s):
    return html.escape("" if s is None else str(s), quote=True)


def fmt(v, nd=2):
    if v is None or v == "":
        return "—"
    try:
        return f"{float(v):.{nd}f}"
    except (TypeError, ValueError):
        return str(v)


def build_raw_lookup(raw_rows):
    lookup = {}
    for r in raw_rows:
        c = r.get("comment", "")
        if c and c not in lookup:
            lookup[c] = {
                "upvotes": r.get("upvotes", ""),
                "post_url": r.get("post_url", ""),
                "author": r.get("author", ""),
            }
    return lookup


_QUOTE_TRANS = str.maketrans({
    "\u2019": "'", "\u2018": "'", "\u02bc": "'",
    "\u201c": '"', "\u201d": '"',
    "\u2013": "-", "\u2014": "-", "\u2212": "-",
    "\u00a0": " ",
})


def _norm(s):
    """Length-preserving punctuation normalization so LLM-extracted quote
    substrings (straight quotes) match the raw comment (curly quotes)."""
    return str(s).translate(_QUOTE_TRANS)


def highlight_comment(comment, quotes):
    """Wrap each pain-point quote substring in <mark>, preserving everything
    else HTML-escaped. Matching is done on punctuation-normalized text so
    curly/straight apostrophes match, then mapped back 1:1 to the raw text."""
    comment = "" if comment is None else str(comment)
    c_norm = _norm(comment)
    spans = []
    for q in quotes:
        q = _norm(q.strip())
        if not q:
            continue
        start = 0
        while True:
            i = c_norm.find(q, start)
            if i < 0:
                break
            spans.append((i, i + len(q)))
            start = i + len(q)
    if not spans:
        return html.escape(comment)
    spans.sort()
    merged = []
    for s, e in spans:
        if merged and s <= merged[-1][1]:
            merged[-1] = (merged[-1][0], max(merged[-1][1], e))
        else:
            merged.append((s, e))
    out = []
    prev = 0
    for s, e in merged:
        out.append(html.escape(comment[prev:s]))
        out.append("<mark>")
        out.append(html.escape(comment[s:e]))
        out.append("</mark>")
        prev = e
    out.append(html.escape(comment[prev:]))
    return "".join(out)


def highlight_words(text, words):
    """Wrap whole-word occurrences of `words` in <mark>, preserving everything
    else HTML-escaped. Used to show the shared vocabulary a job cluster was
    clustered on, inside each member job statement."""
    text = "" if text is None else str(text)
    words = [w for w in (words or []) if w]
    if not text or not words:
        return html.escape(text)
    pattern = re.compile(
        r"\b(" + "|".join(re.escape(w) for w in sorted(words, key=len, reverse=True)) + r")\b",
        re.IGNORECASE,
    )
    out = []
    last = 0
    for m in pattern.finditer(text):
        out.append(html.escape(text[last:m.start()]))
        out.append("<mark>" + html.escape(m.group(0)) + "</mark>")
        last = m.end()
    out.append(html.escape(text[last:]))
    return "".join(out)


_STOPWORDS = set(
    "the and to a of in for on at with an is it that this so be as by or are was "
    "were from but have has had when i me my you your they their them we us our "
    "just really like get got make makes made much many very more less lot lots "
    "kind sort maybe".split()
)


def top_words_of(texts, n=8):
    words = []
    for t in texts:
        words += re.findall(r"[a-z]{3,}", (t or "").lower())
    words = [w for w in words if w not in _STOPWORDS]
    return ", ".join(w for w, _ in Counter(words).most_common(n))


def build_quote_lookup(pain_points):
    """comment text -> list of exact quote substrings (pain-point extraction)."""
    lookup = {}
    for r in pain_points:
        c = r.get("comment", "")
        if c and c not in lookup:
            lookup[c] = [p.strip() for p in (r.get("quotes", "") or "").split("||") if p.strip()]
    return lookup


# ----------------------------------------------------------------------
# Synthesis
# ----------------------------------------------------------------------
def synthesize(data):
    d = data
    raw_lookup = build_raw_lookup(d["raw"])
    quote_lookup = build_quote_lookup(d.get("pain_points", []))
    key = str

    vo_by_comment = {}
    for r in d.get("vo_full", []):
        c = r.get("comment", "")
        if c and c not in vo_by_comment:
            vo_by_comment[c] = r

    def theme_map(rows, theme_col):
        m = {}
        for r in rows:
            c = r.get("comment", "")
            if c and c not in m:
                m[c] = r.get(theme_col, "")
        return m

    approach_theme_by_comment = theme_map(d.get("approach_themes", []), "current_approach_theme")
    hesitation_theme_by_comment = theme_map(d.get("hesitation_themes", []), "hesitation_theme")
    trigger_verb_by_comment = theme_map(d.get("trigger_verbs", []), "trigger_verb")
    trigger_phrase_by_comment = theme_map(d.get("trigger_verbs", []), "trigger_phrase")

    audience_by_comment = {}
    for r in d.get("audience_forces", []):
        c = r.get("comment", "")
        if c and c not in audience_by_comment:
            emotions = [e.strip().lower() for e in (r.get("emotional_drivers", "") or "").split(";") if e.strip()]
            audience_by_comment[c] = {
                "emotions": [e for e in emotions if e != "neutral"],
                "push": r.get("push", "") or "",
                "pull": r.get("pull", "") or "",
                "anxieties": r.get("anxieties", "") or "",
                "habits": r.get("habits", "") or "",
            }

    stage_by_phrase = {}
    for r in d.get("trigger_stages", []):
        p = (r.get("trigger_phrase", "") or "").strip()
        s = (r.get("journey_stage", "") or "").strip()
        if p and s:
            stage_by_phrase[p] = s

    opp_by_id = {}
    for r in d["opp"]:
        opp_by_id[key(r.get("archetype_cluster"))] = r
    pol_by_id = {}
    for r in d["polarity"]:
        pol_by_id[key(r.get("archetype_cluster"))] = r

    segments = []
    for s in d["summary"]:
        sid = key(s.get("archetype_id"))
        o = opp_by_id.get(sid, {})
        p = pol_by_id.get(sid, {})
        segments.append({
            "id": sid,
            "label": s.get("label", ""),
            "n": int(s.get("n_examples") or 0),
            "top_words": s.get("top_words", ""),
            "example": s.get("example", ""),
            "importance": fnum(o.get("importance")),
            "satisfaction": fnum(o.get("satisfaction")),
            "opportunity": fnum(o.get("opportunity")),
            "readiness": fnum(p.get("semantic_polarity")),
            "change_type": p.get("change_type", ""),
        })
    segments.sort(key=lambda x: (x["opportunity"] is None, -(x["opportunity"] or 0)))

    # under-served outcomes and struggles (opportunity desc)
    def rank(list_rows, text_key):
        out = []
        for r in list_rows:
            out.append({
                "text": r.get(text_key, ""),
                "importance": fnum(r.get("importance_score")),
                "satisfaction": fnum(r.get("satisfaction_score")),
                "opportunity": fnum(r.get("opportunity")),
            })
        out.sort(key=lambda x: (x["opportunity"] is None, -(x["opportunity"] or 0)))
        return out[:10]

    top_outcomes = rank(d["outcomes"], "outcome")
    top_struggles = rank(d["struggles"], "struggle")

    # evidence quotes (joined with upvotes)
    # top words per outcome theme (for the job-cluster shared-vocabulary highlight)
    outcome_top_words = {}
    for r in d["arch_rows"]:
        o = r.get("outcome_theme", "")
        if o:
            outcome_top_words.setdefault(o, []).append(r.get("outcome", ""))

    arch_rows = []
    for r in d["arch_rows"]:
        hit = raw_lookup.get(r.get("comment", ""), {})
        upv = None
        try:
            upv = int(hit.get("upvotes")) if hit.get("upvotes") not in (None, "") else None
        except (TypeError, ValueError):
            upv = None
        comment = r.get("comment", "")
        vo = vo_by_comment.get(comment, {})
        audience = audience_by_comment.get(comment, {})
        outcome_theme = r.get("outcome_theme", "")
        job_words = [w for w in top_words_of(outcome_top_words.get(outcome_theme, []), 8).split(", ") if w]
        arch_rows.append({
            "comment": comment,
            "situation": r.get("situation", ""),
            "struggle": r.get("struggle", ""),
            "outcome": r.get("outcome", ""),
            "archetype": r.get("archetype_label", ""),
            "archetype_cluster": r.get("archetype_cluster", ""),
            "upvotes": upv,
            "author": hit.get("author", ""),
            "post_url": hit.get("post_url", ""),
            "subreddit": subreddit_from_url(hit.get("post_url", "")),
            "current_approach": r.get("current_approach", ""),
            "hesitation": r.get("hesitation", ""),
            "job_statement": r.get("job_statement", ""),
            "job_hl": highlight_words(r.get("job_statement", ""), job_words),
            "situation_theme": r.get("situation_theme", ""),
            "struggle_theme": r.get("struggle_theme", ""),
            "outcome_theme": outcome_theme,
            "trigger_verb": trigger_verb_by_comment.get(comment, ""),
            "trigger_phrase": trigger_phrase_by_comment.get(comment, ""),
            "journey_stage": stage_by_phrase.get(trigger_phrase_by_comment.get(comment, ""), ""),
            "hl": highlight_comment(comment, quote_lookup.get(comment, [])),
            "current_approach_theme": approach_theme_by_comment.get(comment, ""),
            "hesitation_theme": hesitation_theme_by_comment.get(comment, ""),
            "job_cluster": outcome_theme,
            "satisfaction": fnum(vo_by_comment.get(comment, {}).get("satisfaction")),
            "opportunity_row": fnum(vo_by_comment.get(comment, {}).get("opportunity")),
            "importance": fnum(vo_by_comment.get(comment, {}).get("importance")),
            "motivation_polarity": fnum(vo_by_comment.get(comment, {}).get("motivation_polarity_avg")),
            "habit_actions": r.get("habit_actions", ""),
            "mentioned_alternatives": r.get("mentioned_alternatives", ""),
            "importance_score": fnum(vo.get("importance_score")),
            "satisfaction_score": fnum(vo.get("satisfaction_score")),
            "semantic_polarity": fnum(vo.get("semantic_polarity")),
            "change_readiness": vo.get("change_readiness", "") or "",
            "emotions": audience.get("emotions", []),
            "push": audience.get("push", ""),
            "pull": audience.get("pull", ""),
            "anxieties": audience.get("anxieties", ""),
            "audience_habits": audience.get("habits", ""),
        })
    with_upvotes = [r for r in arch_rows if r["upvotes"] is not None]
    top_quotes = sorted(with_upvotes, key=lambda x: -x["upvotes"])[:8]
    neg_quotes = sorted(with_upvotes, key=lambda x: x["upvotes"])[:10]
    neg_quotes = [q for q in neg_quotes if q["upvotes"] < 0]

    # per-archetype cross comparison, computed from the semantic themes so the
    # segment cards match the Jobs-to-be-done section.
    def _top_pct(ms, field, n=3):
        cnt = Counter(r[field] for r in ms if r[field])
        total = sum(cnt.values()) or 1
        return ", ".join(f"{k} ({v / total * 100:.0f}%)" for k, v in cnt.most_common(n))

    cross_by_id = {}
    for cid in sorted({r["archetype_cluster"] for r in arch_rows if r["archetype_cluster"] != ""}):
        ms = [r for r in arch_rows if r["archetype_cluster"] == cid]
        cross_by_id[str(cid)] = {
            "top_situations": _top_pct(ms, "situation_theme"),
            "top_struggles": _top_pct(ms, "struggle_theme"),
            "top_outcomes": _top_pct(ms, "outcome_theme"),
        }

    # transition chains ranked by count (computed from the semantic themes)
    chain_counter = Counter()
    chain_pairs = defaultdict(int)
    for r in arch_rows:
        s = r["situation_theme"]
        st = r["struggle_theme"]
        o = r["outcome_theme"]
        if s and st and o:
            chain_counter[(s, st, o)] += 1
            chain_pairs[(s, st)] += 1
    chains = []
    for (s, st, o), cnt in chain_counter.items():
        if s == "Noise" or st == "Noise" or o == "Noise":
            continue
        chains.append({"situation": s, "struggle": st, "outcome": o,
                       "count": cnt, "p": cnt / chain_pairs[(s, st)] if chain_pairs[(s, st)] else None})
    chains.sort(key=lambda x: -x["count"])
    top_chains = chains[:15]

    # current-approach / hesitation clusters (from the pipeline's clustering step)
    def load_clusters(rows):
        out = []
        for r in rows:
            out.append({
                "cluster": r.get("cluster", ""),
                "label": r.get("label", ""),
                "count": int(fnum(r.get("count")) or 0),
                "top_words": r.get("top_words", ""),
                "example": r.get("example", ""),
            })
        out.sort(key=lambda x: -x["count"])
        return out

    approach_clusters = load_clusters(d.get("approach_clusters", []))
    hesitation_clusters = load_clusters(d.get("hesitation_clusters", []))

    # broken-incumbent: per approach theme, frequency + avg per-row satisfaction/opportunity
    app_stats = defaultdict(lambda: {"count": 0, "sat": 0.0, "opp": 0.0, "satn": 0, "oppn": 0})
    for r in arch_rows:
        t = r["current_approach_theme"]
        if not t:
            continue
        st = app_stats[t]
        st["count"] += 1
        if r["satisfaction"] is not None:
            st["sat"] += r["satisfaction"]; st["satn"] += 1
        if r["opportunity_row"] is not None:
            st["opp"] += r["opportunity_row"]; st["oppn"] += 1
    broken = []
    for t, st in app_stats.items():
        broken.append({
            "theme": t,
            "count": st["count"],
            "avg_sat": st["sat"] / st["satn"] if st["satn"] else None,
            "avg_opp": st["opp"] / st["oppn"] if st["oppn"] else None,
        })
    # most broken first: lowest avg satisfaction (None sorts last)
    broken.sort(key=lambda x: (x["avg_sat"] is None, x["avg_sat"] if x["avg_sat"] is not None else 0))

    # situation -> approach / hesitation maps
    sit_app = defaultdict(Counter)
    sit_hes = defaultdict(Counter)
    for r in arch_rows:
        sit = r["situation_theme"]
        if sit and r["current_approach_theme"]:
            sit_app[sit][r["current_approach_theme"]] += 1
        if sit and r["hesitation_theme"]:
            sit_hes[sit][r["hesitation_theme"]] += 1

    def top_map(dd, n=3):
        out = []
        for sit in sorted(dd, key=lambda s: -sum(dd[s].values())):
            out.append({"situation": sit, "top": dd[sit].most_common(n)})
        return out

    situation_approach = top_map(sit_app)
    situation_hesitation = top_map(sit_hes)

    # job clusters = the semantic outcome themes (grouped by outcome)
    jc_members = defaultdict(list)
    for r in arch_rows:
        c = r["job_cluster"]
        if c:
            jc_members[c].append(r)

    job_cluster_components = []
    for c in sorted(jc_members, key=lambda c: -len(jc_members[c])):
        ms = jc_members[c]
        job_cluster_components.append({
            "key": c,
            "title": c,
            "count": len(ms),
            "top_words": top_words_of([m["outcome"] for m in ms]),
            "example": ms[0]["outcome"],
            "archetypes": Counter(m["archetype"] for m in ms if m["archetype"]).most_common(),
            "situations": Counter(m["situation_theme"] for m in ms if m["situation_theme"]).most_common(3),
            "struggles": Counter(m["struggle_theme"] for m in ms if m["struggle_theme"]).most_common(3),
        })

    # journey: trigger phrase -> struggle -> outcome, ordered by journey stage + need
    JOURNEY_STAGE_ORDER = ["Acquire", "Maintain", "Problem", "Resolve", "Improve"]
    stage_order = {s: i for i, s in enumerate(JOURNEY_STAGE_ORDER)}

    journey_counter = Counter()
    journey_opp = defaultdict(lambda: {"opp": 0.0, "n": 0})
    journey_pol = defaultdict(lambda: {"pol": 0.0, "n": 0})
    journey_appr = defaultdict(Counter)
    journey_emotions = defaultdict(Counter)
    for r in arch_rows:
        tp = r["trigger_phrase"]
        st = r["struggle_theme"]
        o = r["outcome_theme"]
        if not (tp and st and o):
            continue
        journey_counter[(tp, st, o)] += 1
        if r["opportunity_row"] is not None:
            journey_opp[(tp, st, o)]["opp"] += r["opportunity_row"]
            journey_opp[(tp, st, o)]["n"] += 1
        if r["motivation_polarity"] is not None:
            journey_pol[(tp, st, o)]["pol"] += r["motivation_polarity"]
            journey_pol[(tp, st, o)]["n"] += 1
        if r["current_approach_theme"]:
            journey_appr[(tp, st, o)][r["current_approach_theme"]] += 1
        for e in r["emotions"]:
            journey_emotions[(tp, st, o)][e] += 1

    journey_paths = []
    for (tp, st, o), cnt in journey_counter.items():
        jo = journey_opp[(tp, st, o)]
        jp = journey_pol[(tp, st, o)]
        journey_paths.append({
            "stage": "",
            "phrase": tp,
            "struggle": st,
            "outcome": o,
            "count": cnt,
            "avg_opp": jo["opp"] / jo["n"] if jo["n"] else None,
            "energy": jp["pol"] / jp["n"] if jp["n"] else None,
            "approach": journey_appr[(tp, st, o)].most_common(1)[0][0] if journey_appr[(tp, st, o)] else "",
            "emotions": [e for e, _ in journey_emotions[(tp, st, o)].most_common(2)],
        })
    # stage lookup: build phrase->stage from arch_rows
    phrase_stage = {r["trigger_phrase"]: r["journey_stage"] for r in arch_rows if r["trigger_phrase"]}
    for p in journey_paths:
        p["stage"] = phrase_stage.get(p["phrase"], "")

    # sort by stage order then count; embed ALL paths (the JS filter + a
    # per-stage cap are applied client-side so the dropdowns can see every value)
    journey_paths.sort(key=lambda p: (stage_order.get(p["stage"], 99), -(p["count"] or 0)))

    # struggling moments (Moesta): the trigger is the moment the push becomes
    # strong enough to act. Rank journey paths by emotional intensity. Only the
    # moments with a measurable emotional charge (non-zero motivation polarity)
    # are shown — the rest are neutral.
    struggling_moments = [p for p in journey_paths if p.get("energy") is not None and abs(p["energy"]) > 0.001]
    struggling_moments.sort(key=lambda p: -abs(p["energy"]))
    n_neutral = len([p for p in journey_paths if p.get("energy") is None or abs(p["energy"]) <= 0.001])
    struggling_moments = struggling_moments[:50]

    # attach the original member comments + top segment to each struggling moment
    arch_by_path = defaultdict(list)
    for r in arch_rows:
        arch_by_path[(r["trigger_phrase"], r["struggle_theme"], r["outcome_theme"])].append(r)

    for m in struggling_moments:
        members = arch_by_path.get((m["phrase"], m["struggle"], m["outcome"]), [])
        m["examples"] = [{
            "hl": r["hl"],
            "subreddit": r["subreddit"],
            "upvotes": r["upvotes"],
            "author": r["author"],
            "post_url": r["post_url"],
            "archetype": r["archetype"],
            "trigger_verb": r["trigger_verb"],
            "trigger_phrase": r["trigger_phrase"],
            "journey_stage": r["journey_stage"],
            "job_statement": r["job_statement"],
            "situation": r["situation"],
            "struggle": r["struggle"],
            "outcome": r["outcome"],
            "situation_theme": r["situation_theme"],
            "struggle_theme": r["struggle_theme"],
            "outcome_theme": r["outcome_theme"],
            "current_approach": r["current_approach"],
            "hesitation": r["hesitation"],
            "habit_actions": r["habit_actions"],
            "mentioned_alternatives": r["mentioned_alternatives"],
            "importance": r["importance"],
            "satisfaction": r["satisfaction"],
            "opportunity": r["opportunity_row"],
            "motivation_polarity": r["motivation_polarity"],
            "semantic_polarity": r["semantic_polarity"],
            "change_readiness": r["change_readiness"],
            "emotions": r["emotions"],
            "push": r["push"],
            "pull": r["pull"],
            "anxieties": r["anxieties"],
            "audience_habits": r["audience_habits"],
        } for r in members]
        arch_counter = Counter(r["archetype"] for r in members if r["archetype"])
        m["top_archetype"] = arch_counter.most_common(1)[0][0] if arch_counter else ""

    # pain-point data: join upvotes/subreddit from raw comments, and
    # theme/segment/job context from the archetype rows (for filtering).
    canonical_meta = {}
    for r in d["pain_clusters"]:
        canon = r.get("canonical", "")
        if canon and canon not in canonical_meta:
            canonical_meta[canon] = {
                "cluster": r.get("cluster", ""),
                "cluster_size": int(fnum(r.get("cluster_size")) or 0),
                "unique_posts": int(fnum(r.get("unique_posts")) or 0),
                "representative": r.get("representative", ""),
            }

    arch_by_comment = {r["comment"]: r for r in arch_rows}
    pain_by_comment = {}
    for r in d["pain_points"]:
        comment = r.get("comment", "")
        if not comment:
            continue
        meta = canonical_meta.get(r.get("canonical", ""), {})
        quotes = [p.strip() for p in (r.get("quotes", "") or "").split("||") if p.strip()]
        hit = raw_lookup.get(comment, {})
        try:
            upv = int(hit["upvotes"]) if hit.get("upvotes") not in (None, "") else None
        except (TypeError, ValueError):
            upv = None
        if comment not in pain_by_comment:
            pain_by_comment[comment] = {
                "quotes": quotes,
                "upvotes": upv,
                "subreddit": subreddit_from_url(hit.get("post_url", "")),
                "canonical": r.get("canonical", ""),
                "cluster": meta.get("cluster", ""),
                "representative": meta.get("representative", ""),
                "cluster_size": meta.get("cluster_size", 0),
                "unique_posts": meta.get("unique_posts", 0),
            }
        else:
            pain_by_comment[comment]["quotes"].extend(quotes)

    pain_comments = []
    for comment, pc in pain_by_comment.items():
        ar = arch_by_comment.get(comment)
        pain_comments.append({
            "quotes_hl": highlight_comment(comment, pc["quotes"]),
            "upvotes": pc["upvotes"],
            "subreddit": pc["subreddit"],
            "canonical": pc["canonical"],
            "cluster": pc["cluster"],
            "representative": pc["representative"],
            "cluster_size": pc["cluster_size"],
            "unique_posts": pc["unique_posts"],
            "situation_theme": ar["situation_theme"] if ar else "",
            "struggle_theme": ar["struggle_theme"] if ar else "",
            "outcome_theme": ar["outcome_theme"] if ar else "",
            "archetype": ar["archetype"] if ar else "",
            "job_cluster": ar["job_cluster"] if ar else "",
            "current_approach_theme": ar["current_approach_theme"] if ar else "",
            "hesitation_theme": ar["hesitation_theme"] if ar else "",
            "trigger_verb": ar["trigger_verb"] if ar else "",
            "trigger_phrase": ar["trigger_phrase"] if ar else "",
            "journey_stage": ar["journey_stage"] if ar else "",
        })
    pain_comments.sort(key=lambda x: (x["upvotes"] is None, -(x["upvotes"] or 0)))

    # top pain-point clusters per (struggle, outcome), for each journey row
    pain_agg = defaultdict(lambda: defaultdict(lambda: {"upvotes": 0, "repr": ""}))
    for pc in pain_comments:
        st = pc["struggle_theme"]
        o = pc["outcome_theme"]
        if not st or not o or pc["upvotes"] is None:
            continue
        ckey = pc["cluster"] or pc["canonical"]
        pain_agg[(st, o)][ckey]["upvotes"] += pc["upvotes"]
        if not pain_agg[(st, o)][ckey]["repr"]:
            pain_agg[(st, o)][ckey]["repr"] = pc["representative"] or pc["canonical"]

    pain_by_sso = {}
    for (st, o), clusters in pain_agg.items():
        top = sorted(clusters.values(), key=lambda c: -c["upvotes"])[:2]
        pain_by_sso[(st, o)] = [{"repr": c["repr"], "upvotes": int(c["upvotes"])} for c in top]

    pain_reprs_by_sso = {}
    for (st, o), clusters in pain_agg.items():
        pain_reprs_by_sso[(st, o)] = sorted(c["repr"] for c in clusters.values() if c["repr"])

    for p in journey_paths:
        p["pain"] = pain_by_sso.get((p["struggle"], p["outcome"]), [])
        p["pain_reprs"] = pain_reprs_by_sso.get((p["struggle"], p["outcome"]), [])

    # ---- forces of progress (Moesta): push/pull/anxiety/habit per segment ----
    # Approximations from available signals: push = dissatisfaction (1 - satisfaction/6),
    # pull = importance/6, anxiety = % rows with a hesitation, habit = % rows with habit actions.
    forces_by_cluster = defaultdict(lambda: {"n": 0, "sat": 0.0, "imp": 0.0, "hes": 0, "hab": 0.0})
    for r in arch_rows:
        c = r["archetype_cluster"]
        if not c:
            continue
        f = forces_by_cluster[c]
        f["n"] += 1
        if r["satisfaction"] is not None:
            f["sat"] += r["satisfaction"]
        if r["importance"] is not None:
            f["imp"] += r["importance"]
        if r["hesitation"]:
            f["hes"] += 1
        ha = [h.strip() for h in (r["habit_actions"] or "").split(";") if h.strip()]
        if ha:
            f["hab"] += min(len(ha), 5) / 5.0  # normalized habit-action count

    forces = []
    for seg in segments:
        f = forces_by_cluster.get(seg["id"], {})
        n = f.get("n", 0) or 1
        push = 1.0 - (f.get("sat", 0.0) / n) / 6.0
        pull = (f.get("imp", 0.0) / n) / 6.0
        anxiety = f.get("hes", 0) / n
        habit = f.get("hab", 0.0) / n
        forces.append({
            "label": seg["label"],
            "n": n,
            "push": push,
            "pull": pull,
            "anxiety": anxiety,
            "habit": habit,
            "net": (push + pull) - (anxiety + habit),
        })
    forces.sort(key=lambda x: -x["net"])

    # top habits overall (what people are attached to doing)
    habit_tally = Counter()
    for r in arch_rows:
        for h in (r["habit_actions"] or "").split(";"):
            h = h.strip()
            if h:
                habit_tally[h] += 1
    top_habits = habit_tally.most_common(15)

    # market size proxy
    n_comments = len(d["raw"])
    n_authors = len({r.get("author") for r in d["raw"] if r.get("author")})
    n_pain_clusters = len({r.get("cluster") for r in d["pain_clusters"] if r.get("cluster")})

    return {
        "segments": segments,
        "cross_by_id": cross_by_id,
        "top_chains": top_chains,
        "top_outcomes": top_outcomes,
        "top_struggles": top_struggles,
        "top_quotes": top_quotes,
        "neg_quotes": neg_quotes,
        "arch_rows": arch_rows,
        "readiness": d["readiness"],
        "approach_clusters": approach_clusters,
        "hesitation_clusters": hesitation_clusters,
        "broken_incumbents": broken,
        "situation_approach": situation_approach,
        "situation_hesitation": situation_hesitation,
        "job_cluster_components": job_cluster_components,
        "journey_paths": journey_paths,
        "struggling_moments": struggling_moments,
        "n_neutral": n_neutral,
        "forces": forces,
        "top_habits": top_habits,
        "pain_comments": pain_comments,
        "n_comments": n_comments,
        "n_authors": n_authors,
        "n_jtbd_rows": len(arch_rows),
        "n_pain_clusters": n_pain_clusters,
    }


# ----------------------------------------------------------------------
# HTML rendering
# ----------------------------------------------------------------------
CSS = """
:root { color-scheme: light; }
* { box-sizing: border-box; }
body { margin:0; font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif; color:#1a1a1a; background:#f6f7f8; line-height:1.55; }
.page { max-width:880px; margin:0 auto; padding:40px 32px 80px; }
h1 { font-size:1.9em; margin:0 0 4px; }
.meta { color:#777; margin-bottom:24px; }
h2 { font-size:1.3em; border-bottom:2px solid #e0e0e0; padding-bottom:6px; margin-top:44px; }
h3 { font-size:1.05em; margin:24px 0 8px; }
p { margin:8px 0; }
.exec { background:#fff; border:1px solid #e0e0e0; border-radius:12px; padding:24px 28px; box-shadow:0 1px 4px rgba(0,0,0,.04); }
.exec-grid { display:flex; gap:14px; flex-wrap:wrap; margin:16px 0; }
.stat { background:#f2f4f6; border-radius:8px; padding:12px 16px; min-width:130px; flex:1; }
.stat .k { font-size:.72em; text-transform:uppercase; color:#888; }
.stat .v { font-size:1.5em; font-weight:700; }
.stat .s { font-size:.82em; color:#666; }
.verdict { margin-top:16px; padding:14px 18px; background:#eef7ee; border-left:4px solid #2e7d32; border-radius:6px; }
table { width:100%; border-collapse:collapse; font-size:.9em; background:#fff; border:1px solid #eee; }
th { text-align:left; padding:8px 10px; border-bottom:2px solid #333; background:#f3f5f7; }
td { padding:8px 10px; border-bottom:1px solid #f0f0f0; vertical-align:top; }
td.num, th.num { text-align:right; font-family:"SF Mono",Menlo,monospace; }
.neg { color:#c0392b; font-weight:700; }
.quote { background:#fff; border:1px solid #eee; border-radius:8px; padding:12px 16px; margin:10px 0; }
.quote .q { font-style:italic; }
.quote .m { font-size:.8em; color:#888; margin-top:6px; }
.subreddit { font-size:.8em; color:#666; }
.chain { font-family:"SF Mono",Menlo,monospace; font-size:.85em; }
.theme { cursor:pointer; color:#2b6cb0; border-bottom:1px dotted #2b6cb0; }
.theme:hover { background:#eef4fb; }
.detail-head { font-weight:600; margin:16px 0 8px; padding:8px 12px; background:#eef4fb; border-radius:6px; }
.job { font-size:.88em; color:#444; margin-top:4px; font-family:"SF Mono",Menlo,monospace; }
.sso { font-size:.86em; color:#333; margin-top:4px; }
.pill { display:inline-block; padding:1px 8px; border-radius:10px; font-size:.76em; background:#e8f0fb; color:#2b6cb0; }
.small { color:#888; font-size:.85em; }
.detail-window { max-height:560px; overflow-y:auto; border:1px solid #e6e6e6; border-radius:8px; padding:8px; background:#fafbfc; }
.ex { background:#fff; border:1px solid #ececec; border-radius:8px; margin:8px 0; overflow:hidden; }
.ex:first-child { margin-top:0; }
.ex-head { display:flex; align-items:baseline; gap:8px; padding:8px 12px; cursor:pointer; user-select:none; }
.ex-head:hover { background:#f2f7fb; }
.ex-head .arch { font-weight:600; font-size:.92em; }
.ex-head .sub { color:#777; font-size:.82em; }
.ex-head .votes { margin-left:auto; color:#888; font-size:.8em; white-space:nowrap; }
.ex-head .carat { color:#aaa; font-size:.8em; margin-left:2px; transition:transform .15s; }
.ex.open .ex-head .carat { transform:rotate(90deg); }
.ex-comment { padding:2px 14px 10px; font-size:.95em; line-height:1.55; color:#1a1a1a; }
.ex-comment mark { background:#ffe38a; padding:0 2px; border-radius:3px; color:#1a1a1a; }
.ex-comment.clamp { display:-webkit-box; -webkit-line-clamp:3; -webkit-box-orient:vertical; overflow:hidden; }
.ex.open .ex-comment { display:block; -webkit-line-clamp:unset; overflow:visible; }
.ex-body { display:none; padding:10px 14px 12px; border-top:1px dashed #e6e6e6; margin:0 4px 6px; }
.ex.open .ex-body { display:block; }
.ex-field { margin:8px 0; }
.ex-field .lbl { font-size:.7em; text-transform:uppercase; letter-spacing:.03em; color:#888; }
.ex-field .val { font-size:.9em; color:#222; margin-top:2px; }
.ex-field .val.job { font-family:"SF Mono",Menlo,monospace; font-size:.85em; color:#333; }
.ex-job { padding:4px 14px 8px; font-size:1.04em; line-height:1.5; color:#1a1a1a; border-left:3px solid #2b6cb0; margin:2px 12px; }
.ex-job mark { background:#ffe38a; padding:0 2px; border-radius:3px; color:#1a1a1a; }
.jobcluster { background:#fff; border:1px solid #ececec; border-radius:8px; padding:12px 16px; margin:10px 0; }
.jc-head { font-size:.95em; margin-bottom:6px; }
.jc-head .theme { margin-left:10px; }
.jc-example { font-style:italic; color:#333; font-size:.9em; margin:6px 0; }
.jc-meta { font-size:.82em; color:#555; margin-top:3px; }
.pain-filter { background:#eef4fb; border:1px solid #d7e4f0; border-radius:6px; padding:8px 12px; margin:10px 0; font-size:.9em; }
.pain-filter .clear-btn { margin-left:10px; cursor:pointer; border:1px solid #2b6cb0; background:#fff; color:#2b6cb0; border-radius:4px; padding:1px 8px; font-size:.82em; }
.pain-tabs { display:flex; gap:8px; margin:12px 0; flex-wrap:wrap; }
.pain-tab { cursor:pointer; border:1px solid #ccc; background:#fff; color:#333; border-radius:6px; padding:6px 12px; font-size:.88em; }
.pain-tab.active { background:#2b6cb0; color:#fff; border-color:#2b6cb0; }
.pain-window { max-height:620px; overflow-y:auto; border:1px solid #e6e6e6; border-radius:8px; padding:10px; background:#fafbfc; }
.pain-quote { background:#fff; border:1px solid #ececec; border-radius:8px; padding:10px 12px; margin:8px 0; }
.pain-quote .pq-meta { font-size:.78em; color:#888; margin-bottom:4px; }
.pain-quote .pq-canon { font-size:.85em; font-weight:600; color:#333; margin-bottom:4px; }
.pain-quote .pq-text { font-size:.92em; line-height:1.5; color:#1a1a1a; }
.pain-quote .pq-text mark { background:#ffe38a; padding:0 2px; border-radius:3px; }
.pain-quote.neg { border-left:3px solid #c0392b; }
.pain-quote .votes { font-family:"SF Mono",Menlo,monospace; }
.pain-quote .votes.pos { color:#2e7d32; }
.pain-quote .votes.neg { color:#c0392b; font-weight:700; }
.quadrant-svg { display:block; }
.quadrant-dot { cursor:pointer; }
#pain-tooltip { position: fixed; display: none; background: #1a1a1a; color: #fff; padding: 8px 11px; border-radius: 6px; font-size: .82em; line-height: 1.4; max-width: 300px; pointer-events: none; z-index: 1000; box-shadow: 0 2px 8px rgba(0,0,0,.3); }
.pain-label { font-size: 9px; fill: #555; pointer-events: none; }
.pain-back { cursor:pointer; color:#2b6cb0; font-size:.85em; margin-bottom:8px; display:inline-block; }
.journey-filters { display:flex; gap:14px; flex-wrap:wrap; margin:12px 0; }
.jf-label { font-size:.85em; color:#444; }
.jf-label select { margin-left:6px; padding:4px 6px; font-size:.88em; border:1px solid #ccc; border-radius:5px; background:#fff; max-width:260px; }
.energy-dot { display:inline-block; width:9px; height:9px; border-radius:50%; margin-right:4px; vertical-align:middle; }
.energy-tag { font-size:.72em; color:#666; background:#f0f2f5; border-radius:8px; padding:1px 7px; margin-right:4px; white-space:nowrap; }
.moment-card { background:#fff; border:1px solid #ececec; border-radius:8px; padding:12px 16px; margin:8px 0; }
.moment-head { margin-bottom:6px; cursor:pointer; user-select:none; }
.moment-head .carat { color:#aaa; font-size:.8em; margin-left:6px; transition:transform .15s; }
.moment-card.open .moment-head .carat { transform:rotate(90deg); }
.moment-meta { font-size:.78em; color:#888; margin-left:8px; }
.moment-row { font-size:.9em; color:#333; margin-top:3px; }
.m-lbl { display:inline-block; min-width:34px; font-size:.7em; text-transform:uppercase; letter-spacing:.03em; color:#888; font-weight:600; }
.moment-body { display:none; border-top:1px dashed #e6e6e6; margin-top:8px; padding-top:8px; }
.moment-card.open .moment-body { display:block; }
.moment-example { background:#fafbfc; border:1px solid #ececec; border-radius:6px; padding:8px 12px; margin:6px 0; }
.moment-job { font-family:"SF Mono",Menlo,monospace; font-size:.82em; color:#333; margin-top:4px; }
.jtbd-block { margin-top:10px; padding-top:8px; border-top:1px solid #e8e8e8; }
.jtbd-block-title { font-size:.72em; font-weight:700; text-transform:uppercase; letter-spacing:.06em; color:#2b6cb0; margin-bottom:6px; }
.jtbd-grid { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:8px 14px; }
.jtbd-field.wide { grid-column:1 / -1; }
.jtbd-label { font-size:.68em; text-transform:uppercase; letter-spacing:.03em; color:#888; }
.jtbd-value { font-size:.86em; color:#222; margin-top:1px; white-space:pre-wrap; }
.signal-grid { grid-template-columns:repeat(3,minmax(0,1fr)); }
.pq-meta a { color:#2b6cb0; }
.moment-comment { max-height:180px; overflow-y:auto; margin-top:4px; padding:8px; background:#fff; border:1px solid #eee; border-radius:5px; }
.moment-comment mark { background:#ffe38a; padding:0 2px; border-radius:3px; }
"""


def render_report(prefix, s):
    title = prefix.replace("_", " ").title()

    exec_html = f"""
    <div class="exec">
      <h1>{esc(title)}</h1>
      <div class="meta">Market-opportunity report · generated from JTBD pipeline output · {esc(prefix)}</div>
      <div class="exec-grid">
        <div class="stat"><div class="k">Comments mined</div><div class="v">{s['n_comments']}</div><div class="s">raw source comments</div></div>
        <div class="stat"><div class="k">Unique authors</div><div class="v">{s['n_authors']}</div><div class="s">people heard</div></div>
        <div class="stat"><div class="k">Customer segments</div><div class="v">{len(s['segments'])}</div><div class="s">distinct archetypes</div></div>
        <div class="stat"><div class="k">JTBD examples</div><div class="v">{s['n_jtbd_rows']}</div><div class="s">situation→struggle→outcome</div></div>
        <div class="stat"><div class="k">Pain-point clusters</div><div class="v">{s['n_pain_clusters']}</div><div class="s">independent corroboration</div></div>
      </div>
      {_verdict_html(s)}
    </div>
    """

    segments_html = _segments_html(s)
    chains_html = _chains_html(s) + _jtbd_interactive_html(s)
    struggling_html = _struggling_html(s)
    journey_html = _journey_html(s) + _journey_interactive_html(s)
    job_clusters_html = _job_clusters_html(s)
    approach_html = _approach_html(s)
    hesitation_html = _hesitation_html(s)
    forces_html = _forces_html(s)
    opp_html = _opportunity_html(s)
    evidence_html = _evidence_html(s)
    pain_html = _pain_html(s) + _pain_interactive_html(s)
    recommend_html = _recommend_html(s)

    html_doc = f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate">
<title>{esc(title)} — Market Opportunity Report</title>
<style>{CSS}</style>
</head>
<body>
<div class="page">
{exec_html}
{segments_html}
{chains_html}
{struggling_html}
{journey_html}
{job_clusters_html}
{approach_html}
{hesitation_html}
{forces_html}
{opp_html}
{evidence_html}
{pain_html}
{recommend_html}
<p class="small" style="margin-top:40px">Generated by generate_market_report.py · figures are raw pipeline outputs, not normalized across runs.</p>
</div>
</body>
</html>"""
    return html_doc


def _verdict_html(s):
    if not s["segments"]:
        return ""
    top = s["segments"][0]
    top_outcome = s["top_outcomes"][0] if s["top_outcomes"] else None
    parts = []
    parts.append(f"The highest-opportunity segment is <b>{esc(top['label'])}</b> "
                 f"(opportunity {fmt(top['opportunity'])}, {top['n']} examples), "
                 f"driven by importance {fmt(top['importance'])} vs satisfaction {fmt(top['satisfaction'])}.")
    if top_outcome:
        parts.append(f"The single most under-served outcome is <b>{esc(top_outcome['text'])}</b> "
                     f"(opportunity {fmt(top_outcome['opportunity'])}).")
    parts.append("Opportunity = importance + (importance − satisfaction): a high score means people care a lot "
                 "about something they are currently not getting. That is the whitespace a product can win.")
    return '<div class="verdict">' + " ".join(parts) + "</div>"


def _segments_html(s):
    rows = ""
    for seg in s["segments"]:
        c = s["cross_by_id"].get(seg["id"], {})
        rows += f"""<tr>
          <td><b><span class="theme" data-col="archetype" data-val="{esc(seg['label'])}">{esc(seg['label'])}</span></b> <span class="pill">{esc(seg['change_type'])}</span><br>
              <span class="small">{esc(seg.get('top_words',''))}</span></td>
          <td class="num">{seg['n']}</td>
          <td class="num">{fmt(seg['importance'])}</td>
          <td class="num">{fmt(seg['satisfaction'])}</td>
          <td class="num"><b>{fmt(seg['opportunity'])}</b></td>
          <td class="num">{fmt(seg['readiness'])}</td>
          <td class="small">{esc(c.get('top_situations',''))}<br>{esc(c.get('top_struggles',''))}<br>{esc(c.get('top_outcomes',''))}</td>
        </tr>"""
    return f"""<h2>Customer segments</h2>
    <p>Who the market is made of, ranked by opportunity. Read opportunity together with its two components:
    high importance + low satisfaction is the attractive case; high satisfaction means the need is already met.</p>
    <table><thead><tr><th>Segment</th><th class="num">Examples</th><th class="num">Importance</th>
    <th class="num">Satisfaction</th><th class="num">Opportunity</th><th class="num">Readiness</th><th>Top situations / struggles / outcomes</th></tr></thead>
    <tbody>{rows}</tbody></table>"""


def _theme(col, val):
    return f'<span class="theme" data-col="{col}" data-val="{esc(val)}">{esc(val)}</span>'


def _chains_html(s):
    rows = ""
    for c in s["top_chains"]:
        rows += f"""<tr>
          <td class="chain">{_theme('situation_theme', c['situation'])} → {_theme('struggle_theme', c['struggle'])} → {_theme('outcome_theme', c['outcome'])}</td>
          <td class="num">{c['count']}</td>
          <td class="num">{fmt(c['p'])}</td>
        </tr>"""
    if not rows:
        rows = "<tr><td colspan=3 class=small>No non-noise chains found.</td></tr>"
    return f"""<h2>Jobs-to-be-done</h2>
    <p>The most common causal chains in the market: what situation people are in, what struggle it creates, and the
    outcome they want. "P" is the probability of that outcome given the situation and struggle. <b>Click any
    situation / struggle / outcome</b> to see the underlying examples and raw comments.</p>
    <table><thead><tr><th>Situation → Struggle → Outcome</th><th class="num">Count</th><th class="num">P(outcome)</th></tr></thead>
    <tbody>{rows}</tbody></table>"""


JTBD_SCRIPT = """<script>
const JTBD_ROWS = __ROWS__;
function esc2(s) {
  return String(s == null ? '' : s).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
}
const COL_LABELS = { situation_theme: 'Situation', struggle_theme: 'Struggle', outcome_theme: 'Outcome', current_approach_theme: 'Current approach', hesitation_theme: 'Hesitation', job_cluster: 'Job', archetype: 'Segment', trigger_verb: 'Trigger', trigger_phrase: 'Trigger' };
function showQuotes(col, val) {
  const el = document.getElementById('jtbd-detail');
  const rows = JTBD_ROWS.filter(function (r) { return r[col] === val; });
  const jobMode = (col === 'job_cluster');
  const head = jobMode
    ? '<div class="detail-head">Job cluster ' + esc2(val) + ' &mdash; ' + rows.length + ' job' + (rows.length === 1 ? '' : 's') + ' <span class="small">(highlighted words are the cluster&rsquo;s shared vocabulary; click a row to see the source comment)</span></div>'
    : '<div class="detail-head">' + esc2(COL_LABELS[col]) + ': "' + esc2(val) + '" &mdash; ' + rows.length + ' example' + (rows.length === 1 ? '' : 's') + ' <span class="small">(click a row to expand; highlighted phrases are the pain points the pipeline extracted)</span></div>';
  const parts = [head];
  if (rows.length === 0) {
    parts.push('<p class="small">No examples match this theme (it may only appear in the transition data, not in a saved example row).</p>');
    el.innerHTML = parts.join('');
    return;
  }
  parts.push('<div class="detail-window">');
  for (var i = 0; i < rows.length; i++) {
    var r = rows[i];
    parts.push('<div class="ex">');
    parts.push('<div class="ex-head">');
    parts.push('<span class="arch">' + esc2(r.archetype) + '</span>');
    if (r.subreddit) parts.push('<span class="sub">' + esc2(r.subreddit) + '</span>');
    parts.push('<span class="votes">' + (r.upvotes == null ? '&mdash;' : r.upvotes) + ' votes</span>');
    parts.push('<span class="carat">&#9656;</span>');
    parts.push('</div>');
    if (jobMode) {
      parts.push('<div class="ex-job">' + (r.job_hl || esc2(r.job_statement || r.situation)) + '</div>');
      parts.push('<div class="ex-body">');
      parts.push('<div class="ex-field"><div class="lbl">Situation</div><div class="val">' + esc2(r.situation) + '</div></div>');
      parts.push('<div class="ex-field"><div class="lbl">Struggle</div><div class="val">' + esc2(r.struggle) + '</div></div>');
      parts.push('<div class="ex-field"><div class="lbl">Outcome</div><div class="val">' + esc2(r.outcome) + '</div></div>');
      if (r.current_approach) parts.push('<div class="ex-field"><div class="lbl">Current approach</div><div class="val">' + esc2(r.current_approach) + '</div></div>');
      if (r.hesitation) parts.push('<div class="ex-field"><div class="lbl">Hesitation</div><div class="val">' + esc2(r.hesitation) + '</div></div>');
      parts.push('<div class="ex-field"><div class="lbl">Source comment</div><div class="val">' + r.hl + '</div></div>');
      parts.push('</div>');
    } else {
      parts.push('<div class="ex-comment clamp">' + r.hl + '</div>');
      parts.push('<div class="ex-body">');
      if (r.job_statement) parts.push('<div class="ex-field"><div class="lbl">Job statement</div><div class="val job">' + esc2(r.job_statement) + '</div></div>');
      parts.push('<div class="ex-field"><div class="lbl">Situation</div><div class="val">' + esc2(r.situation) + '</div></div>');
      parts.push('<div class="ex-field"><div class="lbl">Struggle</div><div class="val">' + esc2(r.struggle) + '</div></div>');
      parts.push('<div class="ex-field"><div class="lbl">Outcome</div><div class="val">' + esc2(r.outcome) + '</div></div>');
      if (r.current_approach) parts.push('<div class="ex-field"><div class="lbl">Current approach</div><div class="val">' + esc2(r.current_approach) + '</div></div>');
      if (r.hesitation) parts.push('<div class="ex-field"><div class="lbl">Hesitation</div><div class="val">' + esc2(r.hesitation) + '</div></div>');
      parts.push('</div>');
    }
    parts.push('</div>');
  }
  parts.push('</div>');
  el.innerHTML = parts.join('');
  el.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
}
document.addEventListener('click', function (e) {
  var t = e.target.closest('.theme');
  if (t) { showQuotes(t.getAttribute('data-col'), t.getAttribute('data-val')); return; }
  var h = e.target.closest('.ex-head');
  if (h) h.parentElement.classList.toggle('open');
});
</script>"""


def _jtbd_interactive_html(s):
    rows_json = json.dumps(s["arch_rows"], ensure_ascii=True).replace("<", "\\u003c")
    script = JTBD_SCRIPT.replace("__ROWS__", rows_json)
    return '<div id="jtbd-detail"></div>\n' + script


def _cluster_list_html(clusters, theme_col):
    """Clickable cluster list. Each label links into the shared quote window."""
    items = "".join(
        f'<li><span class="theme" data-col="{theme_col}" data-val="{esc(c["label"])}">{esc(c["label"])}</span> '
        f'<span class="small">({c["count"]})</span> &nbsp; <span class="small">{esc(c["top_words"])}</span></li>'
        for c in clusters if c["label"]
    )
    return f'<ol style="margin:8px 0; line-height:1.9;">{items}</ol>'


def _approach_html(s):
    clusters = s.get("approach_clusters", [])
    broken = s.get("broken_incumbents", [])
    sit_map = s.get("situation_approach", [])

    if not clusters:
        return ('<h2>Current approaches (what people do today)</h2>'
                '<p class="small">Not yet computed — run the "Cluster current approaches &amp; hesitations" '
                'pipeline step first.</p>')

    broken_rows = "".join(
        f'<tr><td>{esc(b["theme"])}</td><td class=num>{b["count"]}</td>'
        f'<td class=num>{fmt(b["avg_sat"])}</td><td class=num>{fmt(b["avg_opp"])}</td></tr>'
        for b in broken[:12]
    )

    sit_rows = "".join(
        f'<tr><td>{esc(row["situation"])}</td><td class="small">'
        + " &nbsp; ".join(f'{esc(t)} ({n})' for t, n in row["top"])
        + "</td></tr>"
        for row in sit_map
    )

    return f"""<h2>Current approaches (what people do today)</h2>
    <p>The competition: what customers are <i>currently hiring</i> to do the job — workarounds, hacks, and
    incumbents. <b>Click a cluster to read its examples.</b></p>
    {_cluster_list_html(clusters, 'current_approach_theme')}
    <h3>Broken incumbents (the workarounds people hate)</h3>
    <p>Frequency × per-row satisfaction. High count + low satisfaction = a widely-used approach that isn't
    working = the thing your product displaces. Sorted most-broken first (satisfaction ascending).</p>
    <table><thead><tr><th>Current approach</th><th class="num">Count</th><th class="num">Avg satisfaction</th><th class="num">Avg opportunity</th></tr></thead>
    <tbody>{broken_rows}</tbody></table>
    <h3>Situation → current approach</h3>
    <p>For each situation, what people currently do about it.</p>
    <table><thead><tr><th>Situation</th><th>Top current approaches</th></tr></thead>
    <tbody>{sit_rows}</tbody></table>"""


def _hesitation_html(s):
    clusters = s.get("hesitation_clusters", [])
    sit_map = s.get("situation_hesitation", [])

    if not clusters:
        return ('<h2>Hesitations (what stops them switching)</h2>'
                '<p class="small">Not yet computed — run the "Cluster current approaches &amp; hesitations" '
                'pipeline step first.</p>')

    sit_rows = "".join(
        f'<tr><td>{esc(row["situation"])}</td><td class="small">'
        + " &nbsp; ".join(f'{esc(t)} ({n})' for t, n in row["top"])
        + "</td></tr>"
        for row in sit_map
    )

    return f"""<h2>Hesitations (what stops them switching)</h2>
    <p>The adoption barriers — why customers haven't solved this yet despite the struggle. This is the risk a new
    product must design around. <b>Click a cluster to read its examples.</b></p>
    {_cluster_list_html(clusters, 'hesitation_theme')}
    <h3>Situation → hesitation</h3>
    <p>For each situation, the barriers holding people back.</p>
    <table><thead><tr><th>Situation</th><th>Top hesitations</th></tr></thead>
    <tbody>{sit_rows}</tbody></table>"""


def _job_clusters_html(s):
    comps = s.get("job_cluster_components", [])
    if not comps:
        return ('<h2>Job-statement clusters</h2>'
                '<p class="small">Not yet computed — job-statement clustering runs in the noun-phrase clustering '
                'pipeline step.</p>')

    def dist(items):
        return " &nbsp; ".join(f"{esc(t)} ({n})" for t, n in items)

    cards = ""
    for c in comps:
        cards += f"""<div class="jobcluster">
          <div class="jc-head"><b>{esc(c['title'])}</b> &middot; {c['count']} jobs
            <span class="theme" data-col="job_cluster" data-val="{esc(c['key'])}">show examples &#9656;</span>
          </div>
          <div class="jc-meta"><b>Top words:</b> {esc(c['top_words'])}</div>
          <div class="jc-example">&ldquo;{esc(c['example'])}&rdquo;</div>
          <div class="jc-meta"><b>Segments:</b> {dist(c['archetypes'])}</div>
          <div class="jc-meta"><b>Situations:</b> {dist(c['situations'])}</div>
          <div class="jc-meta"><b>Struggles:</b> {dist(c['struggles'])}</div>
        </div>"""

    return f"""<h2>Job-statement clusters</h2>
    <p>Jobs grouped by their <b>outcome</b> (the "so I can&hellip;" purpose), clustered semantically and
    LLM-labeled — the same job appears once even when it shows up across different plants, climates, and pots.
    Each cluster shows its top words, an example, and which segments and themes its jobs come from. Click
    <b>show examples</b> to read the underlying jobs (shared vocabulary highlighted).</p>
    {cards}"""


def _forces_html(s):
    forces = s.get("forces", [])
    habits = s.get("top_habits", [])
    if not forces:
        return ('<h2>Forces of progress</h2>'
                '<p class="small">Not yet computed.</p>')

    def pct(v):
        return f"{v * 100:.0f}%"

    rows = ""
    for f in forces:
        net_style = 'color:#2e7d32;font-weight:700' if f["net"] >= 0 else 'color:#c0392b;font-weight:700'
        rows += f"""<tr>
          <td><b>{esc(f['label'])}</b></td>
          <td class="num">{pct(f['push'])}</td>
          <td class="num">{pct(f['pull'])}</td>
          <td class="num">{pct(f['anxiety'])}</td>
          <td class="num">{pct(f['habit'])}</td>
          <td class="num" style="{net_style}">{f['net']:+.2f}</td>
        </tr>"""

    habits_rows = "".join(
        f'<tr><td class="small">{esc(h)}</td><td class="num">{c}</td></tr>' for h, c in habits
    )

    return f"""<h2>Forces of progress</h2>
    <p>Bob Moesta's four forces that decide whether a customer "hires" a new solution: <b>Push</b> (dissatisfaction with
    the current situation), <b>Pull</b> (attraction of the desired outcome), <b>Anxiety</b> (fear of switching), and
    <b>Habit</b> (attachment to current behaviour). A switch happens only when Push + Pull exceeds Anxiety + Habit.
    These are <i>approximations</i> from the pipeline's signals (push = 1 &minus; satisfaction/6, pull = importance/6,
    anxiety = % with a hesitation, habit = % with habit actions) &mdash; not a live Switch interview.</p>
    <table><thead><tr><th>Segment</th><th class="num">Push</th><th class="num">Pull</th><th class="num">Anxiety</th><th class="num">Habit</th><th class="num">Net switch force</th></tr></thead>
    <tbody>{rows}</tbody></table>
    <h3>Top habits (what they are attached to doing)</h3>
    <table><thead><tr><th>Habit action</th><th class="num">Mentions</th></tr></thead><tbody>{habits_rows}</tbody></table>"""


def _struggling_html(s):
    moments = s.get("struggling_moments", [])
    if not moments:
        return ('<h2>Struggling moments</h2>'
                '<p class="small">Not yet computed.</p>')

    def field(label, value, css=""):
        if value is None or value == "" or value == []:
            return ""
        if isinstance(value, list):
            value = " · ".join(str(v) for v in value)
        extra = f" {css}" if css else ""
        return (f'<div class="jtbd-field{extra}"><div class="jtbd-label">{esc(label)}</div>'
                f'<div class="jtbd-value">{esc(value)}</div></div>')

    cards = ""
    for m in moments:
        energy = m["energy"]
        dot = '#c0392b' if energy < -0.1 else ('#2e7d32' if energy > 0.1 else '#aaa')
        emo = " \u00b7 ".join(m["emotions"]) if m["emotions"] else ""
        examples_html = ""
        for e in m["examples"]:
            votes = e["upvotes"] if e["upvotes"] is not None else "—"
            source = esc(e["subreddit"])
            if e["post_url"]:
                source = (f'<a href="{esc(e["post_url"])}" target="_blank" '
                          f'rel="noopener noreferrer">{source or "source"}</a>')
            context = "".join([
                field("Journey stage", e["journey_stage"]),
                field("Trigger", e["trigger_phrase"]),
                field("Trigger verb", e["trigger_verb"]),
                field("Situation", e["situation"], "wide"),
                field("Situation theme", e["situation_theme"]),
                field("Customer segment", e["archetype"]),
            ])
            job = "".join([
                field("Struggle", e["struggle"], "wide"),
                field("Outcome", e["outcome"], "wide"),
                field("Struggle theme", e["struggle_theme"]),
                field("Outcome theme", e["outcome_theme"]),
            ])
            forces = "".join([
                field("Push signal", e["push"]),
                field("Current approach / habit", e["current_approach"], "wide"),
                field("Pull signal", e["pull"]),
                field("Anxiety signal", e["anxieties"]),
                field("Hesitation", e["hesitation"], "wide"),
                field("Habit actions", e["habit_actions"], "wide"),
                field("Audience habits", e["audience_habits"], "wide"),
                field("Mentioned alternatives", e["mentioned_alternatives"], "wide"),
            ])
            signals = "".join([
                field("Importance", fmt(e["importance"])),
                field("Satisfaction", fmt(e["satisfaction"])),
                field("Opportunity", fmt(e["opportunity"])),
                field("Emotions", e["emotions"]),
                field("Emotional polarity", fmt(e["motivation_polarity"])),
                field("Semantic readiness", fmt(e["semantic_polarity"])),
                field("Change readiness", e["change_readiness"]),
            ])
            examples_html += f"""<div class="moment-example">
              <div class="pq-meta"><span class="votes">{votes} votes</span> &middot; {source}{' &middot; ' + esc(e['author']) if e['author'] else ''}</div>
              <div class="jtbd-block"><div class="jtbd-block-title">Context</div><div class="jtbd-grid">{context}</div></div>
              <div class="jtbd-block"><div class="jtbd-block-title">Job</div>
                {f'<div class="moment-job">{esc(e["job_statement"])}</div>' if e['job_statement'] else ''}
                <div class="jtbd-grid">{job}</div>
              </div>
              <div class="jtbd-block"><div class="jtbd-block-title">Forces of progress</div><div class="jtbd-grid">{forces}</div></div>
              <div class="jtbd-block"><div class="jtbd-block-title">Signals</div><div class="jtbd-grid signal-grid">{signals}</div></div>
              <div class="jtbd-block"><div class="jtbd-block-title">Evidence</div>
                <div class="pq-text moment-comment">{e['hl']}</div>
              </div>
            </div>"""
        meta = []
        meta.append(f"{m['count']} comment{'s' if m['count'] != 1 else ''}")
        if m.get("top_archetype"):
            meta.append(esc(m["top_archetype"]))
        cards += f"""<div class="moment-card">
          <div class="moment-head" onclick="this.parentElement.classList.toggle('open')">
            <span class="energy-dot" style="background:{dot}"></span>
            <b>{esc(m['phrase'])}</b>{' <span class="small">(' + esc(emo) + ')</span>' if emo else ''}
            <span class="moment-meta">{' &middot; '.join(meta)}</span>
            <span class="carat">&#9656;</span>
          </div>
          <div class="moment-row"><span class="m-lbl">push</span> {esc(m['struggle'])}{f'<span class="small"> &middot; still: {esc(m["approach"])}</span>' if m['approach'] else ''}</div>
          <div class="moment-row"><span class="m-lbl">pull</span> {esc(m['outcome'])}</div>
          <div class="moment-body">{examples_html}</div>
        </div>"""

    return f"""<h2>Struggling moments</h2>
    <p>Bob Moesta: the trigger is the moment the "push" becomes strong enough to act &mdash; the struggling moment.
    Each card is one such moment, ranked by emotional intensity: what happened (the trigger), what made it unbearable
    (the push), and what they wanted instead (the pull). <b>Click a card</b> to read the original comments behind it.
    The {s['n_neutral']} journeys with neutral emotion are omitted.</p>
    {cards}"""


def _journey_html(s):
    paths = s.get("journey_paths", [])
    if not paths:
        return ('<h2>Journey</h2>'
                '<p class="small">Not yet computed — run the "Extract trigger verbs" pipeline step first.</p>')

    def distinct(field):
        return sorted({p[field] for p in paths if p[field]})

    def dropdown(label, field, values):
        opts = '<option value="">All</option>' + ''.join(
            f'<option value="{esc(v)}">{esc(v)}</option>' for v in values
        )
        return f'<label class="jf-label">{label} <select class="jf-select" data-field="{field}" onchange="setJourneyFilter()">{opts}</select></label>'

    stages = ["Acquire", "Maintain", "Problem", "Resolve", "Improve"]
    pain_values = sorted({r for p in paths for r in p.get("pain_reprs", [])})
    filter_bar = (
        dropdown("Struggle", "struggle", distinct("struggle")) +
        dropdown("Outcome", "outcome", distinct("outcome")) +
        dropdown("Pain point", "pain", pain_values)
    )

    return f"""<h2>Journey</h2>
    <p>The customer journey as a flow: what <b>triggers</b> the job, what <b>struggle</b> it hits, and the
    <b>outcome</b> they want — ordered by journey stage. <b>Need</b> = average opportunity (higher = more
    under-served, red). <b>Pain points</b> show the most-upvoted pain clusters for that stage. Use the filters to
    pivot — e.g. pick a struggle to see which stages feed into it.</p>
    <div class="journey-filters">{filter_bar}</div>
    <div id="journey-view"></div>"""


JOURNEY_SCRIPT = """<script>
const JOURNEY_PATHS = __PATHS__;
const JOURNEY_STAGE_ORDER = __STAGES__;
function journeyFilterState() {
  var f = {};
  document.querySelectorAll('.jf-select').forEach(function (sel) { f[sel.getAttribute('data-field')] = sel.value; });
  return f;
}
function setJourneyFilter() { renderJourney(); }
function renderJourney() {
  var f = journeyFilterState();
  var filtered = JOURNEY_PATHS.filter(function (p) {
    return (!f.struggle || p.struggle === f.struggle) &&
           (!f.outcome || p.outcome === f.outcome) &&
           (!f.pain || (p.pain_reprs && p.pain_reprs.indexOf(f.pain) !== -1));
  });
  var anyFilter = !!(f.struggle || f.outcome || f.pain);
  function stageIndex(s) { var i = JOURNEY_STAGE_ORDER.indexOf(s); return i === -1 ? 99 : i; }
  filtered.sort(function (a, b) {
    var sa = stageIndex(a.stage), sb = stageIndex(b.stage);
    if (sa !== sb) return sa - sb;
    return b.count - a.count;
  });
  var shown = filtered;
  if (!anyFilter) {
    // cap per stage only when unfiltered, so every journey stage is represented
    var byStage = {}, seq = [];
    filtered.forEach(function (p) {
      var s = p.stage || 'Other';
      if (!byStage[s]) { byStage[s] = []; seq.push(s); }
      byStage[s].push(p);
    });
    seq.sort(function (a, b) { return stageIndex(a) - stageIndex(b); });
    shown = [];
    seq.forEach(function (s) { shown = shown.concat(byStage[s].slice(0, 14)); });
  }
  var rows = '', prevStage = null;
  shown.forEach(function (p) {
    var stage = p.stage || 'Other';
    var stageCell = stage === prevStage ? '' : stage;
    prevStage = stage;
    var painCell = '';
    if (p.pain && p.pain.length) {
      painCell = p.pain.map(function (pc) {
        return '<div class="small">' + esc2(pc.repr) + ' <span class="votes pos">(' + pc.upvotes + '&uarr;)</span></div>';
      }).join('');
    }
    var energyDot = '';
    if (p.energy != null) {
      var m = Math.max(-1, Math.min(1, p.energy));
      var eColor = m < -0.1 ? '#c0392b' : (m > 0.1 ? '#2e7d32' : '#aaa');
      var eOp = (0.35 + Math.min(0.65, Math.abs(m) * 0.65)).toFixed(2);
      energyDot = '<span class="energy-dot" style="background:' + eColor + ';opacity:' + eOp + '" title="emotion ' + p.energy.toFixed(2) + '"></span>';
    }
    var emotionTag = (p.emotions && p.emotions.length)
      ? ' <span class="energy-tag">' + esc2(p.emotions.join(' \u00b7 ')) + '</span>' : '';
    var approachLine = p.approach ? '<div class="small" style="color:#777">currently: ' + esc2(p.approach) + '</div>' : '';
    var need = p.avg_opp == null ? '&mdash;' : p.avg_opp.toFixed(2);
    var needStyle = p.avg_opp == null ? 'color:#999' : (p.avg_opp >= 3 ? 'color:#c0392b;font-weight:700' : (p.avg_opp > 1.05 ? 'color:#b7791f' : 'color:#999'));
    rows += '<tr><td class="small">' + esc2(stageCell) + '</td>' +
      '<td>' + energyDot + emotionTag + ' <span class="theme" data-col="trigger_phrase" data-val="' + esc2(p.phrase) + '">' + esc2(p.phrase) + '</span></td>' +
      '<td class="small">' + esc2(p.struggle) + ' &rarr; ' + esc2(p.outcome) + approachLine + '</td>' +
      '<td>' + painCell + '</td>' +
      '<td class="num">' + p.count + '</td>' +
      '<td class="num" style="' + needStyle + '">' + need + '</td></tr>';
  });
  document.getElementById('journey-view').innerHTML =
    '<table><thead><tr><th>Journey stage</th><th>Trigger</th><th>Struggle &rarr; Outcome</th><th>Pain points</th><th class="num">Count</th><th class="num">Need</th></tr></thead><tbody>' + rows + '</tbody></table>';
}
renderJourney();
</script>"""


def _journey_interactive_html(s):
    paths_json = json.dumps(s.get("journey_paths", []), ensure_ascii=True).replace("<", "\\u003c")
    stages_json = json.dumps(["Acquire", "Maintain", "Problem", "Resolve", "Improve"])
    return JOURNEY_SCRIPT.replace("__PATHS__", paths_json).replace("__STAGES__", stages_json)


def _opportunity_html(s):
    def opp_rows(rows):
        return "".join(
            f"<tr><td>{esc(r['text'])}</td><td class=num>{fmt(r['importance'])}</td>"
            f"<td class=num>{fmt(r['satisfaction'])}</td><td class=num><b>{fmt(r['opportunity'])}</b></td></tr>"
            for r in rows
        )
    readiness_rows = ""
    for r in s["readiness"]:
        readiness_rows += f"<tr><td>{esc(r.get('segment_label',''))}</td><td>{esc(r.get('archetypes',''))}</td>" \
            f"<td class=num>{fmt(r.get('avg_opportunity'))}</td><td class=num>{fmt(r.get('avg_readiness'))}</td>" \
            f"<td class=num>{fmt(r.get('avg_emotion'))}</td></tr>"

    return f"""<h2>Opportunity analysis</h2>
    <p>What specifically is under-served, at two levels of detail. Satisfaction can be negative (net-negative sentiment),
    which inflates opportunity — that's by design, it means people are actively frustrated.</p>
    <h3>Most under-served outcomes (the end-state people want but aren't getting)</h3>
    <table><thead><tr><th>Outcome</th><th class="num">Importance</th><th class="num">Satisfaction</th><th class="num">Opportunity</th></tr></thead>
    <tbody>{opp_rows(s['top_outcomes'])}</tbody></table>
    <h3>Most under-served struggles (the blockers)</h3>
    <table><thead><tr><th>Struggle</th><th class="num">Importance</th><th class="num">Satisfaction</th><th class="num">Opportunity</th></tr></thead>
    <tbody>{opp_rows(s['top_struggles'])}</tbody></table>
    <h3>Where to enter</h3>
    <table><thead><tr><th>Readiness segment</th><th>Archetypes</th><th class="num">Avg opportunity</th><th class="num">Avg readiness</th><th class="num">Avg emotion</th></tr></thead>
    <tbody>{readiness_rows}</tbody></table>"""


def _evidence_html(s):
    def quote_html(q, label):
        return f"""<div class="quote">
          <div class="m"><b>{esc(label)}</b> · {esc(q['archetype'])} · <span class="subreddit">{esc(q['subreddit'])}</span> · {q['upvotes']} votes</div>
          <div class="q">"{esc(q['comment'])}"</div>
          <div class="m">Situation: {esc(q['situation'])}</div>
        </div>"""

    top = "".join(quote_html(q, "Most-upvoted") for q in s["top_quotes"])
    neg = "".join(quote_html(q, "Community-disagreed") for q in s["neg_quotes"]) or \
        '<p class="small">No negative-upvote examples in this run.</p>'
    return f"""<h2>Evidence</h2>
    <p>Verbatim comments, not paraphrases. Most-upvoted quotes show the strongest shared sentiment; the
    community-disagreed quotes are signals the pipeline's satisfaction score (built from struggle-text sentiment)
    can miss — the community actively pushed back on this advice.</p>
    <h3>Most upvoted</h3>{top}
    <h3>Community disagreed (negative upvotes)</h3>{neg}"""


def _pain_html(s):
    n = len(s["pain_comments"])
    return f"""<h2>Pain points</h2>
    <p>Independently-extracted pain points, now with upvotes joined from the raw comments. <b>Click any theme,
    segment, or job</b> in the sections above to filter these to that subset, then switch between the three views.
    Upvotes = community agreement with the pain point (higher = more people share it).</p>
    <div class="pain-filter" id="pain-filter">Showing all {n} pain points.</div>
    <div class="pain-tabs">
      <button class="pain-tab active" data-view="clusters" onclick="switchPainView('clusters')">1 &middot; Most upvoted clusters</button>
      <button class="pain-tab" data-view="quadrant" onclick="switchPainView('quadrant')">2 &middot; Breadth &times; agreement</button>
      <button class="pain-tab" data-view="quotes" onclick="switchPainView('quotes')">3 &middot; Individual pain points</button>
    </div>
    <div class="pain-window" id="pain-view"></div>
    <div id="pain-tooltip"></div>"""


PAIN_SCRIPT = """<script>
const PAIN_COMMENTS = __PAIN__;
let painFilter = null;
let painView = 'clusters';
const PAIN_COLS = { situation_theme:'Situation', struggle_theme:'Struggle', outcome_theme:'Outcome', archetype:'Segment', job_cluster:'Job', current_approach_theme:'Current approach', hesitation_theme:'Hesitation', trigger_verb:'Trigger', trigger_phrase:'Trigger' };

function setPainFilter(col, val) { painFilter = { col: col, val: val }; renderPain(); }
function clearPainFilter() { painFilter = null; renderPain(); }
function switchPainView(v) {
  painView = v;
  document.querySelectorAll('.pain-tab').forEach(function (b) { b.classList.toggle('active', b.getAttribute('data-view') === v); });
  renderPain();
}
function filteredPain() {
  if (!painFilter) return PAIN_COMMENTS;
  var col = painFilter.col, val = painFilter.val;
  return PAIN_COMMENTS.filter(function (c) { return c[col] === val; });
}
function clusterAggs(list) {
  var by = {};
  list.forEach(function (c) {
    if (!c.cluster) return;
    if (!by[c.cluster]) by[c.cluster] = { representative: c.representative, cluster_size: c.cluster_size, unique_posts: c.unique_posts, n: 0, sum: 0, cnt: 0 };
    var g = by[c.cluster];
    g.n += 1;
    if (c.upvotes != null) { g.sum += c.upvotes; g.cnt += 1; }
  });
  var out = Object.keys(by).map(function (k) {
    var g = by[k];
    return { cluster: k, representative: g.representative, cluster_size: g.cluster_size, unique_posts: g.unique_posts, n: g.n, total_upvotes: g.sum, avg_upvotes: g.cnt ? (g.sum / g.cnt) : null };
  });
  out.sort(function (a, b) { return b.total_upvotes - a.total_upvotes; });
  return out;
}

function renderPain() {
  var list = filteredPain();
  var banner = document.getElementById('pain-filter');
  if (painFilter) {
    banner.innerHTML = 'Filtered by ' + esc2(PAIN_COLS[painFilter.col] || painFilter.col) + ': "' + esc2(painFilter.val) + '" &mdash; ' + list.length + ' pain point' + (list.length === 1 ? '' : 's') + ' <button class="clear-btn" onclick="clearPainFilter()">show all</button>';
  } else {
    banner.innerHTML = 'Showing all ' + PAIN_COMMENTS.length + ' pain points (click any theme / segment / job above to filter).';
  }
  if (painView === 'clusters') renderPainClusters(list);
  else if (painView === 'quadrant') renderPainQuadrant(list);
  else renderPainQuotes(list);
}

function renderPainClusters(list) {
  var aggs = clusterAggs(list).slice(0, 30);
  var rows = aggs.map(function (g) {
    return '<tr><td class="small">' + esc2(g.representative) + '</td>' +
      '<td class="num">' + g.total_upvotes + '</td>' +
      '<td class="num">' + (g.avg_upvotes == null ? '&mdash;' : g.avg_upvotes.toFixed(1)) + '</td>' +
      '<td class="num">' + g.n + '</td>' +
      '<td class="num">' + g.cluster_size + '</td></tr>';
  }).join('');
  document.getElementById('pain-view').innerHTML =
    '<p class="small">Ranked by total upvotes across member comments — the pain points the community most strongly agrees on.</p>' +
    '<table><thead><tr><th>Representative pain point</th><th class="num">Total upvotes</th><th class="num">Avg</th><th class="num">Comments</th><th class="num">Cluster size</th></tr></thead><tbody>' + rows + '</tbody></table>';
}

function painQuoteCard(c) {
  var cls = c.upvotes < 0 ? 'neg' : 'pos';
  return '<div class="pain-quote' + (c.upvotes < 0 ? ' neg' : '') + '">' +
    '<div class="pq-meta"><span class="votes ' + cls + '">' + c.upvotes + ' votes</span> &middot; ' + esc2(c.subreddit) + (c.archetype ? ' &middot; ' + esc2(c.archetype) : '') + '</div>' +
    '<div class="pq-canon">' + esc2(c.canonical) + '</div>' +
    '<div class="pq-text">' + c.quotes_hl + '</div></div>';
}

function showPainTip(e, el) {
  var tip = document.getElementById('pain-tooltip');
  tip.innerHTML = '<b>' + esc2(el.getAttribute('data-repr')) + '</b><br>' +
    el.getAttribute('data-avg') + ' upvotes/comment &middot; ' + el.getAttribute('data-total') + ' total<br>' +
    el.getAttribute('data-csize') + ' mentions &middot; ' + el.getAttribute('data-n') + ' comments<br>' +
    '<span style="opacity:.65">click to read the comments</span>';
  tip.style.display = 'block';
  tip.style.left = (e.clientX + 14) + 'px';
  tip.style.top = (e.clientY + 14) + 'px';
}
function hidePainTip() { document.getElementById('pain-tooltip').style.display = 'none'; }

function showPainCluster(el) {
  var cluster = el.getAttribute('data-cluster');
  var repr = el.getAttribute('data-repr');
  hidePainTip();
  var members = PAIN_COMMENTS.filter(function (c) { return c.cluster === cluster && c.upvotes != null; })
    .sort(function (a, b) { return b.upvotes - a.upvotes; });
  var html = '<div class="pain-back" onclick="renderPain()">&larr; back to the chart</div>' +
    '<div class="pq-canon" style="margin-bottom:6px">' + esc2(repr) + ' — ' + members.length + ' comment' + (members.length === 1 ? '' : 's') + '</div>';
  html += members.map(painQuoteCard).join('');
  document.getElementById('pain-view').innerHTML = html;
}

function renderPainQuadrant(list) {
  var aggs = clusterAggs(list).filter(function (g) { return g.avg_upvotes != null; }).slice(0, 50);
  var W = 720, H = 460, pad = 64;
  var maxC = Math.max.apply(null, aggs.map(function (g) { return g.cluster_size; }).concat([1]));
  var maxA = Math.max.apply(null, aggs.map(function (g) { return g.avg_upvotes; }).concat([1]));
  var maxT = Math.max.apply(null, aggs.map(function (g) { return g.total_upvotes; }).concat([1]));
  function x(c) { return pad + (c / maxC) * (W - 2 * pad); }
  function y(a) { return H - pad - (a / maxA) * (H - 2 * pad); }
  var labeled = aggs.slice().sort(function (a, b) { return b.total_upvotes - a.total_upvotes; }).slice(0, 14);
  var labelSet = {};
  labeled.forEach(function (g) { labelSet[g.cluster] = true; });
  var dots = aggs.map(function (g) {
    var r = 4 + Math.sqrt(g.total_upvotes / maxT) * 22;
    var cx = x(g.cluster_size), cy = y(g.avg_upvotes);
    var label = labelSet[g.cluster]
      ? '<text class="pain-label" x="' + (cx + r + 4) + '" y="' + (cy + 3) + '">' + esc2(g.representative.length > 30 ? g.representative.slice(0, 29) + '…' : g.representative) + '</text>'
      : '';
    return '<g><circle class="quadrant-dot" data-cluster="' + esc2(g.cluster) + '" data-repr="' + esc2(g.representative) + '" data-total="' + g.total_upvotes + '" data-avg="' + g.avg_upvotes.toFixed(1) + '" data-n="' + g.n + '" data-csize="' + g.cluster_size + '" cx="' + cx + '" cy="' + cy + '" r="' + r + '" fill="#2b6cb0" fill-opacity="0.6" stroke="#2b6cb0" onmousemove="showPainTip(event, this)" onmouseleave="hidePainTip()" onclick="showPainCluster(this)"></circle>' + label + '</g>';
  }).join('');
  var svg = '<svg class="quadrant-svg" width="' + W + '" height="' + H + '">' +
    '<line x1="' + pad + '" y1="' + (H - pad) + '" x2="' + (W - pad) + '" y2="' + (H - pad) + '" stroke="#ccc"/>' +
    '<line x1="' + pad + '" y1="' + pad + '" x2="' + pad + '" y2="' + (H - pad) + '" stroke="#ccc"/>' +
    '<text x="' + (W / 2) + '" y="' + (H - 14) + '" text-anchor="middle" font-size="12" fill="#666">How often it is mentioned (cluster size) →</text>' +
    '<text x="18" y="' + (H / 2) + '" transform="rotate(-90 18 ' + (H / 2) + ')" text-anchor="middle" font-size="12" fill="#666">How strongly agreed (avg upvotes / comment) →</text>' +
    '<text x="' + (W - pad) + '" y="' + (pad - 8) + '" text-anchor="end" font-size="11" fill="#c0392b">▲ common &amp; strongly agreed</text>' +
    '<text x="' + pad + '" y="' + (H - pad + 16) + '" font-size="11" fill="#999">common but not agreed</text>' +
    dots + '</svg>';
  document.getElementById('pain-view').innerHTML =
    '<p class="small">Each bubble is a pain-point cluster. <b>Up = people strongly agree with it; right = it comes up often.</b> The top-right corner (common AND strongly agreed) is the strongest signal. Bubble size = total upvotes. <b>Hover</b> for details, <b>click</b> to read its comments. The largest clusters are labelled.</p>' + svg;
}

function renderPainQuotes(list) {
  var withVotes = list.filter(function (c) { return c.upvotes != null; }).sort(function (a, b) { return b.upvotes - a.upvotes; });
  var top = withVotes.slice(0, 40);
  var neg = withVotes.filter(function (c) { return c.upvotes < 0; }).sort(function (a, b) { return a.upvotes - b.upvotes; });
  var html = '<p class="small">Individual pain-point comments sorted by upvotes — read the most-agreed-with pain points in their own words (highlighted = the exact phrases extracted).</p>';
  html += top.map(painQuoteCard).join('');
  if (neg.length) {
    html += '<h3 style="margin-top:18px">Community disagreed (negative upvotes)</h3>';
    html += neg.map(painQuoteCard).join('');
  }
  document.getElementById('pain-view').innerHTML = html;
}

renderPain();
document.addEventListener('click', function (e) {
  var t = e.target.closest('.theme');
  if (t) setPainFilter(t.getAttribute('data-col'), t.getAttribute('data-val'));
});
</script>"""


def _pain_interactive_html(s):
    pain_json = json.dumps(s["pain_comments"], ensure_ascii=True).replace("<", "\\u003c")
    return PAIN_SCRIPT.replace("__PAIN__", pain_json)


def _recommend_html(s):
    if not s["segments"]:
        return "<h2>Verdict</h2><p>Not enough data to form a verdict.</p>"
    top = s["segments"][0]
    top_outcome = s["top_outcomes"][0] if s["top_outcomes"] else None
    top_struggle = s["top_struggles"][0] if s["top_struggles"] else None
    lines = [
        "<h2>Verdict &amp; recommendation</h2>",
        f"<p><b>Segment to target:</b> {esc(top['label'])} — the highest-opportunity, most under-served audience.</p>",
    ]
    if top_outcome and top_struggle:
        lines.append(
            f"<p><b>What to build for:</b> help them reach <i>{esc(top_outcome['text'])}</i> "
            f"by removing <i>{esc(top_struggle['text'])}</i>.</p>"
        )
    lines.append(
        "<p><b>Go / no-go:</b> this report establishes that a market exists and where the whitespace is. "
        "It does <i>not</i> by itself establish demand size (competitor/ASO/Google signals live in a separate "
        "validation step). If the top opportunity is driven by a real, recurring, under-served need, proceed to "
        "demand sizing; if it's dominated by a one-off or off-topic cluster (check the Evidence and segment "
        "deep-dives), revisit before committing.</p>"
    )
    lines.append(
        "<p class='small'>Caveats: opportunity scores are raw pipeline outputs on two different scales "
        "(segment-level ~2–4, outcome-level ~1.5–2.5) — compare within a table, not across tables. Archetypes can "
        "contain off-topic examples (embedding-space neighbours); always read the quotes before trusting a number.</p>"
    )
    return "".join(lines)


# ----------------------------------------------------------------------
# CLI
# ----------------------------------------------------------------------
def generate(prefix):
    data = load(prefix)
    s = synthesize(data)
    doc = render_report(prefix, s)
    out_dir = os.path.join(BASE_DIR, "reports")
    os.makedirs(out_dir, exist_ok=True)
    out_path = os.path.join(out_dir, f"{prefix}_market_report.html")
    with open(out_path, "w", encoding="utf-8") as f:
        f.write(doc)
    top = s["segments"][0]["label"] if s["segments"] else "(none)"
    print(f"Wrote {out_path}")
    print(f"  {s['n_comments']} comments · {s['n_authors']} authors · {len(s['segments'])} segments · "
          f"{s['n_jtbd_rows']} JTBD rows · {s['n_pain_clusters']} pain clusters")
    print(f"  top segment: {top}")
    return out_path


def main():
    datasets = discover_datasets()
    args = [a for a in sys.argv[1:] if not a.startswith("--")]
    if "--all" in sys.argv or not args:
        targets = datasets
    else:
        targets = [a for a in args if a in datasets]
        if args and not targets:
            print(f"Unknown dataset '{args[0]}'. Available: {', '.join(datasets) or '(none)'}")
            sys.exit(1)
    if not targets:
        print("No dataset runs found (looking for *_jtbd_comments__archetype_summary.csv).")
        sys.exit(1)
    for t in targets:
        generate(t)


if __name__ == "__main__":
    main()