"""
Full pipeline:
1. Loads your JTBD dataset (e.g., master_jtbd_comments.csv)
2. Loads / defines your use cases
3. Computes similarity (situation + struggle + outcome)
4. Matches top N JTBDs per use case
5. Detects additional industries appearing in JTBDs but not in your use cases
6. Exports:
   - jtbd_usecase_matches.csv
   - jtbd_usecase_matches.md
   - unmatched_jtbd.csv (JTBDs not linked to any use case)
   - new_industries.txt (industries found only in JTBDs)
"""

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# ============================================================
# CONFIGURATION
# ============================================================

JTBD_FILE = "master_jtbd_comments.csv"
TOP_N = 10
SIM_THRESHOLD = 0.0  # keep all similarities (set >0.02 for stricter filtering)

OUTPUT_MATCH_MD = "jtbd_usecase_matches.md"
OUTPUT_MATCH_CSV = "jtbd_usecase_matches.csv"
OUTPUT_UNMATCHED = "unmatched_jtbd.csv"
OUTPUT_NEW_INDS = "new_industries.txt"

# Your 13 use cases
USE_CASES = {
    "Pipeline integrity, oil rig monitoring and early leakage warning": "oil gas monitoring leakage pipeline integrity maintenance",
    "Wellhead damage monitoring from sandstorms": "oil gas wellhead monitoring sandstorm damage maintenance",
    "Upstream explorations": "oil gas upstream exploration discovery",
    "Gas station site selection / Retail store location selection": "oil gas retail site selection location planning geospatial analysis",
    "Sea oil spill monitoring": "oil gas sea spill monitoring environmental detection satellite dynamic monitoring",
    "Network planning and Line-of-sight calculations": "telecommunications network planning line of sight antenna mapping analysis",
    "Network capacity monitoring": "telecommunications network monitoring capacity utilization optimization infrastructure maintenance",
    "Farm sourcing (palm-oil monitoring and mapping, hazelnut farm sourcing)": "agriculture farm sourcing mapping crop monitoring supply chain",
    "Supply chain planning and monitoring": "logistics transport supply chain planning monitoring ports warehouses disruption tracking",
    "Monitoring crop health, yield prediction and irrigation optimization": "agriculture crop health yield prediction irrigation monitoring remote sensing satellite imagery",
    "Assessment of property-level risk": "insurance risk modelling flood fire storm risk assessment geospatial evaluation",
    "Environmental Monitoring & ESG compliance": "esg environmental monitoring compliance sustainability deforestation pollution emissions tracking",
    "Disaster Response & Humanitarian Aid": "humanitarian disaster response mapping situational awareness flood earthquake crisis management"
}

# Industries tied to use cases
USECASE_INDUSTRIES = {
    "Oil & Gas": [
        "Pipeline integrity, oil rig monitoring and early leakage warning",
        "Wellhead damage monitoring from sandstorms",
        "Upstream explorations",
        "Gas station site selection / Retail store location selection",
        "Sea oil spill monitoring",
    ],
    "Telecommunications": [
        "Network planning and Line-of-sight calculations",
        "Network capacity monitoring",
    ],
    "Agriculture": [
        "Farm sourcing (palm-oil monitoring and mapping, hazelnut farm sourcing)",
        "Monitoring crop health, yield prediction and irrigation optimization",
    ],
    "Logistics & Transport": ["Supply chain planning and monitoring"],
    "Insurance & Risk Modelling": ["Assessment of property-level risk"],
    "Cross-industry / ESG": ["Environmental Monitoring & ESG compliance"],
    "Humanitarian / Non-profit": ["Disaster Response & Humanitarian Aid"],
}

# Keywords to detect new industries in JTBDs
EXTRA_INDUSTRY_KEYWORDS = {
    "Urban Planning": ["urban", "planning", "zoning", "city", "municipal"],
    "Education": ["education", "teacher", "school", "student", "university"],
    "Public Health": ["hospital", "health", "disease", "public health"],
    "Real Estate": ["property", "real estate", "housing"],
    "Environment": ["forest", "biodiversity", "ecosystem", "environmental"],
    "Utilities": ["electric", "power", "utility", "grid"],
}

# ============================================================
# LOAD JTBD DATA
# ============================================================

jtbd = pd.read_csv(JTBD_FILE)
jtbd = jtbd[jtbd["jtbd"].str.upper() == "YES"].copy()
jtbd["text"] = jtbd[["situation", "struggle", "outcome"]].fillna("").agg(" ".join, axis=1)

# ============================================================
# TF-IDF SIMILARITY
# ============================================================

vectorizer = TfidfVectorizer(stop_words="english")
jtbd_vecs = vectorizer.fit_transform(jtbd["text"])

matches = []

for uc, desc in USE_CASES.items():
    uc_vec = vectorizer.transform([desc])
    sims = cosine_similarity(uc_vec, jtbd_vecs).flatten()
    top_idx = [i for i in sims.argsort()[::-1] if sims[i] > SIM_THRESHOLD][:TOP_N]

    for i in top_idx:
        row = jtbd.iloc[i]
        matches.append({
            "use_case": uc,
            "situation": row.situation,
            "struggle": row.struggle,
            "outcome": row.outcome,
            "similarity": sims[i],
        })

matches_df = pd.DataFrame(matches)
matches_df.to_csv(OUTPUT_MATCH_CSV, index=False)

# Markdown export
with open(OUTPUT_MATCH_MD, "w", encoding="utf-8") as f:
    for uc in USE_CASES.keys():
        subset = matches_df[matches_df["use_case"] == uc]
        if subset.empty:
            continue
        f.write(f"## {uc}\n\n")
        f.write("| # | Situation | Struggle | Outcome |\n")
        f.write("|---|------------|-----------|----------|\n")
        for i, row in enumerate(subset.itertuples(index=False), 1):
            s = str(row.situation).replace("\n", " ").strip()
            st = str(row.struggle).replace("\n", " ").strip()
            o = str(row.outcome).replace("\n", " ").strip()
            f.write(f"| {i} | {s} | {st} | {o} |\n")
        f.write("\n")

print(f"✅ Saved matched JTBDs:\n  - {OUTPUT_MATCH_MD}\n  - {OUTPUT_MATCH_CSV}")

# ============================================================
# DETECT NEW INDUSTRIES IN JTBDs
# ============================================================

jtbd["industry_detected"] = None
for ind, kws in EXTRA_INDUSTRY_KEYWORDS.items():
    mask = jtbd["text"].str.lower().apply(lambda x: any(kw in x for kw in kws))
    jtbd.loc[mask, "industry_detected"] = ind

provided_industries = set(USECASE_INDUSTRIES.keys())
found_industries = set(jtbd["industry_detected"].dropna().unique())
new_industries = found_industries - provided_industries

with open(OUTPUT_NEW_INDS, "w", encoding="utf-8") as f:
    for ind in sorted(new_industries):
        f.write(f"{ind}\n")

print(f"✅ New industries detected (not in your use cases): {', '.join(new_industries) if new_industries else 'None found.'}")
print(f"📝 Saved to {OUTPUT_NEW_INDS}")

# ============================================================
# FIND UNMATCHED JTBDs
# ============================================================

jtbd["key"] = jtbd[["situation", "struggle", "outcome"]].fillna("").agg(" | ".join, axis=1)
matches_df["key"] = matches_df[["situation", "struggle", "outcome"]].fillna("").agg(" | ".join, axis=1)

unmatched = jtbd[~jtbd["key"].isin(matches_df["key"])]
unmatched.to_csv(OUTPUT_UNMATCHED, index=False)

print(f"✅ Exported {len(unmatched)} unmatched JTBDs → {OUTPUT_UNMATCHED}")
print("🎉 All done!")

# ============================================================
# MAP EACH JTBD TO NEWLY DETECTED INDUSTRIES
# ============================================================

industry_hits = []
for ind, kws in EXTRA_INDUSTRY_KEYWORDS.items():
    mask = jtbd["text"].str.lower().apply(lambda x: any(kw in x for kw in kws))
    for _, row in jtbd[mask].iterrows():
        industry_hits.append({
            "industry": ind,
            "situation": row["situation"],
            "struggle": row["struggle"],
            "outcome": row["outcome"]
        })

industry_df = pd.DataFrame(industry_hits)
industry_df.to_csv("jtbd_new_industries_mapping.csv", index=False)

print(f"✅ Exported {len(industry_df)} JTBDs mapped to new industries → jtbd_new_industries_mapping.csv")