import os import re import argparse import numpy as np import pandas as pd from pathlib import Path from collections import Counter from sklearn.cluster import KMeans from sklearn.preprocessing import StandardScaler from sklearn.metrics import silhouette_samples, silhouette_score from openai import OpenAI import pickle import hashlib import json from jsonschema import validate, ValidationError # === Initialize client === client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) # === helper: embeddings with cache === def embed_texts(texts, model_name="text-embedding-3-large", batch_size=100, cache_dir="embeddings_cache"): os.makedirs(cache_dir, exist_ok=True) joined = "\n".join(sorted(texts)).encode("utf-8") hash_id = hashlib.md5(model_name.encode("utf-8") + joined).hexdigest()[:12] cache_path = Path(cache_dir) / f"embeddings_{hash_id}.pkl" if cache_path.exists(): print(f"🔁 Loading cached embeddings from {cache_path}") with open(cache_path, "rb") as f: return pickle.load(f) print(f"⚙️ Generating new embeddings for {len(texts)} texts using {model_name} ...") embeddings = [] for i in range(0, len(texts), batch_size): batch = texts[i:i + batch_size] response = client.embeddings.create(model=model_name, input=batch) embeddings.extend([d.embedding for d in response.data]) embeddings = np.array(embeddings) with open(cache_path, "wb") as f: pickle.dump(embeddings, f) print(f"💾 Saved embeddings cache → {cache_path}") return embeddings # === helper: clustering === def cluster_kmeans(vectors, n_clusters): scaled = StandardScaler().fit_transform(vectors) kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10) labels = kmeans.fit_predict(scaled) return labels, kmeans.cluster_centers_ # === helper: silhouette === def compute_silhouette_by_cluster(embeddings, labels): if len(set(labels)) < 2: print("⚠️ Not enough clusters for silhouette score.") return pd.DataFrame(columns=["cluster", "mean_silhouette", "count"]) sample_sil = silhouette_samples(embeddings, labels) overall = silhouette_score(embeddings, labels) print(f"✅ Global silhouette score: {overall:.3f}") cluster_scores = [] for c in sorted(set(labels)): mask = labels == c cluster_scores.append({ "cluster": c, "mean_silhouette": float(np.mean(sample_sil[mask])), "count": int(np.sum(mask)) }) return pd.DataFrame(cluster_scores) # === helper: stopwords === def load_stopwords(): return set(["the", "and", "for", "that", "with", "you", "this", "are", "from", "was", "have", "not", "can", "but", "all", "your", "one", "use", "when", "want", "how", "why", "where", "what", "which", "they", "their", "our", "its", "because", "them", "too", "had", "has", "those", "these", "then"]) # === helper: schema for validation === CLUSTER_SCHEMA = { "type": "object", "properties": { "latent_theme": {"type": "string"}, "labels": { "type": "object", "properties": { "general_label": {"type": "string"}, "specific_label": {"type": "string"} }, "required": ["general_label", "specific_label"] }, "subclusters": { "type": "array", "items": { "type": "object", "properties": { "subcluster_label": {"type": "string"}, "description": {"type": "string"} }, "required": ["subcluster_label", "description"] } } }, "required": ["latent_theme", "labels", "subclusters"] } # === helper: semantic labeling via GPT === def describe_cluster_gpt(job_texts): joined_text = "\n".join(job_texts) prompt = f""" You are an expert in semantic clustering and product research using the Job-To-Be-Done (JTBD) framework. You will receive a list of job statements (each describing a user need). Your goal: Analyze them and produce a concise JSON object **strictly following the JSON Schema below**. ### JSON SCHEMA {json.dumps(CLUSTER_SCHEMA, indent=2)} Now analyze the following job statements and return your output **strictly matching this JSON Schema**. Do not include any text, commentary, or markdown outside the JSON. JOB STATEMENTS: {joined_text} """ try: response = client.chat.completions.create( model="gpt-4o-mini", messages=[{"role": "user", "content": prompt}], temperature=0.3, ) result = response.choices[0].message.content.strip() data = json.loads(result) validate(instance=data, schema=CLUSTER_SCHEMA) return data except Exception as e: print(f"⚠️ Failed to parse or validate GPT response: {e}") return { "latent_theme": "", "labels": {"general_label": "", "specific_label": ""}, "subclusters": [] } # === MAIN === parser = argparse.ArgumentParser() parser.add_argument("-i", "--input", required=True) args = parser.parse_args() df = pd.read_csv(args.input) input_path = Path(args.input) base_name = input_path.stem df_yes = df[df["jtbd"] == "YES"].drop_duplicates(subset=["comment"]) if "job_statement" in df_yes.columns: print("\n=== JOB STATEMENT CLUSTERING ===") job_texts = df_yes["job_statement"].dropna().unique().tolist() if len(job_texts) < 3: print("Not enough job statements to cluster.") else: job_emb = embed_texts(job_texts, model_name="text-embedding-3-large") n_clusters = min(12, len(job_texts)) labels, centers = cluster_kmeans(job_emb, n_clusters) df_jobs = pd.DataFrame({"job_statement": job_texts, "job_cluster": labels}) df_sil = compute_silhouette_by_cluster(job_emb, labels) # === simple lexical summaries summaries = [] for c in sorted(set(labels)): subset = df_jobs[df_jobs["job_cluster"] == c] blob = " ".join(subset["job_statement"].tolist()).lower() words = [w for w in re.findall(r"[a-z]{3,}", blob) if w not in load_stopwords()] common = [w for w, _ in Counter(words).most_common(8)] summaries.append({ "cluster": c, "count": len(subset), "top_words": ", ".join(common), "example": subset["job_statement"].iloc[0][:200] }) df_summary = pd.DataFrame(summaries).merge(df_sil, on="cluster", how="left") # === save cluster summaries out_summary = input_path.parent / f"{base_name}__job_statement_clusters.csv" df_summary.to_csv(out_summary, index=False) print("\nTop Job Statement Clusters:\n") print(df_summary.to_string(index=False)) # === map clusters back to main df df_map = df_jobs.set_index("job_statement")["job_cluster"].to_dict() df_yes["job_cluster"] = df_yes["job_statement"].map(df_map) out_path2 = input_path.parent / f"{base_name}__jtbd_with_job_clusters.csv" df_yes.to_csv(out_path2, index=False) print(f"Saved JTBD data with job clusters → {out_path2}") # === semantic labeling via GPT # === semantic labeling via GPT (with checkpointing) print("\n=== Generating semantic cluster summaries with GPT ===") meta_path = input_path.parent / f"{base_name}__cluster_summaries.json" cluster_summaries = [] # Dacă există deja fișierul (în caz că rularea a picat), îl continuăm if meta_path.exists(): try: cluster_summaries = json.loads(meta_path.read_text(encoding="utf-8")) processed_clusters = {c["cluster"] for c in cluster_summaries if "cluster" in c} print(f"🔁 Resuming from checkpoint: {len(processed_clusters)} clusters already processed.") except Exception: print("⚠️ Existing summaries file could not be read; starting fresh.") processed_clusters = set() else: processed_clusters = set() # Loop prin fiecare cluster for c in sorted(set(labels)): if c in processed_clusters: print(f"⏩ Skipping cluster {c} (already processed).") continue cluster_statements = df_jobs[df_jobs["job_cluster"] == c]["job_statement"].tolist() print(f"🔹 Processing cluster {c} ({len(cluster_statements)} statements)...") summary = describe_cluster_gpt(cluster_statements) summary["cluster"] = int(c) cluster_summaries.append(summary) # 🔸 Save checkpoint după fiecare cluster try: with open(meta_path, "w", encoding="utf-8") as f: json.dump(cluster_summaries, f, indent=2, ensure_ascii=False) print(f"💾 Checkpoint saved after cluster {c} → {meta_path}") except Exception as e: print(f"⚠️ Failed to save checkpoint for cluster {c}: {e}") print(f"\n✅ All cluster summaries saved → {meta_path}")