import pandas as pd import sys import os from transformers import pipeline import re # ✅ Use GoEmotions model import os from transformers import pipeline from huggingface_hub import snapshot_download def load_goemotions_pipeline(): model_name = "SamLowe/roberta-base-go_emotions" try: # First try: load directly (uses cache if available) return pipeline( "text-classification", model=model_name, return_all_scores=True, truncation=True ) except Exception as e: print(f"⚠️ Online load failed: {e}\nTrying local snapshot...") # Ensure snapshot is downloaded local_dir = snapshot_download(model_name) # Load from local snapshot return pipeline( "text-classification", model=local_dir, return_all_scores=True, truncation=True ) # Usage: emotion_classifier = load_goemotions_pipeline() # Force mappings PUSH = {"anger", "annoyance", "disappointment", "disapproval", "disgust", "embarrassment", "grief", "remorse", "sadness"} PULL = {"admiration", "amusement", "caring", "curiosity", "desire", "excitement", "gratitude", "joy", "love", "optimism", "pride", "relief"} ANXIETIES = {"fear", "nervousness", "confusion"} # ---------------------- # Feature extraction # ---------------------- def detect_forces(text, threshold=0.3): """Run GoEmotions and map to JTBD forces.""" if not isinstance(text, str) or not text.strip(): return None, None, None, None, None # Run model, get full distribution results = emotion_classifier(text[:512], top_k=None) # results is already a list of dicts emotions = [r["label"] for r in results if r["score"] >= threshold] forces = { "push": [e for e in emotions if e in PUSH], "pull": [e for e in emotions if e in PULL], "anxieties": [e for e in emotions if e in ANXIETIES], "habits": [], # leave empty, can be filled later } return ( "; ".join(emotions) if emotions else None, "; ".join(forces["push"]) if forces["push"] else None, "; ".join(forces["pull"]) if forces["pull"] else None, "; ".join(forces["anxieties"]) if forces["anxieties"] else None, "; ".join(forces["habits"]) if forces["habits"] else None, ) # ---------------------- # Main script # ---------------------- if __name__ == "__main__": infile = sys.argv[1] outfile = infile.replace(".csv", "_audience_forces.csv") # Load input df = pd.read_csv(infile) # Resume if partial output exists if os.path.exists(outfile): processed_df = pd.read_csv(outfile) start_idx = len(processed_df) print(f"🔄 Resuming from row {start_idx}...") else: processed_df = pd.DataFrame() start_idx = 0 results = [] for i in range(start_idx, len(df)): text = df.loc[i, "comment"] # 🛑 Skip junky short comments right away if not isinstance(text, str) or len(text.strip().split()) < 5: results.append({ "comment": text, "emotional_drivers": None, "push": None, "pull": None, "anxieties": None, "habits": None, }) continue row = detect_forces(text) results.append({ "comment": text, "emotional_drivers": row[0], "push": row[1], "pull": row[2], "anxieties": row[3], "habits": row[4], }) # Save every 100 rows if (i + 1) % 100 == 0 or i == len(df) - 1: chunk = pd.DataFrame(results) combined = pd.concat([processed_df, chunk], ignore_index=True) combined.to_csv(outfile, index=False) print(f"💾 Saved progress at row {i+1}/{len(df)}") results = [] processed_df = combined print(f"✅ Completed. Full output saved to {outfile}") # import pandas as pd # # Load your CSV # df = pd.read_csv("r_socialskills_infinite_scroll_comments_audience_forces.csv") # # Ensure the column exists and is treated as string # df['emotional_drivers'] = df['emotional_drivers'].astype(str).str.strip() # # Split emotions by comma, remove whitespace # df['emotions_list'] = df['emotional_drivers'].apply( # lambda x: [e.strip() for e in x.split(",") if e.strip() and e.lower() != "nan"] # ) # # Flatten list of all emotions # all_emotions = [em for sublist in df['emotions_list'] for em in sublist] # # Count frequencies # emotion_counts = pd.Series(all_emotions).value_counts() # print("=== Top Emotions ===") # print(emotion_counts.head(20)) # # Analyze combinations of emotions # df['emotion_combo'] = df['emotions_list'].apply(lambda lst: ", ".join(sorted(lst))) # combo_counts = df['emotion_combo'].value_counts() # print("\n=== Top Emotion Combinations ===") # print(combo_counts.head(20)) # # Save results to CSV # emotion_counts.to_csv("emotion_counts.csv", header=["count"]) # combo_counts.to_csv("emotion_combinations.csv", header=["count"]) # print("\nSaved 'emotion_counts.csv' and 'emotion_combinations.csv'") # df['emotional_drivers'] = df['emotional_drivers'].astype(str).str.lower().str.strip() # # Define priority groups # high_priority = {"curiosity","realization","confusion","anxiety", # "disapproval","annoyance","disappointment","sadness"} # medium_priority = {"caring","joy","admiration"} # low_priority = {"neutral","approval","gratitude","love","amusement"} # def assign_priority(drivers): # drivers = [d.strip() for d in drivers.split(",") if d.strip()] # if any(d in high_priority for d in drivers): # return "High" # if any(d in medium_priority for d in drivers): # return "Medium" # return "Low" # df["jtbd_priority"] = df["emotional_drivers"].apply(assign_priority) # # Count results # print(df["jtbd_priority"].value_counts()) # # # Save subsets # # df[df["jtbd_priority"]=="High"].to_csv("jtbd_high_priority.csv", index=False) # # df[df["jtbd_priority"]=="Medium"].to_csv("jtbd_medium_priority.csv", index=False) # # Clean comment text # df['comment'] = df['comment'].astype(str).str.strip() # # Structural filters # def passes_structural_filters(text): # words = text.split() # if len(words) < 8: # minimum length # return False # if not re.search(r"\b(I|me|my)\b", text, re.IGNORECASE): # first-person # return False # if not re.search(r"\b(because|so I can|when|in order to)\b", text, re.IGNORECASE): # return False # return True # df["structural_pass"] = df["comment"].apply(passes_structural_filters) # # Apply both emotion priority + structural filters # df["jtbd_final"] = df.apply( # lambda row: row["jtbd_priority"] if row["structural_pass"] else "Skip", axis=1 # ) # # Count results # print(df["jtbd_final"].value_counts()) # # Clean up # df['comment'] = df['comment'].astype(str).str.strip() # df['emotional_drivers'] = df['emotional_drivers'].astype(str).str.lower().str.strip() # # Structural filters # def passes_structural_filters(text): # words = text.split() # if len(words) < 8: # minimum length # return False # if not re.search(r"\b(I|me|my)\b", text, re.IGNORECASE): # first-person # return False # if not re.search(r"\b(because|so I can|when|in order to)\b", text, re.IGNORECASE): # causal markers # return False # return True # df["structural_pass"] = df["comment"].apply(passes_structural_filters) # # Keep only comments that pass filters # df_structural = df[df["structural_pass"] == True] # # Split emotions by comma and flatten # df_structural['emotions_list'] = df_structural['emotional_drivers'].apply( # lambda x: [e.strip() for e in x.split(",") if e.strip() and e.lower() != "nan"] # ) # all_emotions_structural = [em for sublist in df_structural['emotions_list'] for em in sublist] # # Count frequencies # emotion_counts_structural = pd.Series(all_emotions_structural).value_counts() # print("=== Emotions in structurally valid JTBD comments ===") # print(emotion_counts_structural.head(20))