import re import spacy import pandas as pd from sentence_transformers import SentenceTransformer from sklearn.cluster import KMeans # ------------------------- # 1. Setup # ------------------------- print("Loading spaCy model...") nlp = spacy.load("en_core_web_sm") print("Loading sentence transformer...") embedder = SentenceTransformer("all-MiniLM-L6-v2") # ------------------------- # Expanded dictionaries # ------------------------- # Perception verbs: how others see me perception_verbs = { "look","appear","seem","act","present","show","think","see", "judge","evaluate","view","perceive","consider","regard","believe","notice","call", "come","sound","feel" } # Aspirational verbs: who I want to become aspirational_verbs = { "be","become","grow","develop","turn","transform", "improve","achieve","reach","attain","evolve","change","succeed" } # Respect/Admiration/Trust verbs respect_verbs = { "respect","admire","trust","appreciate","value","honor","praise","acknowledge","recognize","validate" } # Audiences: who’s judging me audiences = { "people","others","friends","coworkers","boss","family","partner","manager","teacher","colleagues", "classmates","peers","society","team","employer","recruiter","interviewer","hr","supervisor", "clients","customers","professor","mentor","students","neighbors","community","followers", "audience","strangers","everyone" } # ------------------------- # Main extraction function # ------------------------- def extract_social_jobs(text: str, debug: bool = False): if not isinstance(text, str) or not text.strip(): return None doc = nlp(text) candidates = [] # ------------------------- # 1. Perception jobs (parsing) # ------------------------- for token in doc: if token.lemma_ in perception_verbs and token.pos_ == "VERB": audience = [w.text for w in token.sent if w.lemma_.lower() in audiences] descriptors = [child.text for child in token.subtree if child.pos_ in ("ADJ","NOUN")] if audience and descriptors: phrase = f"Be seen as {' '.join(descriptors)} by {' '.join(audience)}" candidates.append(("perception", phrase)) # ------------------------- # 2. Aspirational jobs (parsing) # ------------------------- for token in doc: if token.lemma_ in aspirational_verbs and token.pos_ == "VERB": if any(child.lemma_ in {"want","wish","hope","like"} for child in token.head.children): complements = [c.text for c in token.children if c.dep_ in ("acomp","attr","oprd","dobj","xcomp","ccomp")] if complements: phrase = f"Become {' '.join(complements)}" candidates.append(("aspirational", phrase)) # Regex fallback for aspirational regex_matches = re.findall(r"\bI (?:want|wish|hope|would like) to (?:be|become|improve|achieve|succeed)\s+([^.,;!?]+)", text, re.IGNORECASE) for m in regex_matches: candidates.append(("aspirational_regex", f"Become {m.strip()}")) # ------------------------- # 3. Respect/Admire/Trust jobs (parsing) # ------------------------- for token in doc: if token.lemma_ in respect_verbs and token.pos_ == "VERB": subject = [w.text for w in token.lefts if w.dep_ in ("nsubj","nsubjpass") and w.text.lower() not in {"i","me"}] if "me" in [w.text.lower() for w in token.rights] or "me" in [w.text.lower() for w in token.children]: if subject: phrase = f"Be {token.lemma_}ed by {' '.join(subject)}" else: phrase = f"Be {token.lemma_}ed" candidates.append(("respect", phrase)) # ------------------------- # 4. Regex fallbacks # ------------------------- regex_rules = [] # Avoidance regex avoid_matches = re.findall(r"\bI (?:don’t|do not|never) want to (?:look|seem|feel|appear|sound)\s+([^.,;!?]+)", text, re.IGNORECASE) for m in avoid_matches: regex_rules.append(("avoidance", f"Avoid being seen as {m.strip()}")) embar_matches = re.findall(r"\bI (?:don’t|do not|never) want to (?:embarrass|shame)\s+myself", text, re.IGNORECASE) for _ in embar_matches: regex_rules.append(("avoidance", "Avoid embarrassing myself")) # Roles regex role_matches = re.findall(r"\bmy (\w+)\s+(?:expects?|wants?) me to\s+([^.,;!?]+)", text, re.IGNORECASE) for audience, role in role_matches: regex_rules.append(("role", f"Be seen as someone who can {role.strip()} by {audience.strip()}")) # Comparisons regex comp_matches = re.findall(r"\bI (?:don’t|do not|never) want to (?:fall|lag|be left) behind\s+(?:my\s+)?(\w+)", text, re.IGNORECASE) for group in comp_matches: regex_rules.append(("comparison", f"Keep up with {group.strip()}")) # Only add regex if parsing caught nothing if not candidates: candidates.extend(regex_rules) if not candidates: return None # Debug logging if debug: print(f"\nTEXT: {text}") for rule, phrase in candidates: print(f" [{rule}] -> {phrase}") # Return just the phrases return list(set([phrase for _, phrase in candidates])) # ------------------------- # 3. Main pipeline # ------------------------- def main(input_csv: str, output_csv: str, num_clusters: int = 6): print(f"Loading data from {input_csv}...") df = pd.read_csv(input_csv) if "comment" not in df.columns: raise ValueError("CSV must have a column named 'comment'.") print("Extracting social job candidates...") df["social_jobs"] = df["comment"].apply(extract_social_jobs) df_jobs = df.explode("social_jobs").dropna(subset=["social_jobs"]).reset_index(drop=True) print(f"Extracted {len(df_jobs)} social job phrases from {len(df)} comments.") # ------------------------- # 4. Clustering # ------------------------- print("Embedding social jobs...") job_texts = df_jobs["social_jobs"].tolist() embeddings = embedder.encode(job_texts, show_progress_bar=True) print(f"Clustering into {num_clusters} groups...") kmeans = KMeans(n_clusters=num_clusters, random_state=42, n_init=10) df_jobs["cluster"] = kmeans.fit_predict(embeddings) # ------------------------- # 5. Save + preview # ------------------------- print(f"Saving clustered social jobs to {output_csv}...") df_jobs.to_csv(output_csv, index=False) # Preview: show top examples per cluster for c in range(num_clusters): examples = df_jobs[df_jobs["cluster"] == c]["social_jobs"].head(5).tolist() print(f"\nCluster {c} examples:") for e in examples: print(" -", e) if __name__ == "__main__": import sys if len(sys.argv) < 3: print("Usage: python extract_social_jobs.py input.csv output.csv [num_clusters]") else: input_csv = sys.argv[1] output_csv = sys.argv[2] num_clusters = int(sys.argv[3]) if len(sys.argv) > 3 else 6 main(input_csv, output_csv, num_clusters)