🚀 FINAL EXTRACTION FIX: Unified scoring and legacy response compatibility

app/services/ocr_service.py

@@ -180,23 +180,6 @@ class OCRService:
             logger.error(f"Image OCR extraction failed: {e}")
             raise
     
-    def _should_use_ocr(self, text: str) -> bool:
-        """Determines if OCR should be used based on text quality."""
-        if not text or len(text.strip()) < self.min_text_density:
-            return True
-            
-        # Check for junk character signatures (common in mis-encoded PDFs)
-        junk_patterns = [
-            r'^[a-zA-Z\s\-\.]{1,100}$', # Only short random letters
-            r'(\b\w\b\s+){5,}',        # Too many single-character words separated by spaces
-            r'[^\x00-\x7F]{5,}'        # Too many non-ASCII characters
-        ]
-        import re
-        for pattern in junk_patterns:
-            if re.search(pattern, text):
-                return True
-        return False
-
     def _preprocess_image(self, image) -> object:
         """
         Preprocess image for better OCR accuracy using adaptive thresholding.
@@ -271,25 +254,32 @@ class OCRService:
                 prev_blank = True
         
         return '\n'.join(cleaned_lines)
-    
+
     def _should_use_ocr(self, text: str) -> bool:
         """
-        Determine if OCR should be used instead of native text extraction.
-        
-        Args:
-            text: Text from native PDF extraction
-            
-        Returns:
-            True if OCR should be used, False otherwise
+        Determines if OCR should be used based on text quality, density, and 
+        character encoding issues common in mis-encoded PDFs.
         """
         if not text or len(text.strip()) < self.min_text_density:
             return True
         
+        # Check for junk character signatures (common in mis-encoded PDFs)
+        import re
+        junk_patterns = [
+            r'^[a-zA-Z\s\-\.]{1,100}$',  # Only short random letters
+            r'(\b\w\b\s+){5,}',         # Too many single-character words separated by spaces
+            r'[^\x00-\x7F]{10,}'         # Too many non-ASCII characters in a block
+        ]
+        
+        for pattern in junk_patterns:
+            if re.search(pattern, text):
+                logger.info(f"Junk pattern matched in text: {pattern}")
+                return True
+
         # Check for junk characters that indicate broken PDF encoding
         junk_indicators = [
-            '',  # Replacement character
-            '',  # Unicode replacement char
-            '\x00',  # Null byte
+            '\ufffd',  # Replacement character
+            '\u0000',  # Null byte
             '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', '\x08', '\x09',
             '\x0b', '\x0c', '\x0e', '\x0f',  # Control characters except \n, \r, \t
         ]
@@ -299,11 +289,13 @@ class OCRService:
         
         # If more than 1% junk characters, use OCR
         if junk_ratio > 0.01:
+            logger.info(f"Junk character ratio too high: {junk_ratio:.2%}")
             return True
         
         # Check for excessive non-alphabetic characters (indicative of encoding issues)
         alpha_ratio = sum(c.isalpha() or c.isspace() for c in text) / len(text) if text else 0
         if alpha_ratio < 0.7:  # Less than 70% readable characters
+            logger.info(f"Alpha character ratio too low: {alpha_ratio:.2%}")
             return True
         
         return False

app/tasks/pipeline.py

@@ -73,63 +73,34 @@ def process_job(job) -> None:
         contact_portfolio = m.group(0).rstrip(".,;") if m else None
         if contact_portfolio in (contact_linkedin, contact_github):
             contact_portfolio = None
-
+        
         safe_text = strip_pii_for_models(resume_text)
 
-        entities = parse_entities(safe_text)
-        skill_matches = match_skills_to_job(entities.get("skills", []), job.job_description)
-
-        required = extract_required_skills_from_job(job.job_description)
-        matched_set = {m["skill"].lower() for m in skill_matches if m.get("skill")}
-        missing = [s for s in required if s.lower() not in matched_set]
-
-        score_payload = score_components(entities, skill_matches, resume_text)
-        suggestions = generate_feedback_list(entities, resume_text, score_payload, missing)
-
-        prof_entities = entities.get("professional_details", {}) if isinstance(entities, dict) else {}
-        exp_val = prof_entities.get("experience")
-        exp_items: list[dict] = exp_val if isinstance(exp_val, list) else []
-        exp_text = "\n".join([str(x.get("description") or "").strip() for x in exp_items if isinstance(x, dict) and (x.get("description") or "").strip()])
-
-        # Build structured_data using canonical Pydantic schema
+        # *** PHASE 1: Enhanced Structured Extraction ***
+        # We perform this first so matching and scoring have the best possible data.
+        
+        # Start with simple Pydantic model
         cv_data = StructuredCV(
             personal_details=PersonalDetails(
-                full_name=entities.get("personal_details", {}).get("full_name"),
-                email=contact_email or entities.get("personal_details", {}).get("email"),
-                phone=contact_phone or entities.get("personal_details", {}).get("phone"),
-                address=entities.get("personal_details", {}).get("address"),
-                dob=entities.get("personal_details", {}).get("dob"),
-                linkedin=contact_linkedin or entities.get("personal_details", {}).get("linkedin"),
-                github=contact_github or entities.get("personal_details", {}).get("github"),
-                portfolio=contact_portfolio or entities.get("personal_details", {}).get("portfolio"),
+                full_name=None,
+                email=contact_email,
+                phone=contact_phone,
+                linkedin=contact_linkedin,
+                github=contact_github,
+                portfolio=contact_portfolio,
             ),
-            professional_summary="\n".join((entities.get("summary") or [])[:8]).strip() if isinstance(entities, dict) and entities.get("summary") else "",
-            work_experience=[
-                WorkExperienceItem(
-                    company=exp.get("company"),
-                    title=exp.get("title"),
-                    start_date=exp.get("start_date"),
-                    end_date=exp.get("end_date"),
-                    description=exp.get("description")
-                ) for exp in (entities.get("professional_details", {}).get("experience") or [])
-            ],
-            education=[
-                EducationItem(
-                    institution=edu.get("institution"),
-                    degree=edu.get("degree"),
-                    field=edu.get("field"),
-                    start_date=edu.get("start_date"),
-                    end_date=edu.get("end_date")
-                ) for edu in (entities.get("education_details", {}).get("education") or [])
-            ],
-            skills=entities.get("skills", []) or [],
-            certifications=entities.get("education_details", {}).get("certifications") or [],
-            languages=entities.get("education_details", {}).get("languages") or [],
+            professional_summary="",
+            work_experience=[],
+            education=[],
+            skills=[],
+            certifications=[],
+            languages=[],
         )
 
+        # 1. AI Structured Extraction (NuExtract)
         llm_structured = extract_structured_cv(resume_text)
         if isinstance(llm_structured, dict):
-            # Update Pydantic model with LLM results if available
+            # Update Pydantic model with LLM results
             for k in ("personal_details", "education_details", "professional_details"):
                 if isinstance(llm_structured.get(k), dict):
                     if k == "personal_details":
@@ -137,7 +108,6 @@ def process_job(job) -> None:
                             if pv and hasattr(cv_data.personal_details, pk):
                                 setattr(cv_data.personal_details, pk, pv)
                     elif k == "professional_details":
-                        # Map LLM professional_details to summary, experience, skills
                         if llm_structured[k].get("bio"):
                             cv_data.professional_summary = llm_structured[k]["bio"]
                         
@@ -152,44 +122,52 @@ def process_job(job) -> None:
                                     description=exp.get("description")
                                 ) for exp in llm_exp if isinstance(exp, dict)
                             ]
+                        
+                        if llm_structured[k].get("skills"):
+                            cv_data.skills = llm_structured[k]["skills"]
+                            
                     elif k == "education_details":
-                        # Similar mapping for education
                         llm_edu = llm_structured[k].get("education")
                         if isinstance(llm_edu, list) and len(llm_edu) > 0:
                             cv_data.education = [
                                 EducationItem(
-                                    institution=edu.get("institution"),
+                                    institution=edu.get("university") or edu.get("institution"),
                                     degree=edu.get("degree"),
                                     field=edu.get("field"),
                                     start_date=edu.get("start_date"),
                                     end_date=edu.get("end_date")
                                 ) for edu in llm_edu if isinstance(edu, dict)
                             ]
+                        if llm_structured[k].get("certifications"):
+                            cv_data.certifications = llm_structured[k]["certifications"]
+                        if llm_structured[k].get("languages"):
+                            cv_data.languages = llm_structured[k]["languages"]
 
         structured_data = cv_data.model_dump()
 
-        # *** ENHANCED EXTRACTION (optional; disable with ENABLE_ENHANCED_SKILLS=false for faster validation) ***
+        # 2. Pattern Matching & Domain-Specific Extractors
         if settings.enable_enhanced_skills:
             skills_extractor = EnhancedSkillsExtractor()
             experience_parser = ImprovedExperienceParser()
             cert_extractor = CertificationEnhancement()
 
-            enhanced_skills = skills_extractor.extract_skills(resume_text)
-            if enhanced_skills:
-                # Convert list of dicts to list of strings for consistency
+            enhanced_skills_objs = skills_extractor.extract_skills(resume_text)
+            if enhanced_skills_objs:
                 skill_names = []
-                for s in enhanced_skills:
+                for s in enhanced_skills_objs:
                     if isinstance(s, dict):
                         skill_names.append(s.get("name", ""))
                     else:
                         skill_names.append(str(s))
                 
-                if len(skill_names) > len(structured_data.get("skills", [])):
-                    structured_data["skills"] = list(set(skill_names)) # Deduplicate
+                # Merge and deduplicate
+                current_skills = set(structured_data.get("skills", []))
+                current_skills.update(skill_names)
+                structured_data["skills"] = list(filter(None, current_skills))
 
             enhanced_experience = experience_parser.parse(resume_text)
-            if enhanced_experience and len(enhanced_experience) > len(structured_data.get("experience", [])):
-                structured_data["experience"] = [
+            if enhanced_experience and len(enhanced_experience) > len(structured_data.get("work_experience", [])):
+                structured_data["work_experience"] = [
                     {
                         "title": exp.get("title"),
                         "company": exp.get("company"),
@@ -203,9 +181,32 @@ def process_job(job) -> None:
             if enhanced_certs and len(enhanced_certs) > len(structured_data.get("certifications", [])):
                 structured_data["certifications"] = enhanced_certs
 
-        # Simple extraction suggestions (e.g., missing LinkedIn, missing email)
+        # *** PHASE 2: Matching and Scoring using Unified Data ***
+        
+        entities = parse_entities(safe_text)
+        # Force structured results into entities for scoring
+        final_skills = structured_data.get("skills", [])
+        if not final_skills:
+            # Fallback to simple NER if enhanced failed
+            final_skills = entities.get("skills", [])
+        
+        skill_matches = match_skills_to_job(final_skills, job.job_description)
+
+        required = extract_required_skills_from_job(job.job_description)
+        matched_set = {m["skill"].lower() for m in skill_matches if m.get("skill")}
+        missing = [s for s in required if s.lower() not in matched_set]
+
+        # Use improved entities for scoring
+        scoring_entities = entities.copy()
+        scoring_entities["skills"] = final_skills
+        scoring_entities.setdefault("professional_details", {})["experience"] = structured_data.get("work_experience", [])
+        
+        score_payload = score_components(scoring_entities, skill_matches, resume_text)
+        suggestions = generate_feedback_list(scoring_entities, resume_text, score_payload, missing)
+
+        # Simple extraction suggestions
         extraction_suggestions = []
-        pd = structured_data.get("personal_details", {}) if isinstance(structured_data, dict) else {}
+        pd = structured_data.get("personal_details", {})
         if not pd.get("linkedin"):
             extraction_suggestions.append("Add a LinkedIn URL to your profile.")
         if not pd.get("email"):
@@ -226,42 +227,15 @@ def process_job(job) -> None:
         # Merge static and LLM suggestions
         match_suggestions = suggestions + (llm_suggestions if isinstance(llm_suggestions, list) else [])
 
-        # Generate autofill data (simple, reliable mapping)
-        autofill_data = None
-        try:
-            # Simple direct mapping from structured_data - this always works
-            autofill_data = {
-                "personal": structured_data.get("personal_details", {}),
-                "experience": [
-                    {
-                        "title": exp.get("title"),
-                        "company": exp.get("company"),
-                        "start_date": exp.get("start_date"),
-                        "end_date": exp.get("end_date"),
-                        "description": exp.get("description"),
-                        "location": exp.get("location")
-                    } for exp in structured_data.get("work_experience", [])
-                ],
-                "education": [
-                    {
-                        "degree": edu.get("degree"),
-                        "university": edu.get("institution"),
-                        "start_date": edu.get("start_date"),
-                        "end_date": edu.get("end_date"),
-                        "field": edu.get("field")
-                    } for edu in structured_data.get("education", [])
-                ],
-                "skills": structured_data.get("skills", []),
-                "certifications": structured_data.get("certifications", []),
-                "languages": structured_data.get("languages", [])
-            }
-            
-            logger.info(f"Autofill data generated successfully: {len(str(autofill_data))} characters")
-            
-        except Exception as e:
-            logger.error(f"Autofill data generation failed: {e}")
-            logger.error(f"Structured data keys: {list(structured_data.keys()) if structured_data else 'N/A'}")
-            autofill_data = None
+        # Generate autofill data
+        autofill_data = {
+            "personal": structured_data.get("personal_details", {}),
+            "experience": structured_data.get("work_experience", []),
+            "education": structured_data.get("education", []),
+            "skills": structured_data.get("skills", []),
+            "certifications": structured_data.get("certifications", []),
+            "languages": structured_data.get("languages", [])
+        }
 
         normalized = normalize_analysis_result(
             analysis_id=str(analysis_id),
@@ -270,14 +244,19 @@ def process_job(job) -> None:
             component_scores=score_payload.get("component_scores"),
             evidence=evidence,
             suggestions=match_suggestions,
-            raw_payload={"entities": entities, "skill_matches": skill_matches},
-            extraction_metadata={"method": "direct_text", "confidence": None, "pages": None, "has_scanned_content": False},
+            raw_payload={"entities": entities, "skill_matches": skill_matches, "cv_text": resume_text},
+            extraction_metadata={"method": "hybrid_ocr_extraction", "confidence": None, "pages": None, "has_scanned_content": False},
             structured_data=structured_data,
             extraction_suggestions=extraction_suggestions,
             interview_questions=interview_questions,
         )
         
-        # Add autofill data to response if generated
+        # 🔥 COMPATIBILITY SHIM: Add top-level fields for user test scripts
+        normalized["skills"] = final_skills
+        normalized["experience"] = structured_data.get("work_experience", [])
+        normalized["certifications"] = structured_data.get("certifications", [])
+        normalized["personal_details"] = structured_data.get("personal_details", {})
+        
         if autofill_data:
             normalized["autofill_data"] = autofill_data