import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
import hashlib
import os
from dotenv import load_dotenv
load_dotenv(override=True)
import re
import fitz
import json
import requests
import threading
import queue
import time
import math
import random
import concurrent.futures
import tiktoken
from typing import Any, List

from tenacity import retry, wait_exponential_jitter, stop_after_attempt

from app.services.parser.hybrid_parser import HybridParser
from app.services.parser.smart_chunker import build_llm_context_from_chunks, generate_chunks, generate_parent_child_chunks
from app.services.parser.semantic_retriever import SemanticRetriever
from app.services.ai.ai_gateway import AIGateway, create_ai_gateway
from app.services.parser.llm_extractor import (
    clean_llm_json_string,
    sanitize_llm_field,
    LiteratureMetadata,
    LiteraturePillars,
    ExtractionResponse,
    extract_pillars_with_llm,
)

from pydantic import BaseModel, ConfigDict


class EdgeReasoningResponse(BaseModel):
    """Structured output schema for edge reasoning JSON responses.

    Enforces additionalProperties=false for OpenAI Structured Outputs
    strict mode compatibility.
    """

    model_config = ConfigDict(json_schema_extra={"additionalProperties": False})

    background: str
    methodology: str
    objective: str
    gap: str
    futurework: str

# Supabase repair utilities will be used at runtime only when needed
def _repair_supabase_if_missing(
    file_hash: str,
    pdf_path: str,
    filename: str,
    upload_queue: queue.Queue | None = None,
) -> None:
    """
    If a Neo4j node exists but Supabase lacks the raw chunks, extract and push chunks.

    This helper is intentionally lightweight and called only when we detect the
    Neo4j node exists but Supabase does not contain a matching `file_hash`.
    """
    try:
        from app.services.storage.supabase_client import check_supabase_hash, start_uploader_worker
    except Exception:
        logger.error("Supabase client utilities not available; cannot repair Supabase.", exc_info=True)
        return

    try:
        exists_in_supabase = check_supabase_hash(file_hash)
    except Exception as e:
        logger.error(f"Error checking Supabase for hash {file_hash}: {e}", exc_info=True)
        return

    if exists_in_supabase:
        return

    logger.warning(f"Neo4j node found but Supabase missing for {file_hash}. Attempting repair (push chunks)...")
    try:
        md = hybrid_parser.parse_pdf(pdf_path)
        if not md:
            logger.error(f"Repair failed: unable to extract markdown from {pdf_path}")
            return

        chunks = generate_chunks(md, file_hash=file_hash, title=filename)

        if not chunks:
            logger.warning(f"Repair aborted: no chunks produced for {pdf_path}")
            return

        own_worker = False
        if upload_queue is None:
            upload_queue, worker_thread = start_uploader_worker()
            own_worker = True
        else:
            worker_thread = None
        for c in chunks:
            upload_queue.put(c)

        # Graceful shutdown after upload only when this helper created the worker.
        if own_worker and worker_thread is not None:
            upload_queue.put("STOP")
            worker_thread.join(timeout=30)
        logger.info(f"Repair completed: pushed {len(chunks)} chunks to Supabase for {file_hash}")

    except Exception as e:
        logger.error(f"Exception during Supabase repair for {file_hash}: {e}", exc_info=True)
        return

# Inisialisasi Hybrid Parser global
hybrid_parser = HybridParser()

task_queue = queue.Queue()
# Global Semaphore to limit concurrent API calls to Alibaba DashScope to prevent 429 Too Many Requests
api_semaphore = threading.BoundedSemaphore(3)

# Tracker untuk mengukur waktu upload Supabase per dokumen.
# Key: file_hash, Value: {"start": float, "end": float, "chunks_remaining": int}
_supabase_timing: dict[str, dict] = {}
_supabase_timing_lock = threading.Lock()

def _env_flag(name: str, default: str = "0") -> bool:
    return os.getenv(name, default).strip().lower() in {"1", "true", "yes", "on"}


# When True, retrieval/semantic steps may use OpenAI embeddings instead of Alibaba.
# Must be defined at module import time because this file is imported by FastAPI routes.
USE_OPENAI_EMBEDDING = _env_flag("USE_OPENAI_EMBEDDING", "0")

# Tiktoken encoders untuk penghitungan token yang akurat.
# cl100k_base digunakan oleh GPT-4, text-embedding-3-small, dan model Qwen (kompatibel).
_tiktoken_encoder = tiktoken.get_encoding("cl100k_base")

# Mode audit untuk tuning tanpa manual review queue.
DRY_RUN_AUDIT_MODE = _env_flag("KG_DRY_RUN_AUDIT", "0")
AUDIT_VERBOSE = _env_flag("KG_AUDIT_VERBOSE", "1")

# Konfigurasi pilar dan threshold matrix baseline redesign_2.
PILLARS = ["background", "methodology", "gap", "objective", "futurework"]

# Static fallback threshold (digunakan jika JSON tidak tersedia)
PILLAR_THRESHOLD_MATRIX_FALLBACK = {
    "background": 0.6975,
    "methodology": 0.7208,
    "gap": 0.8115,
    "objective": 0.7300,
    "futurework": 0.8096,
}

VECTOR_TOP_K = 5

def get_current_thresholds() -> dict[str, float]:
    """
    Load dynamic thresholds from config/thresholds.json.
    Falls back to static PILLAR_THRESHOLD_MATRIX_FALLBACK if JSON is missing or invalid.
    
    Returns:
        dict[str, float]: Threshold values per pillar
    """
    config_path = os.path.join(os.path.dirname(__file__), "..", "..", "config", "thresholds.json")
    
    try:
        with open(config_path, "r") as f:
            data = json.load(f)
            
        thresholds = {
            pillar: data.get(pillar, PILLAR_THRESHOLD_MATRIX_FALLBACK[pillar])
            for pillar in PILLARS
        }
        
        logger.info("Loaded dynamic thresholds from JSON", extra={
            "source": "config/thresholds.json",
            "thresholds": thresholds
        })
        
        return thresholds
        
    except (FileNotFoundError, json.JSONDecodeError, KeyError) as e:
        logger.warning("Failed to load thresholds from JSON, using fallback", extra={
            "error": str(e),
            "fallback": PILLAR_THRESHOLD_MATRIX_FALLBACK
        })
        return PILLAR_THRESHOLD_MATRIX_FALLBACK.copy()

# Konfigurasi semantic coherence gate lintas pilar.
COHERENCE_MIN_MATCHED_PILLARS = 3 #Dipilih karena di bawah 3 matched pillar, edge cenderung lemah dan tidak koheren.
COHERENCE_GATE_RULES = {
    "methodology": ["objective", "gap"],
    "objective": ["background", "methodology", "gap"],
    "gap": ["objective", "methodology"],
    "futurework": ["objective", "methodology", "gap"],
}

# Gate kualitas reasoning supaya edge yang di-merge tidak memakai alasan generik.
REASONING_MIN_WORDS = 15
REASONING_FALLBACK_PHRASE = "bukti kemiripan substansial tidak cukup kuat"
REASONING_GENERIC_PATTERNS = [ #Maksud dari "generik" di sini adalah pola kalimat yang terlalu umum dan bisa muncul bahkan untuk node yang tidak relevan sama sekali, sehingga tidak boleh dijadikan satu-satunya alasan untuk menghubungkan dua node. Jika pola-pola ini muncul, harus ada alasan tambahan yang lebih spesifik untuk mendukung edge.
    r"\bsama[- ]?sama membahas\b",
    r"\bkeduanya membahas\b",
    r"\bmemiliki kemiripan\b",
    r"\bsecara umum\b",
    r"\btopik serupa\b",
    r"\bkeduanya berfokus pada\b",
    r"\bdalam konteks yang sama\b",
    r"\bhanya memberikan\b",
    r"\btidak ada bukti eksplisit\b",
]
REASONING_NATURAL_CONNECTORS = [
    "Berangkat dari fokus yang sama pada",
    "Sebagai titik temu, kedua studi ini",
    "Sementara literatur pertama mengeksplorasi",
    "literatur kedua melengkapinya dengan",
    "Keduanya terkonvergensi dalam upaya",
    "Pendekatan yang ditawarkan oleh sumber diperluas oleh target melalui",
    "Secara spesifik, sumber menyoroti",
    "yang sejalan dengan temuan target mengenai",
    "Irisan utama terletak pada",
    "Hal ini ditegaskan kembali oleh literatur target yang",
    "berbeda dengan target yang menerapkan",
    "Namun, keduanya bermuara pada kesimpulan bahwa",
]
REASONING_SPECIFICITY_KEYWORDS = {
    "metode", "metodologi", "pendekatan", "teknik", "strategi", "mekanisme", "prosedur",
    "eksperimen", "pengujian", "analisis", "observasi", "survei", "simulasi", "skenario",
    "evaluasi", "hasil", "metrik", "indikator", "parameter", "kriteria", "kinerja", "efisiensi", "akurasi", "efektivitas",
    "tujuan", "sasaran", "hipotesis", "fokus", "gap", "keterbatasan", "masalah", "kendala", "tantangan", "kelemahan",
    "variabel", "faktor", "pengaruh", "korelasi", "dampak", "penggunaan", "pemanfaatan", "implementasi", "sampel", "dataset", "data"
}
EDGE_TYPE_MAP = {
    "background": "SIMILAR_BACKGROUND",
    "methodology": "SIMILAR_METHODOLOGY",
    "gap": "SIMILAR_GAP",
    "objective": "SIMILAR_OBJECTIVE",
    "futurework": "SIMILAR_FUTUREWORK",
}

def get_md5_hash(filepath: str) -> str:
    """Generate MD5 hash of a file to ensure idempotency."""
    hasher = hashlib.md5()
    with open(filepath, "rb") as f:
        buf = f.read()
        hasher.update(buf)
    return hasher.hexdigest()


def compute_composite_id(user_id: str, project_id: str, file_hash: str) -> str:
    """Compute a stable composite identifier for multi-tenant isolation.

    The composite id prevents data collisions when the same PDF is uploaded by
    different users or in different projects.

    Args:
        user_id: Supabase user id.
        project_id: Project/workspace id.
        file_hash: MD5 hash of the PDF bytes.

    Returns:
        str: Hex-encoded MD5 composite id.
    """

    raw = f"{user_id}:{project_id}:{file_hash}".encode("utf-8")
    return hashlib.md5(raw).hexdigest()


def check_node_exists(
    hash_val: str,
    driver,
    project_id: str | None = None,
    user_id: str | None = None,
) -> bool:
    """Check if a Literature node exists.

    Backward compatibility:
        - If `user_id` is provided, we check `composite_id` first.
        - Otherwise, we fall back to legacy `{hash: <file_hash>}` lookup.

    Args:
        hash_val: File MD5 hash.
        driver: Neo4j driver.
        project_id: Optional project id for composite isolation.
        user_id: Optional user id for composite isolation.

    Returns:
        bool: True when a matching node exists.
    """

    if user_id and project_id:
        composite_id = compute_composite_id(user_id=user_id, project_id=project_id, file_hash=hash_val)
        cypher = "MATCH (n:Literature {composite_id: $composite_id}) RETURN elementId(n) AS node_id LIMIT 1"
        with driver.session() as session:
            record = session.run(cypher, composite_id=composite_id).single()
            if record is not None:
                return True

    cypher = "MATCH (n:Literature {hash: $hash_val}) RETURN elementId(n) AS node_id LIMIT 1"
    with driver.session() as session:
        record = session.run(cypher, hash_val=hash_val).single()
        return record is not None

def sanitize_markdown(markdown_text: str) -> str:
    """Sanitize trailing references/bibliography sections from Markdown text."""
    lines = markdown_text.splitlines()
    if not lines:
        return ""

    heading_patterns = [
        re.compile(r"^\s*#*\s*(references?|bibliography|daftar\s+pustaka|referensi)\s*:?\s*$", re.IGNORECASE),
        re.compile(r"^\s*#*\s*(\d+(?:\.\d+)*|[ivxlcdm]+)[\.)]?\s+(references?|bibliography|daftar\s+pustaka|referensi)\s*:?\s*$", re.IGNORECASE),
    ]
    citation_patterns = [
        re.compile(r"\[[0-9]{1,3}\]"),
        re.compile(r"\([12][0-9]{3}[a-z]?\)"),
        re.compile(r"\bdoi\b", re.IGNORECASE),
    ]

    def looks_like_reference_window(window_lines: list[str]) -> bool:
        non_empty = [ln.strip() for ln in window_lines if ln.strip()]
        if not non_empty:
            return False
        evidence = 0
        for ln in non_empty:
            if any(p.search(ln) for p in citation_patterns):
                evidence += 1
        return evidence >= max(2, len(non_empty) // 3)

    cut_index = None
    min_pos = max(5, int(len(lines) * 0.35))

    for idx, line in enumerate(lines):
        if idx < min_pos:
            continue
        if any(p.match(line) for p in heading_patterns):
            reference_window = lines[idx + 1: idx + 9]
            if looks_like_reference_window(reference_window) or idx >= int(len(lines) * 0.5):
                cut_index = idx
                break

    # Fallback: cari heading referensi yang muncul sebagai blok multiline.
    if cut_index is None:
        fallback = re.search(
            r"(?im)^\s*#*\s*(\d+(?:\.\d+)*|[ivxlcdm]+)?\s*[\.)-]?\s*(references?|bibliography|daftar\s+pustaka|referensi)\s*:?\s*$",
            markdown_text,
        )
        if fallback and fallback.start() > len(markdown_text) * 0.35:
            return markdown_text[:fallback.start()].strip()

    if cut_index is not None:
        return "\n".join(lines[:cut_index]).strip()

    return markdown_text.strip()

def is_valid_embedding(vec: list[float] | None) -> bool:
    """Check embedding validity for Neo4j vector query (finite values and non-zero norm)."""
    if not vec or len(vec) != 1536:
        return False
    if not all(math.isfinite(v) for v in vec):
        return False
    l2 = math.sqrt(sum(v * v for v in vec))
    return l2 > 0.0

def _prepare_chunk_records_for_persistence(
    chunk_records: list[dict],
    *,
    user_id: str | None,
    project_id: str,
) -> list[dict]:
    """Attach canonical persistence metadata before chunk rows are queued."""
    prepared_records = []
    for chunk in chunk_records:
        prepared_chunk = dict(chunk)
        prepared_chunk["user_id"] = user_id
        prepared_chunk["project_id"] = project_id
        prepared_chunk["embedding_status"] = "PENDING"
        prepared_chunk["error_log"] = None
        prepared_records.append(prepared_chunk)
    return prepared_records


class _GatewayChatCompletionsAdapter:
    """Expose the gateway through an OpenAI-compatible completions surface."""

    def __init__(self, gateway: AIGateway) -> None:
        """Initialize the adapter with a concrete gateway instance.

        Args:
            gateway: Active AI gateway runtime.
        """
        self._gateway = gateway

    def create(self, *, messages: list[dict[str, Any]], **kwargs: Any) -> Any:
        """Dispatch a chat completion through the gateway control plane.

        Args:
            messages: LLM messages payload.
            **kwargs: Provider-compatible generation options.

        Returns:
            Any: Raw provider response from the gateway.
        """
        return self._gateway.generate_llm_response(messages=messages, **kwargs)


class _GatewayChatAdapter:
    """Expose the nested chat namespace expected by legacy callers."""

    def __init__(self, gateway: AIGateway) -> None:
        """Initialize the chat adapter for a gateway instance.

        Args:
            gateway: Active AI gateway runtime.
        """
        self.completions = _GatewayChatCompletionsAdapter(gateway)


class GatewayLLMClientAdapter:
    """OpenAI-compatible client facade backed by the centralized AI gateway."""

    def __init__(self, gateway: AIGateway) -> None:
        """Initialize a client facade using the active gateway.

        Args:
            gateway: Active AI gateway runtime.
        """
        self._gateway = gateway
        self.chat = _GatewayChatAdapter(gateway)


def create_gateway_llm_client() -> GatewayLLMClientAdapter:
    """Create a gateway-backed LLM client facade for legacy call sites.

    Returns:
        GatewayLLMClientAdapter: OpenAI-compatible facade backed by the gateway.
    """
    return GatewayLLMClientAdapter(create_ai_gateway())




@retry(wait=wait_exponential_jitter(initial=2, max=60), stop=stop_after_attempt(10), reraise=True)
def embed_text(
    text: str,
    embedding_engine=None,
) -> list[float] | None:
    """Generate vector embedding for a given text through the gateway path."""
    cleaned = (text or "").strip()
    if not cleaned:
        return None

    try:
        active_engine = embedding_engine or create_ai_gateway().get_embedding_engine()
        with api_semaphore:
            vec = active_engine.embed(cleaned[:8000])
    except Exception as e:
        print(f"Peringatan: Gagal menghubungi API Embedding Cloud: {e}")
        return None

    if not is_valid_embedding(vec):
        print("Peringatan: Embedding tidak valid (norm=0/non-finite), pilar akan di-skip.")
        return None
    return vec

def create_literature_node(
    driver,
    file_hash: str,
    filename: str,
    metadata: dict,
    pillars: dict,
    vectors: dict,
    project_id: str = "workspace-dev-01",
    user_id: str | None = None,
    storage_path: str | None = None,
    storage_url: str | None = None,
) -> str:
    """Upsert a Literature node in Neo4j and return its elementId.

    Backward compatibility:
        - If `user_id` is not provided, the node is merged by legacy `hash`.
        - If `user_id` is provided, the node is merged by `composite_id` and keeps
          `hash` as the raw file hash for legacy anchoring (e.g. chat context).

    Args:
        driver: Neo4j driver.
        file_hash: MD5 hash of the PDF.
        filename: Original file name.
        metadata: Extracted metadata.
        pillars: Extracted pillar texts.
        vectors: Pillar embeddings.
        project_id: Project/workspace identifier.
        user_id: Optional user id for multi-tenant isolation.
        storage_path: Optional path in Supabase Storage.
        storage_url: Optional public URL of the PDF.

    Returns:
        str: Neo4j elementId for the node.
    """

    composite_id = None
    if user_id:
        composite_id = compute_composite_id(user_id=user_id, project_id=project_id, file_hash=file_hash)

    if composite_id:
        cypher = """
        MERGE (n:Literature {composite_id: $composite_id})
        SET
            n.hash = $hash_val,
            n.user_id = coalesce(n.user_id, $user_id),
            n.project_id = $project_id,
            n.filename = $filename,
            n.title = $title,
            n.keywords = $keywords,
            n.storage_path = $storage_path,
            n.storage_url = $storage_url,
            n.text_background = $bg_text,
            n.text_methodology = $met_text,
            n.text_gap = $gap_text,
            n.text_objective = $obj_text,
            n.text_futurework = $fut_text,
            n.embedding_background = $bg_vec,
            n.embedding_methodology = $met_vec,
            n.embedding_gap = $gap_vec,
            n.embedding_objective = $obj_vec,
            n.embedding_futurework = $fut_vec,
            n.status = 'PENDING',
            n.status_updated_at = timestamp()
        RETURN elementId(n) AS node_id
        """
    else:
        cypher = """
        MERGE (n:Literature {hash: $hash_val})
        SET
            n.project_id = $project_id,
            n.filename = $filename,
            n.title = $title,
            n.keywords = $keywords,
            n.storage_path = $storage_path,
            n.storage_url = $storage_url,
            n.text_background = $bg_text,
            n.text_methodology = $met_text,
            n.text_gap = $gap_text,
            n.text_objective = $obj_text,
            n.text_futurework = $fut_text,
            n.embedding_background = $bg_vec,
            n.embedding_methodology = $met_vec,
            n.embedding_gap = $gap_vec,
            n.embedding_objective = $obj_vec,
            n.embedding_futurework = $fut_vec,
            n.status = 'PENDING',
            n.status_updated_at = timestamp()
        RETURN elementId(n) AS node_id
        """
    
    # Use LLM-extracted title if valid, otherwise fallback to filename
    extracted_title = metadata.get("title", "").strip()
    if not extracted_title or extracted_title.lower() == "informasi tidak tersedia":
        final_title = filename[:-4] if filename.lower().endswith(".pdf") else filename
    else:
        final_title = extracted_title
        
    params = {
        "hash_val": file_hash,
        "filename": filename,
        "title": final_title,
        "keywords": metadata.get("keywords", []),
        "project_id": project_id,
        "storage_path": storage_path,
        "storage_url": storage_url,
        "bg_text": pillars.get("background", ""),
        "met_text": pillars.get("methodology", ""),
        "gap_text": pillars.get("gap", ""),
        "obj_text": pillars.get("objective", ""),
        "fut_text": pillars.get("futurework", ""),
        "bg_vec": vectors.get("background"),
        "met_vec": vectors.get("methodology"),
        "gap_vec": vectors.get("gap"),
        "obj_vec": vectors.get("objective"),
        "fut_vec": vectors.get("futurework")
    }

    if composite_id:
        params["composite_id"] = composite_id
        params["user_id"] = user_id
    
    with driver.session() as session:
        result = session.run(cypher, **params)
        record = result.single()
        return record["node_id"]

def process_io_bound_pdf(
    pdf_path: str,
    raw_markdown: str,
    metrics: dict,
    t_start_total: float,
    driver,
    llm_client,
    upload_queue: queue.Queue | None = None,
    embedding_engine=None,
    project_id: str = "workspace-dev-01",
    user_id: str | None = None,
    storage_path: str | None = None,
    storage_url: str | None = None,
    original_filename: str | None = None,
) -> tuple[str | None, dict | None]:
    """Execute retrieval, LLM extraction, embedding, and Neo4j injection (I/O Bound)."""
    filename = original_filename if original_filename else os.path.basename(pdf_path)
    file_hash = get_md5_hash(pdf_path)

    sanitized_text = sanitize_markdown(raw_markdown)
    chunk_records = generate_chunks(raw_markdown, file_hash=file_hash, title=filename)
    chunk_records = _prepare_chunk_records_for_persistence(
        chunk_records,
        user_id=user_id,
        project_id=project_id,
    )

    # FIX 1: STAGE 1 - Validate chunks locally (NO upload yet)
    if not chunk_records:
        print(f"Melewati file {pdf_path} karena tidak ada chunk dihasilkan.")
        metrics["status"] = "Failed"
        metrics["error_msg"] = "Tidak ada chunk dihasilkan"
        return None, metrics

    print(f"Chunk mentah berhasil dihasilkan: {len(chunk_records)} chunk (Untuk Supabase) [{filename}]")
    logger.info("Authoritative chunk persistence payload prepared", extra={"file_hash": file_hash, "project_id": project_id, "user_id": user_id, "chunk_count": len(chunk_records)})

    t_start_retrieval = time.time()
    # FASE 2 & 3: Multi-Level Chunking & Semantic Search
    parent_chunks, child_chunks = generate_parent_child_chunks(
        raw_markdown, file_hash=file_hash, title=filename
    )

    if not parent_chunks or not child_chunks:
        print(f"Melewati file {pdf_path} karena gagal membuat Parent/Child chunks.")
        metrics["status"] = "Failed"
        metrics["error_msg"] = "Gagal membuat Parent/Child chunks"
        metrics["time_retrieval"] = round(time.time() - t_start_retrieval, 2)
        return None, metrics

    print(f"Multi-Level Chunk: {len(parent_chunks)} Parent, {len(child_chunks)} Child [{filename}]")

    # Data Purification Layer (JIT sebelum retrieval dan LLM)
    def clean_text_noise(text):
        if not text: return ""
        text = re.sub(r'> \*\*\[Teks Gambar Hasil OCR - .*?\]\*\*(?::)?.*?(?=\n\n|\Z)', '', text, flags=re.DOTALL | re.IGNORECASE)
        text = re.sub(r'\*+- Start of picture text -\*+.*?\*+- End of picture text -\*+', '', text, flags=re.DOTALL | re.IGNORECASE)
        text = re.sub(r'(?im)^.*\|.*\|.*$', '', text)
        text = re.sub(r'(?im)^.*(?:e-?mail|@|korespondensi).*$', '', text)
        return text.strip()

    def is_content_garbage(text):
        if not text: return True
        lowered = text.lower()
        if any(kw in lowered for kw in ["acknowledgment", "acknowledgement", "ucapan terima kasih", "funded by", "didanai oleh", "thank"]):
            return True
        ref_brackets = len(re.findall(r'\[\d{1,3}\]', text))
        ref_years = len(re.findall(r'\(\d{4}\)', text))
        urls = len(re.findall(r'http[s]?://|doi\s*:', text, flags=re.IGNORECASE))
        if (ref_brackets + ref_years + urls) > 4 or urls > 1:
            return True
        return False

    purified_parents = []
    for p in parent_chunks:
        cleaned_content = clean_text_noise(p.get("content", ""))
        if is_content_garbage(cleaned_content) or len(cleaned_content) < 100:
            continue
        p["content"] = cleaned_content
        purified_parents.append(p)
            
    valid_parent_ids = {p["parent_id"] for p in purified_parents}
    purified_children = []
    for c in child_chunks:
        if c.get("parent_id") not in valid_parent_ids:
            continue
        cleaned_content = clean_text_noise(c.get("content", ""))
        if len(cleaned_content) < 30 or is_content_garbage(cleaned_content):
            continue
        c["content"] = cleaned_content
        purified_children.append(c)

    parent_chunks = purified_parents
    child_chunks = purified_children
    print(f"Data Purification Selesai: Tersisa {len(parent_chunks)} Parent, {len(child_chunks)} Child [{filename}]")
    # ----------------------------------------

    retriever = SemanticRetriever()
    query_texts = [
        "latar belakang masalah, urgensi penelitian, motivasi mendasar, fenomena krusial, konteks situasi saat ini, dan alasan utama studi ini dilakukan.",
        "rincian teknis metodologi, arsitektur sistem, desain eksperimen, teknik pengumpulan dataset, algoritma yang diusulkan, sampel, dan prosedur pengujian.",
        "keterbatasan penelitian terdahulu, kelemahan metode sebelumnya, kekosongan literatur, masalah yang belum terselesaikan, namun demikian, akan tetapi, dan hasil yang belum jelas.",
        "tujuan utama penelitian, sasaran spesifik, kontribusi inti makalah, hasil yang ingin dicapai, mengeksplorasi, dan solusi yang diusulkan untuk menyelesaikan masalah.",
        "saran untuk penelitian selanjutnya, rekomendasi pekerjaan di masa depan, arah pengembangan masa depan, dan peluang studi berikutnya untuk mengatasi batasan saat ini."
    ]

    top_parents_per_query = retriever.find_top_k_parents(query_texts, child_chunks, top_k=3)

    # FASE 4: Deduplikasi Set Unik
    unique_parent_ids = set()
    for q, p_ids in top_parents_per_query.items():
        unique_parent_ids.update(p_ids)
        
    print(f"Deduplikasi selesai: Ditemukan {len(unique_parent_ids)} Parent Chunks unik dari 5 pertanyaan pilar. [{filename}]")

    # Ambil teks utuh dari Parent Chunks yang unik
    selected_parent_texts = []
    for p in parent_chunks:
        if p["parent_id"] in unique_parent_ids:
            selected_parent_texts.append(p["content"])

    stuffed_context = "\n\n---\n\n".join(selected_parent_texts)
    
    # Injeksi halaman pertama (Metadata & Judul) agar LLM bisa membaca judul jurnal
    global_context = sanitized_text[:1500]
    stuffed_context = f"--- RAW DOCUMENT HEADER (WARNING: MAY CONTAIN JOURNAL PUBLICATION INFO) ---\n{global_context}\n\n" + stuffed_context
    
    if not stuffed_context:
        print(f"Peringatan: Tidak ada konteks relevan terdeteksi untuk LLM. Menggunakan fallback. [{filename}]")
        stuffed_context = sanitized_text[:25000]

    raw_text = stuffed_context # Assign ke raw_text untuk kompatibilitas dengan extract_pillars_with_llm
    metrics["time_retrieval"] = round(time.time() - t_start_retrieval, 2)
        
    try:
        # FIX 2: STAGE 2 - Extract pillars BEFORE uploading chunks
        # 3. Call LLM for Structured Output
        t_start_llm = time.time()
        
        try:
            extraction_result, usage = extract_pillars_with_llm(
                raw_text,
                llm_client,
                semaphore=api_semaphore,
            )
            pillars = extraction_result.get("pillars", {})
            metadata = extraction_result.get("metadata", {})
        except Exception as llm_err:
            print(f"[ERROR] LLM pillar extraction failed completely for {filename}: {llm_err}")
            empty_response = {
                "metadata": {"title": "Unknown", "keywords": []},
                "pillars": {"background": "", "methodology": "", "gap": "", "objective": "", "futurework": ""}
            }
            extraction_result, usage = empty_response, {"prompt_tokens": 0, "completion_tokens": 0}
            pillars = extraction_result.get("pillars", {})
            metadata = extraction_result.get("metadata", {})

        metrics["time_llm"] = round(time.time() - t_start_llm, 2)
        metrics["token_llm_in"] = len(_tiktoken_encoder.encode(raw_text[:25000]))
        metrics["token_llm_out"] = usage.get("completion_tokens", 0)
        
        # FIX 3: Validate pillar extraction succeeded
        if not pillars or all(not v for v in pillars.values()):
            print(f"Peringatan: Gagal ekstrak pilar dari file {pdf_path}. Melewati pemrosesan.")
            metrics["status"] = "Failed"
            metrics["error_msg"] = "Gagal ekstrak pilar JSON"
            return None, metrics
        
        # 4. Generate 5 Vectors
        vectors = {}
        active_embedding_engine = embedding_engine or create_ai_gateway().get_embedding_engine()
        valid_embedding_count = 0
        total_embed_in = 0
        total_embed_out = 0
        for key in ["background", "methodology", "gap", "objective", "futurework"]:
            pillar_text = (pillars.get(key, "") or "").strip()
            
            if not pillar_text:
                vectors[key] = None
                print(f"Peringatan: Pilar '{key}' kosong, embedding di-skip. [{filename}]")
                continue

            try:
                vec = embed_text(
                    pillar_text,
                    embedding_engine=active_embedding_engine,
                )
                total_embed_in += len(_tiktoken_encoder.encode(pillar_text))
                total_embed_out += 1536
            except Exception as emb_err:
                print(f"[ERROR] Embedding API failed completely for {filename}: {emb_err}")
                vec = None

            vectors[key] = vec
            if is_valid_embedding(vec):
                valid_embedding_count += 1

        metrics["token_embed_in"] = total_embed_in
        metrics["token_embed_out"] = total_embed_out
            
        t_start_db = time.time()
        # 5. Inject Node
        node_id = create_literature_node(
            driver,
            file_hash,
            filename,
            metadata,
            pillars,
            vectors,
            project_id=project_id,
            user_id=user_id,
            storage_path=storage_path,
            storage_url=storage_url,
        )
        metrics["time_db"] = round(time.time() - t_start_db, 2)
        print(f"Memproses Node {node_id}, masuk ke antrean!")

        if valid_embedding_count == 0:
            print(f"Peringatan: Semua embedding pilar tidak valid. Node disimpan, proses edge di-skip. [{filename}]")
            # Still return node_id, but don't queue for edge generation
        
        # FIX 4: STAGE 3 - Only NOW upload chunks (after node creation succeeded)
        if upload_queue is not None and chunk_records:
            with _supabase_timing_lock:
                _supabase_timing[file_hash] = {
                    "start": time.time(),
                    "end": None,
                    "chunks_total": len(chunk_records),
                    "chunks_remaining": len(chunk_records),
                }
            for chunk in chunk_records:
                upload_queue.put(chunk)
            print(f"Chunk berhasil dimasukkan ke antrean Supabase: {len(chunk_records)} chunk [{filename}]")
        
        # Membuang ke Queue untuk diolah relasinya oleh Worker Graph di Background
        if valid_embedding_count > 0:
            task_queue.put(node_id)
        
        metrics["status"] = "Success"
        metrics["time_total"] = round(time.time() - t_start_total, 2)
        return node_id, metrics
        
    except Exception as e:
        import traceback
        print(f"[ERROR] Gagal memproses file {pdf_path} pada fase I/O: {type(e).__name__}: {e}")
        print(f"  Traceback: {traceback.format_exc()[:500]}")
        print(f"  NOTE: Chunks NOT uploaded (transactional protection active) [{filename}]")
        metrics["status"] = "Error"
        metrics["error_msg"] = str(e)
        metrics["time_total"] = round(time.time() - t_start_total, 2)
        return None, metrics

def find_similar_nodes(
    driver,
    node_id: str,
    threshold_matrix: dict[str, float] | None = None,
    top_k: int = VECTOR_TOP_K,
    audit_events: list[dict] | None = None,
) -> list:
    """Find similar nodes in Neo4j using per-pillar threshold matrix."""
    threshold_matrix = threshold_matrix or get_current_thresholds()
    similar_nodes = []
    
    with driver.session() as session:
        for pillar in PILLARS:
            index_name = f"{pillar}_emb"
            prop_name = f"embedding_{pillar}"
            target_text_prop = f"text_{pillar}"
            pillar_threshold = threshold_matrix.get(pillar, PILLAR_THRESHOLD_MATRIX_FALLBACK[pillar])
            
            source_vec_cypher = f"""
            MATCH (source:Literature)
            WHERE elementId(source) = $node_id
            RETURN source.{prop_name} AS source_vec, source.project_id AS source_project_id
            """

            source_rec = session.run(source_vec_cypher, node_id=node_id).single()
            source_vec = source_rec["source_vec"] if source_rec else None
            source_project_id = source_rec["source_project_id"] if source_rec else None
            
            if not is_valid_embedding(source_vec):
                if audit_events is not None:
                    audit_events.append({
                        "stage": "similarity",
                        "status": "REJECT",
                        "pillar": pillar,
                        "target_id": "-",
                        "target_title": "-",
                        "score": 0.0,
                        "threshold": pillar_threshold,
                        "reason": "invalid-source-vector",
                    })
                continue

            cypher = f"""
            MATCH (source:Literature) WHERE elementId(source) = $node_id
            CALL db.index.vector.queryNodes('{index_name}', $top_k, $query_vector)
            YIELD node AS target, score
            WHERE elementId(target) <> elementId(source)
              AND target.project_id = $source_project_id
              AND NOT (source)-[]-(target)
            RETURN elementId(target) AS target_id,
                   target.title AS target_title,
                   target.keywords AS target_keywords,
                   target.{target_text_prop} AS target_pillar_text,
                   score
            ORDER BY score DESC
            """
            
            result = session.run(
                cypher,
                node_id=node_id,
                top_k=top_k,
                query_vector=source_vec,
                source_project_id=source_project_id,
            )
            for record in result:
                score = record["score"]
                passed = score >= pillar_threshold
                if audit_events is not None:
                    audit_events.append({
                        "stage": "similarity",
                        "status": "PASS" if passed else "REJECT",
                        "pillar": pillar,
                        "target_id": record["target_id"],
                        "target_title": record["target_title"],
                        "score": score,
                        "threshold": pillar_threshold,
                        "reason": "score>=threshold" if passed else "score<threshold",
                    })

                if not passed:
                    continue

                similar_nodes.append({
                    "target_id": record["target_id"],
                    "title": record["target_title"],
                    "target_keywords": list(record["target_keywords"]) if record["target_keywords"] else [],
                    "target_pillar_text": record["target_pillar_text"] or "",
                    "score": score,
                    "pillar": pillar,
                    "threshold": pillar_threshold,
                })
                
    return similar_nodes

def bundle_candidates_by_target(similar_nodes: list[dict]) -> dict[str, dict]:
    """Group raw per-pillar candidates into per-target bundles for coherence checks."""
    bundles = {}
    for item in similar_nodes:
        target_id = item["target_id"]
        if target_id not in bundles:
            bundles[target_id] = {
                "target_id": target_id,
                "title": item["title"],
                "keywords": item.get("target_keywords", []),
                "pillar_scores": {},
                "pillar_texts": {},
            }
        bundles[target_id]["pillar_scores"][item["pillar"]] = item["score"]
        bundles[target_id]["pillar_texts"][item["pillar"]] = item.get("target_pillar_text", "")
    return bundles

def validate_semantic_coherence(candidate_bundle: dict) -> tuple[bool, str]:
    """Validate whether a candidate has enough cross-pillar agreement to be merged."""
    pillar_scores = candidate_bundle.get("pillar_scores", {})
    matched_pillars = set(pillar_scores.keys())

    if len(matched_pillars) < COHERENCE_MIN_MATCHED_PILLARS:
        return False, f"single-pillar-match ({sorted(matched_pillars)})"

    # Kandidat dianggap tidak koheren jika pilar kuat berdiri sendiri tanpa pilar pendukung.
    for anchor, required_any in COHERENCE_GATE_RULES.items():
        if anchor in matched_pillars and not any(req in matched_pillars for req in required_any):
            return False, (
                f"anchor-without-support ({anchor} without any of {required_any}, "
                f"matched={sorted(matched_pillars)})"
            )

    return True, "coherent"

def evaluate_domain_compatibility_with_llm(
    client,
    keyword_source: list,
    keyword_target: list,
) -> dict:
    """Gate 3: Evaluate domain compatibility between two nodes using LLM semantic judgment.

    Returns a dict with keys:
    - is_compatible (bool): True if domains intersect or match, False if unrelated.
    - reason (str): Ultra-concise explanation under 10 words.

    Falls back to compatible=True on any error to avoid blocking the pipeline.
    """
    if not client:
        return {"is_compatible": True, "reason": "no-client-fallback"}

    user_content = (
        "Compare two sets of research keywords. Determine if they share the same "
        "overarching academic domain or have a strong interdisciplinary intersection.\n\n"
        "Rules:\n"
        "1. Focus on latent semantic meaning, not exact string matches.\n"
        "2. Treat concepts in different languages as identical if their meanings align.\n"
        "3. Resolve abbreviations to their full concepts.\n\n"
        "Return ONLY a valid JSON object with exactly two keys:\n"
        "- reason: Ultra-concise explanation of their relationship or mismatch (under 10 words).\n"
        "- is_compatible: boolean (true if domains intersect/match, false if unrelated).\n\n"
        f"Source Keywords: {keyword_source}\n"
        f"Target Keywords: {keyword_target}"
    )

    try:
        with api_semaphore:
            response = client.chat.completions.create(
                response_format={"type": "json_object"},
                messages=[
                    {
                        "role": "system",
                        "content": "You are an expert academic domain classifier.",
                    },
                    {"role": "user", "content": user_content},
                ],
                temperature=0.0,
                timeout=30.0,
            )
        raw = response.choices[0].message.content
        if not raw:
            raise ValueError("Empty response from LLM")
        parsed = json.loads(raw)
        return {
            "is_compatible": bool(parsed.get("is_compatible", True)),
            "reason": str(parsed.get("reason", "")).strip(),
        }
    except Exception as e:
        logger.warning(f"[Gate 3] evaluate_domain_compatibility_with_llm failed: {e}. Defaulting to compatible.")
        return {"is_compatible": True, "reason": "evaluation-error-fallback"}

@retry(wait=wait_exponential_jitter(initial=2, max=60), stop=stop_after_attempt(10), reraise=True)
def generate_edge_reasoning(
    client,
    source_title: str,
    target_title: str,
    pillars_data: dict[str, dict[str, str]],
) -> dict[str, str]:
    """Generate evidence-based edge reasoning for multiple pillars in a single batch call."""
    if not client:
        return {p: f"Mock reasoning for {p}" for p in pillars_data.keys()}

    categories = list(pillars_data.keys())

    pillars_text = ""
    for cat in categories:
        src_text = pillars_data[cat]["source"][:900]
        tgt_text = pillars_data[cat]["target"][:900]
        pillars_text += f"\n- [{cat.upper()}]\n  Source: {src_text}\n  Target: {tgt_text}\n"

    # Dynamic helper variables for armored prompt
    banned_phrases_text = ", ".join(
        [p.replace(r"\b", "").replace("[- ]?", " ") for p in REASONING_GENERIC_PATTERNS]
    )
    connectors_text = "\n- ".join(REASONING_NATURAL_CONNECTORS)
    keywords_text = ", ".join(list(REASONING_SPECIFICITY_KEYWORDS))

    prompt = f"""You are a Senior Academic Reviewer and Research Synthesis Specialist. Your job is to analyze excerpts from two research papers and build a Knowledge Graph by explaining their semantic intersection.

## 1. JSON STRUCTURE & MANDATORY KEYS
You must organize your JSON output to match this exact schema. Each key represents a specific academic pillar:
- background: Reasoning regarding the research context and motivation.
- methodology: Reasoning regarding the technical approach and execution.
- objective: Reasoning regarding the goals and contributions.
- gap: Reasoning regarding the limitations and unresolved problems.
- futurework: Reasoning regarding proposed extensions and recommendations.

---

## 2. TASK & STYLE GUIDELINES
For each requested pillar {categories}, generate highly specific reasoning justifying the semantic similarity between the Source and Target papers.
- WORD COUNT: Each paragraph MUST be between {REASONING_MIN_WORDS} and 60 words.
- ONE PARAGRAPH PER PILLAR: Write a single, flowing narrative per key.
- NATURAL ACADEMIC FLOW: Avoid robotic or template-like sentence structures.
- NO LISTS: Do not use bullet points or nested objects within the JSON values.

---

## 3. VOCABULARY & CONNECTORS (QUALITY STANDARDS)
You MUST weave at least one of these analytical terms: [{keywords_text}].
To ensure a natural academic flow, you are STRONGLY encouraged to use these connector phrases (or variations):
- {connectors_text}

---

## 4. GLOBAL CONSTRAINTS (STRICT ADHERENCE)
- ZERO HALLUCINATION: Extract ONLY from the provided excerpts.
- BANNED PHRASES: Never use generic fillers like: {banned_phrases_text}.
- INSUFFICIENT EVIDENCE: If technical overlap is missing, return EXACTLY the phrase "{REASONING_FALLBACK_PHRASE}" for that pillar key.
- PILLAR INTEGRITY: You MUST return all keys listed in Section 1. If a pillar was not requested or no data is provided for it, return an empty string "".

Source Title: {source_title}
Target Title: {target_title}

Compiled Text Excerpts:
{pillars_text}
"""

    with api_semaphore:
        response = client.chat.completions.create(
            response_format={
                "type": "json_schema",
                "json_schema": {
                    "name": "edge_reasoning_schema",
                    "strict": True,
                    "schema": EdgeReasoningResponse.model_json_schema(),
                },
            },
            messages=[
                {
                    "role": "system",
                    "content": (
                        "You are a Senior Academic Reviewer and Research Synthesis "
                        "Specialist. Your job is to analyze excerpts from two research "
                        "papers and build a Knowledge Graph by explaining their semantic "
                        "intersection."
                    ),
                },
                {"role": "user", "content": prompt}
            ],
            temperature=0.2,
            timeout=60.0
        )
    
    raw_json = response.choices[0].message.content
    if not raw_json:
        raise ValueError("Empty response content from LLM")

    validated = EdgeReasoningResponse.model_validate_json(raw_json)
    return validated.model_dump()

def validate_edge_reasoning_quality(reasoning: str) -> tuple[bool, str]:
    """Runtime quality gate untuk menolak reasoning fallback/generik sebelum merge."""
    text = (reasoning or "").strip()
    lowered = text.lower()

    if not text:
        return False, "empty-reasoning"

    if REASONING_FALLBACK_PHRASE in lowered:
        return False, "insufficient-evidence-fallback"

    word_count = len(re.findall(r"\w+", text, flags=re.UNICODE))
    if word_count < REASONING_MIN_WORDS:
        return False, f"too-short(wc={word_count})"

    specificity_hits = sum(1 for kw in REASONING_SPECIFICITY_KEYWORDS if kw in lowered)
    if specificity_hits < 1:
        return False, "missing-specific-evidence"

    generic_hits = any(re.search(pattern, lowered, flags=re.IGNORECASE) for pattern in REASONING_GENERIC_PATTERNS)
    if generic_hits:
        return False, "generic-reasoning-detected"

    return True, "quality-pass"

def create_lexicographical_edges_batch(
    driver,
    edges_data: list[dict],
):
    """Create multiple edges in a single Neo4j transaction using UNWIND."""
    if not edges_data:
        return []

    # Group edges by category/rel_type because MERGE doesn't allow dynamic relationship types directly in UNWIND easily
    grouped_edges = {}
    for edge in edges_data:
        rel_type = EDGE_TYPE_MAP.get(edge["category"])
        if not rel_type:
            print(f"[SKIP][EDGE] Kategori tidak dikenali: {edge['category']}")
            continue
        
        node_min = min(edge["source_id"], edge["target_id"])
        node_max = max(edge["source_id"], edge["target_id"])
        
        if rel_type not in grouped_edges:
            grouped_edges[rel_type] = []
            
        grouped_edges[rel_type].append({
            "node_min": node_min,
            "node_max": node_max,
            "score": edge["score"],
            "reasoning": edge["reasoning"],
            "matched_pillars": edge["matched_pillars"]
        })

    created_rel_ids = []
    with driver.session() as session:
        for rel_type, batch_data in grouped_edges.items():
            cypher = f"""
            UNWIND $batch AS edge
            MATCH (a:Literature) WHERE elementId(a) = edge.node_min
            MATCH (b:Literature) WHERE elementId(b) = edge.node_max
            MERGE (a)-[r:{rel_type}]->(b)
            ON CREATE SET
                r.score = edge.score,
                r.reasoning = edge.reasoning,
                r.matched_pillars = edge.matched_pillars,
                r.gate = 'similarity+coherence'
            ON MATCH SET
                r.score = edge.score,
                r.reasoning = edge.reasoning,
                r.matched_pillars = edge.matched_pillars,
                r.gate = 'similarity+coherence'
            RETURN elementId(r) as rel_id
            """
            result = session.run(cypher, batch=batch_data)
            created_rel_ids.extend([record["rel_id"] for record in result])
            
    return created_rel_ids

def worker_edge_generator(node_id: str, driver, client):
    """Worker function to generate AI Edges for a single node."""
    if node_id is None:
        return
        
    print(f"[Worker] Memeriksa Relasi Kemiripan Untuk Node ID: {node_id}")
    
    try:
        # Eksekusi RD-08 komputasi KNN Threshold
        if driver:
            with driver.session() as session:
                source_rec = session.run(
                    """
                    MATCH (n:Literature)
                    WHERE elementId(n) = $id
                    RETURN n.title as title,
                           n.keywords as keywords,
                           n.text_background as text_background,
                           n.text_methodology as text_methodology,
                           n.text_gap as text_gap,
                           n.text_objective as text_objective,
                           n.text_futurework as text_futurework
                    """,
                    id=node_id,
                ).single()
                source_title = source_rec["title"] if source_rec else "Unknown Source"
                keyword_source = list(source_rec["keywords"]) if source_rec and source_rec["keywords"] else []
                
            audit_events = []
            similar_targets = find_similar_nodes(
                driver,
                node_id,
                threshold_matrix=get_current_thresholds(),
                top_k=VECTOR_TOP_K,
                audit_events=audit_events,
            )

            if AUDIT_VERBOSE:
                for ev in audit_events:
                    print(
                        f"  -> [AUDIT][{ev['stage'].upper()}][{ev['status']}] "
                        f"[{ev['pillar'].upper()}] '{ev['target_title']}' "
                        f"score={ev['score']:.4f} thr={ev['threshold']:.2f} reason={ev['reason']}"
                    )

            edges_to_create = []
            if not similar_targets:
                print("  -> Tidak ada saudara sebidang yang lolos threshold matrix per pilar.")
            else:
                bundles = bundle_candidates_by_target(similar_targets)
                for bundle in bundles.values():
                    coherent, reason = validate_semantic_coherence(bundle)
                    if not coherent:
                        print(
                            f"  -> [REJECT][COHERENCE] Target '{bundle['title']}' "
                            f"ditolak: {reason}"
                        )
                        continue

                    if AUDIT_VERBOSE:
                        print(
                            f"  -> [AUDIT][COHERENCE][PASS] Target '{bundle['title']}' "
                            f"matched_pillars={sorted(bundle['pillar_scores'].keys())}"
                        )

                    domain_result = evaluate_domain_compatibility_with_llm(
                        client,
                        keyword_source,
                        bundle["keywords"],
                    )
                    if not domain_result["is_compatible"]:
                        print(
                            f"  -> [REJECT][DOMAIN] Target '{bundle['title']}' "
                            f"ditolak: {domain_result['reason']}"
                        )
                        continue

                    if AUDIT_VERBOSE:
                        print(
                            f"  -> [AUDIT][DOMAIN][PASS] Target '{bundle['title']}' "
                            f"reason={domain_result['reason']}"
                        )

                    matched_pillars = sorted(bundle["pillar_scores"].keys())
                    
                    pillars_to_reason = {}
                    for pillar_name, score in bundle["pillar_scores"].items():
                        if pillar_name not in EDGE_TYPE_MAP:
                            print(f"  -> [SKIP][EDGE] Pilar tidak canonical: {pillar_name}")
                            continue
                        
                        source_excerpt = (source_rec.get(f"text_{pillar_name}") if source_rec else "") or ""
                        target_excerpt = bundle["pillar_texts"].get(pillar_name, "")

                        pillars_to_reason[pillar_name] = {
                            "source": source_excerpt,
                            "target": target_excerpt,
                            "score": score
                        }

                    if not pillars_to_reason:
                        continue
                        
                    try:
                        reasonings = generate_edge_reasoning(
                            client,
                            source_title,
                            bundle['title'],
                            pillars_to_reason,
                        )
                    except Exception as edge_err:
                        print(f"[ERROR] Edge Reasoning API failed completely: {edge_err}")
                        reasonings = {p: "Reasoning auto-generation failed." for p in pillars_to_reason.keys()}

                    for pillar_name, data in pillars_to_reason.items():
                        score = data["score"]
                        current_thresholds = get_current_thresholds()
                        print(
                            f"  -> Jurnal Ditemukan: '{bundle['title']}' "
                            f"(Skor [{pillar_name.upper()}]: {score:.4f} | "
                            f"Threshold: {current_thresholds[pillar_name]:.2f})"
                        )

                        if DRY_RUN_AUDIT_MODE:
                            print(
                                f"     => [DRY-RUN] Edge [{pillar_name.upper()}] tidak di-merge "
                                f"(source={source_title}, target={bundle['title']})"
                            )
                            continue

                        reasoning = reasonings.get(pillar_name, "Reasoning auto-generation failed.")
                        
                        quality_ok, quality_reason = validate_edge_reasoning_quality(reasoning)
                        if not quality_ok:
                            print(
                                f"  -> [REJECT][REASONING] Target '{bundle['title']}' "
                                f"ditolak pada pilar {pillar_name}: {quality_reason}"
                            )
                            continue

                        edges_to_create.append({
                            "source_id": node_id,
                            "target_id": bundle["target_id"],
                            "score": score,
                            "reasoning": reasoning,
                            "category": pillar_name,
                            "matched_pillars": matched_pillars,
                        })

            if edges_to_create and not DRY_RUN_AUDIT_MODE:
                rel_ids = create_lexicographical_edges_batch(driver, edges_to_create)
                if rel_ids:
                    print(f"     => Berhasil Membentuk {len(rel_ids)} Edge secara Batch!")
                    
            # TANDAI NODE SEBAGAI SELESAI EDGE GENERATION
            if driver and not DRY_RUN_AUDIT_MODE:
                with driver.session() as session:
                    session.run(
                        "MATCH (n:Literature) WHERE elementId(n) = $id SET n.edges_generated = true",
                        id=node_id
                    )
                    print(f"     => [State Stamping] Node {node_id} ditandai edges_generated=true")
                    
    except Exception as e:
        print(f"[Worker Error] ID {node_id} gagal diproses: {e}")

# Blok Eksekusi Utama (Real Ingestion Pipeline)
if __name__ == "__main__":
    from dotenv import load_dotenv
    import sys
    import time
    from neo4j import GraphDatabase
    from app.services.storage.supabase_client import start_uploader_worker, stop_uploader_worker
    load_dotenv(override=True)
    
    NEO4J_URI = os.getenv("NEO4J_URI", "")
    NEO4J_USERNAME = os.getenv("NEO4J_USERNAME", "neo4j")
    NEO4J_PASSWORD = os.getenv("NEO4J_PASSWORD", "")
    PROJECT_ID = os.getenv("PROJECT_ID", "workspace-dev-01")

    if not all([NEO4J_URI, NEO4J_USERNAME, NEO4J_PASSWORD]):
        print("❌ Kredensial Neo4j belum lengkap.")
        sys.exit(1)
        
    try:
        gateway_llm_client = create_gateway_llm_client()
        gateway_embedding_engine = create_ai_gateway().get_embedding_engine()
    except Exception as exc:
        print(f"❌ Gateway AI belum siap: {exc}")
        sys.exit(1)

    driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USERNAME, NEO4J_PASSWORD))
    
    literatur_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "..", "Literature")
    if not os.path.exists(literatur_dir):
        print(f"❌ Direktori '{literatur_dir}' tidak ditemukan. Buat folder dan isi dengan PDF.")
        driver.close()
        sys.exit(1)
        
    pdf_files = [os.path.join(literatur_dir, f) for f in os.listdir(literatur_dir) if f.casefold().endswith(".pdf")]
    
    if not pdf_files:
        print(f"❌ Tidak ada file PDF di folder '{literatur_dir}'.")
        driver.close()
        sys.exit(1)
        
    # LIMIT FILES FOR TESTING
    #pdf_files = pdf_files[:1]
        
    print(f"==== MEMULAI NEO4J INGESTION PIPELINE ====")
    print(f"Target: {len(pdf_files)} Dokumen PDF")
    if DRY_RUN_AUDIT_MODE:
        print("Mode: DRY-RUN AUDIT aktif (edge tidak di-merge, audit log tetap ditampilkan).")
    
    try:
        # Callback untuk mencatat waktu selesai upload Supabase per dokumen.
        def _on_supabase_batch_complete(batch: list) -> None:
            now = time.time()
            for chunk in batch:
                fh = chunk.get("file_hash")
                if not fh:
                    continue
                with _supabase_timing_lock:
                    entry = _supabase_timing.get(fh)
                    if entry is None:
                        continue
                    entry["chunks_remaining"] = max(0, entry["chunks_remaining"] - 1)
                    if entry["chunks_remaining"] == 0 and entry["end"] is None:
                        entry["end"] = now

        # Jalankan Background Worker untuk menyimpan chunk mentah ke Supabase.
        supabase_upload_queue, supabase_worker_thread = start_uploader_worker(
            on_batch_complete=_on_supabase_batch_complete
        )
        
        # Siapkan ThreadPoolExecutor untuk I/O bound tasks
        max_workers = 3 # 3-5 workers is a good balance for rate limits and DB connection pool
        io_futures = []
        edge_futures = []
        all_metrics = []
        
        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
            # Mulai siklus Main Thread untuk mengekstraksi PDF (Nodes) secara sekuensial
            for i, pdf_path in enumerate(pdf_files, 1):
                print(f"\n[{i}/{len(pdf_files)}] Menginisiasi ekstraksi file: {os.path.basename(pdf_path)}")
                
                t_start_total = time.time()
                filename = os.path.basename(pdf_path)
                file_hash = get_md5_hash(pdf_path)
                
                metrics = {
                    "file_name": filename,
                    "file_hash": file_hash,
                    "status": "Skipped",
                    "total_pages": 0,
                    "time_parsing": 0.0,
                    "time_retrieval": 0.0,
                    "time_llm": 0.0,
                    "time_edge_gen": 0.0,
                    "time_db": 0.0,
                    "time_supabase": 0.0,
                    "time_total": 0.0,
                    "token_embed_in": 0,
                    "token_embed_out": 0,
                    "token_llm_in": 0,
                    "token_llm_out": 0,
                    "error_msg": ""
                }

                if check_node_exists(file_hash, driver):
                    try:
                        _repair_supabase_if_missing(
                            file_hash,
                            pdf_path,
                            filename,
                            upload_queue=supabase_upload_queue,
                        )
                    except Exception as e:
                        print(f"Warning: Supabase repair attempt failed for {file_hash}: {e}")

                    print(f"File {pdf_path} skipped (Already exist in Neo4j).")
                    all_metrics.append(metrics)
                    continue
                    
                print(f"Memproses (Parsing) {pdf_path}...")
                t_start_parsing = time.time()

                try:
                    _doc = fitz.open(pdf_path)
                    metrics["total_pages"] = len(_doc)
                    _doc.close()
                except Exception:
                    metrics["total_pages"] = 0

                try:
                    raw_markdown = hybrid_parser.parse_pdf(pdf_path)
                except Exception as e:
                    print(f"Gagal parse PDF {pdf_path}: {e}")
                    raw_markdown = None
                    
                metrics["time_parsing"] = round(time.time() - t_start_parsing, 2)
                
                if not raw_markdown:
                    print(f"Melewati file {pdf_path} karena gagal mengekstrak Markdown/Teks.")
                    metrics["status"] = "Failed"
                    metrics["error_msg"] = "Gagal ekstrak Markdown"
                    all_metrics.append(metrics)
                    continue

                # Serahkan I/O bound work ke ThreadPool
                future = executor.submit(
                    process_io_bound_pdf,
                    pdf_path,
                    raw_markdown,
                    metrics,
                    t_start_total,
                    driver,
                    gateway_llm_client,
                    supabase_upload_queue,
                    gateway_embedding_engine,
                    PROJECT_ID,
                )
                io_futures.append(future)
                
            print("\nSemua PDF telah melewati tahap Parsing (Main Thread). Menunggu proses I/O paralel (Retrieval, LLM, Inject)...")
            
            # Kumpulkan hasil dari I/O futures
            for future in concurrent.futures.as_completed(io_futures):
                try:
                    result = future.result()
                    if result:
                        processed_id, metrics = result
                        if metrics:
                            all_metrics.append(metrics)
                        if processed_id:
                            # Submit task edge generation dengan timing wrapper
                            t_edge_start = time.time()
                            edge_future = executor.submit(
                                worker_edge_generator,
                                processed_id,
                                driver,
                                gateway_llm_client,
                            )
                            # Simpan referensi metrics dan start time untuk update time_edge_gen setelah selesai
                            edge_futures.append((edge_future, metrics, t_edge_start))
                except Exception as e:
                    print(f"[Main Thread Exception] IO bound processing failed: {e}")
            
            print("Memeriksa penyelesaian seluruh relasi Graph (Edge Generation)...")
            
            # ThreadPoolExecutor automatically waits for all futures to complete upon exiting the 'with' block
        
        stop_uploader_worker(timeout=60)

        # Isi time_edge_gen per dokumen setelah semua edge futures selesai
        for edge_future, edge_metrics, t_edge_start in edge_futures:
            try:
                edge_future.result()
            except Exception as e:
                print(f"[Worker Exception] Edge generation failed: {e}")
            edge_metrics["time_edge_gen"] = round(time.time() - t_edge_start, 2)

        # Isi time_supabase per dokumen dari tracker
        for m in all_metrics:
            fh = m.get("file_hash", "")
            with _supabase_timing_lock:
                entry = _supabase_timing.get(fh)
            if entry and entry.get("start") and entry.get("end"):
                m["time_supabase"] = round(entry["end"] - entry["start"], 2)
            else:
                m["time_supabase"] = 0.0

        # Cetak Matriks Performa Standar Industri
        # Pricing constants
        PRICE_EMBED_PER_M = 0.020   # text-embedding-3-small: $0.020 / 1M tokens
        PRICE_LLM_IN_PER_M = 0.10   # qwen3-30b-a3b: $0.10 / 1M input tokens
        PRICE_LLM_OUT_PER_M = 0.30  # qwen3-30b-a3b: $0.30 / 1M output tokens

        col_doc   = 50
        col_page  = 6
        col_stat  = 10
        col_parse = 9
        col_retr  = 12
        col_llm   = 8
        col_edge  = 9
        col_neo4j = 8
        col_supa  = 9
        col_total = 9
        col_ei    = 10
        col_li    = 9
        col_lo    = 9
        col_ec    = 12
        col_lc    = 12

        sep = (
            "|-" + "-"*col_doc + "-|-" + "-"*col_page + "-|-" + "-"*col_stat + "-|-"
            + "-"*col_parse + "-|-" + "-"*col_retr + "-|-" + "-"*col_llm + "-|-"
            + "-"*col_edge + "-|-" + "-"*col_neo4j + "-|-" + "-"*col_supa + "-|-"
            + "-"*col_total + "-|-" + "-"*col_ei + "-|-" + "-"*col_li + "-|-"
            + "-"*col_lo + "-|-" + "-"*col_ec + "-|-" + "-"*col_lc + "-|"
        )
        total_width = len(sep)

        print("\n" + "="*total_width)
        print("PERFORMANCE MONITORING MATRIX")
        print("="*total_width)
        header = (
            f"| {'Nama Dokumen':<{col_doc}} | {'Pages':<{col_page}} | {'Status':<{col_stat}} |"
            f" {'Parse(s)':<{col_parse}} | {'Retrieval(s)':<{col_retr}} | {'LLM(s)':<{col_llm}} |"
            f" {'EdgeGen(s)':<{col_edge}} | {'Neo4j(s)':<{col_neo4j}} | {'Supa(s)':<{col_supa}} |"
            f" {'Total(s)':<{col_total}} | {'EmbedIn(t)':<{col_ei}} | {'LLMIn(t)':<{col_li}} |"
            f" {'LLMOut(t)':<{col_lo}} | {'EmbedCost($)':<{col_ec}} | {'LLMCost($)':<{col_lc}} |"
        )
        print(header)
        print(sep)
        for m in all_metrics:
            fname = (m['file_name'][:col_doc-3] + '...') if len(m['file_name']) > col_doc else m['file_name']
            embed_in = m.get('token_embed_in', 0)
            llm_in = m.get('token_llm_in', 0)
            llm_out = m.get('token_llm_out', 0)
            embed_cost = embed_in * PRICE_EMBED_PER_M / 1_000_000
            llm_cost = (llm_in * PRICE_LLM_IN_PER_M + llm_out * PRICE_LLM_OUT_PER_M) / 1_000_000
            row = (
                f"| {fname:<{col_doc}} | {m.get('total_pages', 0):<{col_page}} | {m['status']:<{col_stat}} |"
                f" {m['time_parsing']:<{col_parse}.2f} | {m['time_retrieval']:<{col_retr}.2f} | {m['time_llm']:<{col_llm}.2f} |"
                f" {m.get('time_edge_gen', 0.0):<{col_edge}.2f} | {m['time_db']:<{col_neo4j}.2f} | {m.get('time_supabase', 0.0):<{col_supa}.2f} |"
                f" {m['time_total']:<{col_total}.2f} | {embed_in:<{col_ei}} | {llm_in:<{col_li}} |"
                f" {llm_out:<{col_lo}} | ${embed_cost:<{col_ec-1}.6f} | ${llm_cost:<{col_lc-1}.6f} |"
            )
            print(row)
        print("="*total_width)
        
        print("\nEksekusi Pipeline Penuh Selesai Total. Relasi dan Node aman di Cloud.")
                
    finally:
        driver.close()
