Untitled

# pdf_handler.py
# Utilities for extracting text sections from PDFs and chunking them for Telegram

from typing import List, Dict
from pathlib import Path
import re

from PyPDF2 import PdfReader


def _clean_line(line: str) -> str:
    return line.strip().replace('\xa0', ' ')


    def extract_sections_from_pdf(pdf_path: str) -> List[Dict]:
        """Extracts logical sections from a single-page or multi-page editorial PDF.

            Returns a list of dicts: { 'title': str, 'text': str }

                This implementation is intentionally conservative: it will try to remove
                    obvious headers/footers (short repeated lines), and then split the body into
                        sections by detecting lines that look like titles (short, uppercase/Title-case)
                            or large whitespace breaks.
                                """
                                    p = Path(pdf_path)
                                        reader = PdfReader(str(p))

                                            pages = []
                                                for page in reader.pages:
                                                        raw = page.extract_text() or ""
                                                                pages.append(raw)

                                                                    # Merge pages into single text for processing
                                                                        full_text = "\n\n".join(pages)

                                                                            # Heuristics: remove repeated header/footer
                                                                                lines = [l for l in (l.rstrip() for l in full_text.splitlines()) if l.strip()]

                                                                                    # find short lines that appear on multiple pages -> header/footer candidates
                                                                                        counts = {}
                                                                                            for i, line in enumerate(lines):
                                                                                                    key = line.strip()
                                                                                                            if len(key) < 120:
                                                                                                                        counts[key] = counts.get(key, 0) + 1

                                                                                                                            repeated = {s for s, c in counts.items() if c > 1}

                                                                                                                                cleaned_lines = [l for l in lines if l.strip() not in repeated]

                                                                                                                                    # Rejoin and split by two or more newlines or by evident title lines
                                                                                                                                        cleaned_text = "\n".join(cleaned_lines)

                                                                                                                                            # Candidate title regex (very short lines, many capitals or leading dates)
                                                                                                                                                # We'll scan and split where appropriate.
                                                                                                                                                    parts = []
                                                                                                                                                        buffer = []
                                                                                                                                                            current_title = None

                                                                                                                                                                def flush():
                                                                                                                                                                        nonlocal buffer, current_title
                                                                                                                                                                                if buffer:
                                                                                                                                                                                            txt = "\n".join(buffer).strip()
                                                                                                                                                                                                        if txt:
                                                                                                                                                                                                                        parts.append({
                                                                                                                                                                                                                                                "title": current_title or "Untitled",
                                                                                                                                                                                                                                                                    "text": txt,
                                                                                                                                                                                                                        })
                                                                                                                                                                                                                                buffer = []
                                                                                                                                                                                                                                        current_title = None

                                                                                                                                                                                                                                            for line in cleaned_text.splitlines():
                                                                                                                                                                                                                                                    l = line.strip()
                                                                                                                                                                                                                                                            # Heuristic for title line: short (<60), mostly uppercase or titlecase and not a normal sentence
                                                                                                                                                                                                                                                                    is_title = False
                                                                                                                                                                                                                                                                            if 1 <= len(l) <= 60:
                                                                                                                                                                                                                                                                                        # many uppercase letters or starts with a date-ish token
                                                                                                                                                                                                                                                                                                    uppercase_ratio = sum(1 for ch in l if ch.isupper()) / max(1, len(l))
                                                                                                                                                                                                                                                                                                                if uppercase_ratio > 0.35 or re.match(r"^\d{1,2}\s|^[A-Za-z]{3,9}\s\d{1,2}", l):
                                                                                                                                                                                                                                                                                                                                is_title = True
                                                                                                                                                                                                                                                                                                                                            # Also treat lines that look like "FRONTLINE" or "EDITORIAL" as titles
                                                                                                                                                                                                                                                                                                                                                        if l.isupper() or (len(l.split()) <= 4 and l.istitle()):
                                                                                                                                                                                                                                                                                                                                                                        is_title = True

                                                                                                                                                                                                                                                                                                                                                                                if is_title and buffer:
                                                                                                                                                                                                                                                                                                                                                                                            # start of a new article
                                                                                                                                                                                                                                                                                                                                                                                                        flush()
                                                                                                                                                                                                                                                                                                                                                                                                                    current_title = l
                                                                                                                                                                                                                                                                                                                                                                                                                                continue

                                                                                                                                                                                                                                                                                                                                                                                                                                        if is_title and not buffer:
                                                                                                                                                                                                                                                                                                                                                                                                                                                    # title followed by content
                                                                                                                                                                                                                                                                                                                                                                                                                                                                current_title = l
                                                                                                                                                                                                                                                                                                                                                                                                                                                                            continue

                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    buffer.append(l)

                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        # final flush
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            flush()

                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                # If nothing found, return the whole as one section
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    if not parts:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            txt = cleaned_text.strip()
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    return [{"title": "Untitled", "text": txt}]

                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        # Trim leading/trailing junk for each part
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            for p in parts:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    p["text"] = re.sub(r"\n{2,}", "\n\n", p["text"]).strip()

                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        return parts


                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        def chunk_text_for_telegram(text: str, max_chars: int = 3800) -> List[str]:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            """Split a long article text into parts appropriate for Telegram messages.

                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                Keeps paragraphs intact where possible.
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    """
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        if len(text) <= max_chars:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                return [text]

                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    paragraphs = [p.strip() for p in re.split(r"\n{2,}", text) if p.strip()]
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        chunks = []
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            cur = ""
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                for para in paragraphs:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        if not cur:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    cur = para
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                continue
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        if len(cur) + 2 + len(para) <= max_chars:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    cur = cur + "\n\n" + para
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            else:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        chunks.append(cur)
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    cur = para
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        if cur:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                chunks.append(cur)

                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    # As a fallback, if any chunk is still too big, break by characters
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        final = []
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            for c in chunks:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    if len(c) <= max_chars:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                final.append(c)
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        else:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    # break on sentence boundaries naively
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                sentences = re.split(r"(?<=[.!?])\s+", c)
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            cur2 = ""
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        for s in sentences:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        if not cur2:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            cur2 = s
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                continue
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                if len(cur2) + 1 + len(s) <= max_chars:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    cur2 = cur2 + " " + s
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    else:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        final.append(cur2)
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            cur2 = s
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        if cur2:
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        final.append(cur2)
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            return final
                                                                                                                                                                                                                        })