PyPost/PyPost.py

#!/usr/bin/env python3
import os
import sys
import subprocess
import platform
from pathlib import Path
from jinja2 import Environment, FileSystemLoader
import base64
import random
import yaml

import marko
from marko.ext.gfm import GFM
from watchdog.observers import Observer

from log.Logger import *
from hashes.hashes import hash_list
from htmlhandler import htmlhandler as Handler
from lua import plugin_manager

# Import your LaTeX extension
from hashes.util.LaTeXRenderer import LaTeXExtension

plugin_manager = plugin_manager.PluginManager()
plugin_manager.load_all()  # load plugins

# Use absolute paths
ROOT = Path(os.path.abspath("."))
MARKDOWN_DIR = ROOT / "markdown"
HTML_DIR = ROOT / "html"

# Determine executable extension based on OS
exe_ext = ".exe" if platform.system() == "Windows" else ""
RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "release" / f"fastmd{exe_ext}"

# Fallback to debug build if release not found
if not RUST_PARSER_PATH.exists():
    RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "debug" / f"fastmd{exe_ext}"

# Python Markdown parser with table support AND LaTeX extension
markdown_parser = marko.Markdown(extensions=[GFM, LaTeXExtension()])

# Threshold for switching to Rust parser (number of lines)
RUST_PARSER_THRESHOLD = 1000

Logger = Logger()

# Global obfuscate flag, default True
obfuscate = True

def split_yaml_front_matter(md_path: Path) -> tuple[str, str]:
    """Return (yaml_text, markdown_text). YAML is '' if none exists."""
    try:
        text = md_path.read_text(encoding="utf-8")
    except Exception as e:
        Logger.log_error(f"Could not read {md_path}: {e}")
        return '', ''
    if text.startswith("---"):
        parts = text.split("---", 2)
        if len(parts) >= 3:
            yaml_text = parts[1].strip()
            markdown_text = parts[2].lstrip("\n")
            return yaml_text, markdown_text
    return '', text

def count_lines_in_file(file_path: Path) -> int:
    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            return sum(1 for _ in f)
    except Exception as e:
        Logger.log_error(f"Could not count lines in {file_path}: {e}")
        return 0

def should_use_rust_parser(md_path: Path) -> bool:
    if not RUST_PARSER_PATH.exists():
        return False
    line_count = count_lines_in_file(md_path)
    use_rust = line_count > RUST_PARSER_THRESHOLD
    if use_rust:
        Logger.log_rust_usage(f"Using Rust parser for {md_path} ({line_count} lines)")
    else:
        Logger.log_debug(f"Using Python parser for {md_path} ({line_count} lines)")
    return use_rust

def parse_markdown_with_rust(md_path: Path) -> str:
    try:
        result = subprocess.run(
            [str(RUST_PARSER_PATH), str(md_path)],
            capture_output=True,
            text=True,
            encoding='utf-8',
            check=True
        )
        return result.stdout
    except subprocess.CalledProcessError as e:
        Logger.log_error(f"Rust parser failed for {md_path}: {e}")
        Logger.log_error(f"stderr: {e.stderr}")
        raise
    except Exception as e:
        Logger.log_error(f"Error running Rust parser for {md_path}: {e}")
        raise

def render_markdown(md_path: Path):
    """Render a single markdown file to HTML, stripping YAML front matter."""
    yaml_text, markdown_text = split_yaml_front_matter(md_path)

    # Choose parser
    if should_use_rust_parser(md_path):
        try:
            html_body = parse_markdown_with_rust(md_path)
        except Exception as e:
            Logger.log_warning(f"Rust parser failed for {md_path}, falling back to Python parser: {e}")
            html_body = markdown_parser.convert(markdown_text)
    else:
        html_body = markdown_parser.convert(markdown_text)

    # Extract title from filename or first H1
    title = md_path.stem
    for line in markdown_text.splitlines():
        if line.startswith("# "):
            title = line[2:].strip()
            break

    # Plugin pre_template hook
    Logger.log_debug(f"Calling pre_template hook for {md_path}")
    modified = plugin_manager.run_hook("pre_template", str(md_path), html_body)
    if modified is not None:
        html_body = modified
        Logger.log_debug("pre_template hook modified the content")
    else:
        Logger.log_debug("pre_template hook returned None")

    # Jinja template
    env = Environment(loader=FileSystemLoader("html/base"))
    template = env.get_template("template.html")
    hash1, hash2 = random.sample(hash_list, 2)

    clean_jinja_html = template.render(
        title=title,
        html_body=html_body,
        now=time.asctime(time.localtime()),
        hash1=base64.b64encode(hash1.encode("utf-8")).decode("utf-8"),
        hash2=base64.b64encode(hash2.encode("windows-1252")).decode("utf-8"),
        timestamp=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
    )

    # Plugin post_render hook
    post_mod = plugin_manager.run_hook("post_render", str(md_path), clean_jinja_html)
    if post_mod is not None:
        clean_jinja_html = post_mod

    # Write output
    HTML_DIR.mkdir(exist_ok=True)
    relative_path = md_path.relative_to(MARKDOWN_DIR)
    out_path = HTML_DIR / relative_path.with_suffix(".html")
    out_path.parent.mkdir(parents=True, exist_ok=True)
    out_path.write_text(clean_jinja_html, encoding="utf-8")
    Logger.log_debug(f"Rendered: {md_path} -> {out_path}")

def extract_summary(md_path: Path) -> tuple[str, str] | None:
    yaml_text, _ = split_yaml_front_matter(md_path)
    if not yaml_text:
        Logger.log_debug(f"No YAML front matter in {md_path}")
        return None
    try:
        data = yaml.safe_load(yaml_text)
        summary = data.get("summary")
        if summary:
            html_name = md_path.with_suffix(".html").name
            return html_name, summary
        else:
            Logger.log_debug(f"No 'summary' key in YAML of {md_path}")
    except Exception as e:
        Logger.log_warning(f"Failed to parse YAML summary in {md_path}: {e}")
    return None

def remove_html(md_path: Path):
    relative_path = md_path.relative_to(MARKDOWN_DIR)
    out_path = HTML_DIR / relative_path.with_suffix(".html")
    if out_path.exists():
        out_path.unlink()
        Logger.log_debug(f"Removed: {out_path}")

def initial_scan(markdown_dir: Path):
    Logger.log_info(f"Starting initial scan of markdown files in {markdown_dir}...")
    for md in markdown_dir.rglob("*.md"):
        render_markdown(md)

def build_rust_parser() -> bool:
    fastmd_dir = ROOT / "fastmd"
    if not fastmd_dir.exists():
        Logger.log_error(f"fastmd directory not found at {fastmd_dir}")
        return False
    cargo_toml = fastmd_dir / "Cargo.toml"
    if not cargo_toml.exists():
        Logger.log_error(f"Cargo.toml not found at {cargo_toml}")
        return False
    Logger.log_info("Attempting to build Rust parser with 'cargo build --release'...")
    try:
        result = subprocess.run(
            ["cargo", "build", "--release"],
            cwd=str(fastmd_dir),
            capture_output=True,
            text=True,
            check=True
        )
        Logger.log_info("Rust parser built successfully!")
        Logger.log_debug(f"Build output: {result.stdout}")
        return True
    except subprocess.CalledProcessError as e:
        Logger.log_error(f"Failed to build Rust parser: {e}")
        Logger.log_error(f"Build stderr: {e.stderr}")
        return False
    except FileNotFoundError:
        Logger.log_error("cargo command not found. Please install Rust and Cargo.")
        return False
    except Exception as e:
        Logger.log_error(f"Unexpected error building Rust parser: {e}")
        return False

if __name__ == "__main__":
    # Handle alternative markdown folder
    if not MARKDOWN_DIR.exists():
        alt_root = ROOT / "PyPost"
        if alt_root.exists() and alt_root.is_dir():
            Logger.log_warning(f"Default 'markdown' directory not found, switching ROOT to: {alt_root}")
            ROOT = alt_root
            MARKDOWN_DIR = ROOT / "markdown"
            HTML_DIR = ROOT / "html"
            RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "release" / f"fastmd{exe_ext}"
            if not RUST_PARSER_PATH.exists():
                RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "debug" / f"fastmd{exe_ext}"
        else:
            Logger.log_error(f"Markdown directory not found: {MARKDOWN_DIR}")
            Logger.log_warning("Please create a 'markdown' directory or use a 'PyPost' directory with one inside it.")
            sys.exit(1)

    # Build Rust parser if missing
    if not RUST_PARSER_PATH.exists():
        Logger.log_warning(f"Rust parser not found at {RUST_PARSER_PATH}")
        if build_rust_parser():
            RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "release" / f"fastmd{exe_ext}"
            if not RUST_PARSER_PATH.exists():
                RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "debug" / f"fastmd{exe_ext}"
            if RUST_PARSER_PATH.exists():
                Logger.log_info(f"Rust parser built and found at: {RUST_PARSER_PATH}")
            else:
                Logger.log_error("Build succeeded but parser binary not found")
                Logger.log_warning("Will use Python parser for all files")
        else:
            Logger.log_error("Failed to build Rust parser")
            Logger.log_warning("Will use Python parser for all files")
    else:
        Logger.log_info(f"Rust parser found at: {RUST_PARSER_PATH}")

    if RUST_PARSER_PATH.exists():
        Logger.log_info(f"Will use Rust parser for files with more than {RUST_PARSER_THRESHOLD} lines")
    else:
        Logger.log_warning("Using Python parser for all files")

    initial_scan(MARKDOWN_DIR)
    event_handler = Handler()
    observer = Observer()
    observer.schedule(event_handler, str(MARKDOWN_DIR), recursive=True)
    observer.start()
    Logger.log_info(f"Started monitoring {MARKDOWN_DIR} for changes.")
    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        observer.stop()
    observer.join()