#!/usr/bin/env python3 import os import sys import subprocess import platform from pathlib import Path from jinja2 import Environment, FileSystemLoader import base64 import random import yaml import marko from marko.ext.gfm import GFM from watchdog.observers import Observer from log.Logger import * from hashes.hashes import hash_list from htmlhandler import htmlhandler as Handler from lua import plugin_manager # Import your LaTeX extension from hashes.util.LaTeXRenderer import LaTeXExtension plugin_manager = plugin_manager.PluginManager() plugin_manager.load_all() # load plugins # Use absolute paths ROOT = Path(os.path.abspath(".")) MARKDOWN_DIR = ROOT / "markdown" HTML_DIR = ROOT / "html" # Determine executable extension based on OS exe_ext = ".exe" if platform.system() == "Windows" else "" RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "release" / f"fastmd{exe_ext}" # Fallback to debug build if release not found if not RUST_PARSER_PATH.exists(): RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "debug" / f"fastmd{exe_ext}" # Python Markdown parser with table support AND LaTeX extension markdown_parser = marko.Markdown(extensions=[GFM, LaTeXExtension()]) # Threshold for switching to Rust parser (number of lines) RUST_PARSER_THRESHOLD = 1000 Logger = Logger() # Global obfuscate flag, default True obfuscate = True def split_yaml_front_matter(md_path: Path) -> tuple[str, str]: """Return (yaml_text, markdown_text). YAML is '' if none exists.""" try: text = md_path.read_text(encoding="utf-8") except Exception as e: Logger.log_error(f"Could not read {md_path}: {e}") return '', '' if text.startswith("---"): parts = text.split("---", 2) if len(parts) >= 3: yaml_text = parts[1].strip() markdown_text = parts[2].lstrip("\n") return yaml_text, markdown_text return '', text def count_lines_in_file(file_path: Path) -> int: try: with open(file_path, 'r', encoding='utf-8') as f: return sum(1 for _ in f) except Exception as e: Logger.log_error(f"Could not count lines in {file_path}: {e}") return 0 def should_use_rust_parser(md_path: Path) -> bool: if not RUST_PARSER_PATH.exists(): return False line_count = count_lines_in_file(md_path) use_rust = line_count > RUST_PARSER_THRESHOLD if use_rust: Logger.log_rust_usage(f"Using Rust parser for {md_path} ({line_count} lines)") else: Logger.log_debug(f"Using Python parser for {md_path} ({line_count} lines)") return use_rust def parse_markdown_with_rust(md_path: Path) -> str: try: result = subprocess.run( [str(RUST_PARSER_PATH), str(md_path)], capture_output=True, text=True, encoding='utf-8', check=True ) return result.stdout except subprocess.CalledProcessError as e: Logger.log_error(f"Rust parser failed for {md_path}: {e}") Logger.log_error(f"stderr: {e.stderr}") raise except Exception as e: Logger.log_error(f"Error running Rust parser for {md_path}: {e}") raise def render_markdown(md_path: Path): """Render a single markdown file to HTML, stripping YAML front matter.""" yaml_text, markdown_text = split_yaml_front_matter(md_path) # Choose parser if should_use_rust_parser(md_path): try: html_body = parse_markdown_with_rust(md_path) except Exception as e: Logger.log_warning(f"Rust parser failed for {md_path}, falling back to Python parser: {e}") html_body = markdown_parser.convert(markdown_text) else: html_body = markdown_parser.convert(markdown_text) # Extract title from filename or first H1 title = md_path.stem for line in markdown_text.splitlines(): if line.startswith("# "): title = line[2:].strip() break # Plugin pre_template hook Logger.log_debug(f"Calling pre_template hook for {md_path}") modified = plugin_manager.run_hook("pre_template", str(md_path), html_body) if modified is not None: html_body = modified Logger.log_debug("pre_template hook modified the content") else: Logger.log_debug("pre_template hook returned None") # Jinja template env = Environment(loader=FileSystemLoader("html/base")) template = env.get_template("template.html") hash1, hash2 = random.sample(hash_list, 2) clean_jinja_html = template.render( title=title, html_body=html_body, now=time.asctime(time.localtime()), hash1=base64.b64encode(hash1.encode("utf-8")).decode("utf-8"), hash2=base64.b64encode(hash2.encode("windows-1252")).decode("utf-8"), timestamp=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), ) # Plugin post_render hook post_mod = plugin_manager.run_hook("post_render", str(md_path), clean_jinja_html) if post_mod is not None: clean_jinja_html = post_mod # Write output HTML_DIR.mkdir(exist_ok=True) relative_path = md_path.relative_to(MARKDOWN_DIR) out_path = HTML_DIR / relative_path.with_suffix(".html") out_path.parent.mkdir(parents=True, exist_ok=True) out_path.write_text(clean_jinja_html, encoding="utf-8") Logger.log_debug(f"Rendered: {md_path} -> {out_path}") def extract_summary(md_path: Path) -> tuple[str, str] | None: yaml_text, _ = split_yaml_front_matter(md_path) if not yaml_text: Logger.log_debug(f"No YAML front matter in {md_path}") return None try: data = yaml.safe_load(yaml_text) summary = data.get("summary") if summary: html_name = md_path.with_suffix(".html").name return html_name, summary else: Logger.log_debug(f"No 'summary' key in YAML of {md_path}") except Exception as e: Logger.log_warning(f"Failed to parse YAML summary in {md_path}: {e}") return None def remove_html(md_path: Path): relative_path = md_path.relative_to(MARKDOWN_DIR) out_path = HTML_DIR / relative_path.with_suffix(".html") if out_path.exists(): out_path.unlink() Logger.log_debug(f"Removed: {out_path}") def initial_scan(markdown_dir: Path): Logger.log_info(f"Starting initial scan of markdown files in {markdown_dir}...") for md in markdown_dir.rglob("*.md"): render_markdown(md) def build_rust_parser() -> bool: fastmd_dir = ROOT / "fastmd" if not fastmd_dir.exists(): Logger.log_error(f"fastmd directory not found at {fastmd_dir}") return False cargo_toml = fastmd_dir / "Cargo.toml" if not cargo_toml.exists(): Logger.log_error(f"Cargo.toml not found at {cargo_toml}") return False Logger.log_info("Attempting to build Rust parser with 'cargo build --release'...") try: result = subprocess.run( ["cargo", "build", "--release"], cwd=str(fastmd_dir), capture_output=True, text=True, check=True ) Logger.log_info("Rust parser built successfully!") Logger.log_debug(f"Build output: {result.stdout}") return True except subprocess.CalledProcessError as e: Logger.log_error(f"Failed to build Rust parser: {e}") Logger.log_error(f"Build stderr: {e.stderr}") return False except FileNotFoundError: Logger.log_error("cargo command not found. Please install Rust and Cargo.") return False except Exception as e: Logger.log_error(f"Unexpected error building Rust parser: {e}") return False if __name__ == "__main__": # Handle alternative markdown folder if not MARKDOWN_DIR.exists(): alt_root = ROOT / "PyPost" if alt_root.exists() and alt_root.is_dir(): Logger.log_warning(f"Default 'markdown' directory not found, switching ROOT to: {alt_root}") ROOT = alt_root MARKDOWN_DIR = ROOT / "markdown" HTML_DIR = ROOT / "html" RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "release" / f"fastmd{exe_ext}" if not RUST_PARSER_PATH.exists(): RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "debug" / f"fastmd{exe_ext}" else: Logger.log_error(f"Markdown directory not found: {MARKDOWN_DIR}") Logger.log_warning("Please create a 'markdown' directory or use a 'PyPost' directory with one inside it.") sys.exit(1) # Build Rust parser if missing if not RUST_PARSER_PATH.exists(): Logger.log_warning(f"Rust parser not found at {RUST_PARSER_PATH}") if build_rust_parser(): RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "release" / f"fastmd{exe_ext}" if not RUST_PARSER_PATH.exists(): RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "debug" / f"fastmd{exe_ext}" if RUST_PARSER_PATH.exists(): Logger.log_info(f"Rust parser built and found at: {RUST_PARSER_PATH}") else: Logger.log_error("Build succeeded but parser binary not found") Logger.log_warning("Will use Python parser for all files") else: Logger.log_error("Failed to build Rust parser") Logger.log_warning("Will use Python parser for all files") else: Logger.log_info(f"Rust parser found at: {RUST_PARSER_PATH}") if RUST_PARSER_PATH.exists(): Logger.log_info(f"Will use Rust parser for files with more than {RUST_PARSER_THRESHOLD} lines") else: Logger.log_warning("Using Python parser for all files") initial_scan(MARKDOWN_DIR) event_handler = Handler() observer = Observer() observer.schedule(event_handler, str(MARKDOWN_DIR), recursive=True) observer.start() Logger.log_info(f"Started monitoring {MARKDOWN_DIR} for changes.") try: while True: time.sleep(1) except KeyboardInterrupt: observer.stop() observer.join()