311 lines
11 KiB
Python
311 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
import os
|
|
import sys
|
|
import time
|
|
import subprocess
|
|
import platform
|
|
from pathlib import Path
|
|
|
|
import marko
|
|
from marko.ext.gfm import GFM
|
|
from watchdog.observers import Observer
|
|
from watchdog.events import FileSystemEventHandler
|
|
|
|
from log.Logger import *
|
|
from hashes.obfuscation.Obfuscator import Obfuscator
|
|
from htmlhandler import htmlhandler as Handler
|
|
|
|
# Use absolute paths
|
|
ROOT = Path(os.path.abspath("."))
|
|
MARKDOWN_DIR = ROOT / "markdown"
|
|
HTML_DIR = ROOT / "html"
|
|
|
|
# Determine executable extension based on OS
|
|
exe_ext = ".exe" if platform.system() == "Windows" else ""
|
|
RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "release" / f"fastmd{exe_ext}"
|
|
|
|
# Fallback to debug build if release not found
|
|
if not RUST_PARSER_PATH.exists():
|
|
RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "debug" / f"fastmd{exe_ext}"
|
|
|
|
# Create Python markdown parser with table support (fallback for small files)
|
|
markdown_parser = marko.Markdown(extensions=[GFM])
|
|
|
|
# Threshold for switching to Rust parser (number of lines)
|
|
RUST_PARSER_THRESHOLD = 500
|
|
|
|
Logger = Logger()
|
|
|
|
# Global obfuscate flag, default True
|
|
obfuscate = True
|
|
|
|
def count_lines_in_file(file_path: Path) -> int:
|
|
"""Count the number of lines in a file."""
|
|
try:
|
|
with open(file_path, 'r', encoding='utf-8') as f:
|
|
return sum(1 for _ in f)
|
|
except Exception as e:
|
|
Logger.log_error(f"Could not count lines in {file_path}: {e}")
|
|
return 0
|
|
|
|
def should_use_rust_parser(md_path: Path) -> bool:
|
|
"""Determine if we should use the Rust parser based on file size."""
|
|
if not RUST_PARSER_PATH.exists():
|
|
return False
|
|
|
|
line_count = count_lines_in_file(md_path)
|
|
use_rust = line_count > RUST_PARSER_THRESHOLD
|
|
|
|
if use_rust:
|
|
Logger.log_rust_usage(f"Using Rust parser for {md_path} ({line_count} lines)")
|
|
else:
|
|
Logger.log_debug(f"Using Python parser for {md_path} ({line_count} lines)")
|
|
|
|
return use_rust
|
|
|
|
def parse_markdown_with_rust(md_path: Path) -> str:
|
|
"""Parse markdown using the Rust parser."""
|
|
try:
|
|
# Run the Rust parser
|
|
result = subprocess.run(
|
|
[str(RUST_PARSER_PATH), str(md_path)],
|
|
capture_output=True,
|
|
text=True,
|
|
encoding='utf-8',
|
|
check=True
|
|
)
|
|
return result.stdout
|
|
except subprocess.CalledProcessError as e:
|
|
Logger.log_error(f"Rust parser failed for {md_path}: {e}")
|
|
Logger.log_error(f"stderr: {e.stderr}")
|
|
raise
|
|
except Exception as e:
|
|
Logger.log_error(f"Error running Rust parser for {md_path}: {e}")
|
|
raise
|
|
|
|
def render_markdown(md_path: Path):
|
|
"""Render a single markdown file to an obfuscated HTML file."""
|
|
try:
|
|
text = md_path.read_text(encoding="utf-8")
|
|
except Exception as e:
|
|
Logger.log_error(f"Could not read {md_path}: {e}")
|
|
return
|
|
|
|
# Decide which parser to use based on file size
|
|
if should_use_rust_parser(md_path):
|
|
try:
|
|
html_body = parse_markdown_with_rust(md_path)
|
|
except Exception as e:
|
|
Logger.log_warning(f"Rust parser failed for {md_path}, falling back to Python parser: {e}")
|
|
html_body = markdown_parser.convert(text)
|
|
else:
|
|
html_body = markdown_parser.convert(text)
|
|
|
|
# Extract title from filename or first H1
|
|
title = md_path.stem
|
|
for line in text.splitlines():
|
|
if line.startswith("# "):
|
|
title = line[2:].strip()
|
|
break
|
|
|
|
import base64
|
|
import random
|
|
from hashes.hashes import hash_list
|
|
# Create clean HTML structure
|
|
# Pick two different hashes from hash_list
|
|
hash1, hash2 = random.sample(hash_list, 2)
|
|
|
|
clean_html = f"""<!doctype html>
|
|
<html lang="en" style="height:100%; margin:0;">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<title>{title}</title>
|
|
<link rel="stylesheet" href="../css/main.css">
|
|
<link rel="icon" type="image/x-icon" href="../css/favicon/favicon.ico">
|
|
<script src="../js/post/normal.js"></script>
|
|
</head>
|
|
<body style="display:flex; flex-direction:column; min-height:100%; margin:0;">
|
|
<main class="container" style="flex:1;">
|
|
<h1 onclick="window.location.href=window.location.origin" style="cursor:pointer; display:flex; align-items:center; gap:8px; font-size:1.5em; margin:0;">
|
|
<img src="../css/icons/back.webp" width="32" height="32" alt="Back" style="display:block;" />
|
|
{title}
|
|
</h1>
|
|
<img src="../css/icons/written.webp" width="32" height="32" alt="write_img" loading="lazy" style="vertical-align: middle;padding-left:12px; padding-left:40px;" />
|
|
<div class="meta" style="display: inline;">Written @{time.asctime(time.localtime())}</div>
|
|
<hr style="margin:10px 0;" />
|
|
{html_body}
|
|
</main>
|
|
|
|
<footer style="margin-top:auto; width:100%;">
|
|
<hr style="margin:10px 0;" />
|
|
<img src="../css/icons/date.webp" width="16" height="16" alt="date" loading="lazy" style="vertical-align: middle;" />
|
|
{time.strftime("%Y-%m-%d %H:%M:%S")}<br/>
|
|
<img src="../css/icons/magnifier.webp" width="16" height="16" alt="Hash1" loading="lazy" style="display:inline; vertical-align:middle;" />
|
|
Hash 1 (<b>UTF-8</b>)<i>:{base64.b64encode(hash1.encode("utf-8")).decode("utf-8")}</i><br />
|
|
<img src="../css/icons/magnifier.webp" width="16" height="16" alt="Hash2" loading="lazy" style="display:inline; vertical-align:middle;" />
|
|
Hash 2 (<b>Windows-1252</b>)<i>:{base64.b64encode(hash2.encode("windows-1252")).decode("windows-1252")}</i><br />
|
|
</footer>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
# Obfuscate the HTML for browser output
|
|
obfuscated_html = Obfuscator.obfuscate_html(clean_html)
|
|
|
|
# Ensure html directory exists
|
|
HTML_DIR.mkdir(exist_ok=True)
|
|
|
|
# Maintain relative directory structure in html/
|
|
relative_path = md_path.relative_to(MARKDOWN_DIR)
|
|
out_path = HTML_DIR / relative_path.with_suffix(".html")
|
|
|
|
# Create parent directories if needed
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
if obfuscate:
|
|
out_path.write_text(obfuscated_html, encoding="utf-8")
|
|
else:
|
|
out_path.write_text(clean_html, encoding="utf-8")
|
|
|
|
Logger.log_debug(f"Rendered: {md_path} -> {out_path}")
|
|
|
|
def remove_html(md_path: Path):
|
|
relative_path = md_path.relative_to(MARKDOWN_DIR)
|
|
out_path = HTML_DIR / relative_path.with_suffix(".html")
|
|
if out_path.exists():
|
|
out_path.unlink()
|
|
Logger.log_debug(f"Removed: {out_path}")
|
|
|
|
|
|
def initial_scan(markdown_dir: Path):
|
|
Logger.log_info(f"Starting initial scan of markdown files in {markdown_dir}...")
|
|
for md in markdown_dir.rglob("*.md"):
|
|
render_markdown(md)
|
|
|
|
|
|
def build_rust_parser() -> bool:
|
|
"""Attempt to build the Rust parser using cargo."""
|
|
fastmd_dir = ROOT / "fastmd"
|
|
|
|
if not fastmd_dir.exists():
|
|
Logger.log_error(f"fastmd directory not found at {fastmd_dir}")
|
|
return False
|
|
|
|
cargo_toml = fastmd_dir / "Cargo.toml"
|
|
if not cargo_toml.exists():
|
|
Logger.log_error(f"Cargo.toml not found at {cargo_toml}")
|
|
return False
|
|
|
|
Logger.log_info("Attempting to build Rust parser with 'cargo build --release'...")
|
|
|
|
try:
|
|
# Run cargo build --release in the fastmd directory
|
|
result = subprocess.run(
|
|
["cargo", "build", "--release"],
|
|
cwd=str(fastmd_dir),
|
|
capture_output=True,
|
|
text=True,
|
|
check=True
|
|
)
|
|
|
|
Logger.log_info("Rust parser built successfully!")
|
|
Logger.log_debug(f"Build output: {result.stdout}")
|
|
return True
|
|
|
|
except subprocess.CalledProcessError as e:
|
|
Logger.log_error(f"Failed to build Rust parser: {e}")
|
|
Logger.log_error(f"Build stderr: {e.stderr}")
|
|
return False
|
|
except FileNotFoundError:
|
|
Logger.log_error("cargo command not found. Please install Rust and Cargo.")
|
|
return False
|
|
except Exception as e:
|
|
Logger.log_error(f"Unexpected error building Rust parser: {e}")
|
|
return False
|
|
|
|
if __name__ == "__main__":
|
|
# Check for markdown directory
|
|
if not MARKDOWN_DIR.exists():
|
|
alt_root = ROOT / "PyPost"
|
|
if alt_root.exists() and alt_root.is_dir():
|
|
Logger.log_warning(f"Default 'markdown' directory not found, switching ROOT to: {alt_root}")
|
|
ROOT = alt_root
|
|
MARKDOWN_DIR = ROOT / "markdown"
|
|
HTML_DIR = ROOT / "html"
|
|
# Update Rust parser path for new root
|
|
RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "release" / f"fastmd{exe_ext}"
|
|
if not RUST_PARSER_PATH.exists():
|
|
RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "debug" / f"fastmd{exe_ext}"
|
|
else:
|
|
Logger.log_error(f"Markdown directory not found: {MARKDOWN_DIR}")
|
|
Logger.log_warning("Please create a 'markdown' directory or use a 'PyPost' directory with one inside it.")
|
|
sys.exit(1)
|
|
|
|
# Check if Rust parser exists, if not try to build it
|
|
if not RUST_PARSER_PATH.exists():
|
|
Logger.log_warning(f"Rust parser not found at {RUST_PARSER_PATH}")
|
|
|
|
# Try to build the Rust parser
|
|
if build_rust_parser():
|
|
# Update path after successful build
|
|
RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "release" / f"fastmd{exe_ext}"
|
|
if not RUST_PARSER_PATH.exists():
|
|
RUST_PARSER_PATH = ROOT / "fastmd" / "target" / "debug" / f"fastmd{exe_ext}"
|
|
|
|
if RUST_PARSER_PATH.exists():
|
|
Logger.log_info(f"Rust parser built and found at: {RUST_PARSER_PATH}")
|
|
else:
|
|
Logger.log_error("Build succeeded but parser binary not found")
|
|
Logger.log_warning("Will use Python parser for all files")
|
|
else:
|
|
Logger.log_error("Failed to build Rust parser")
|
|
Logger.log_warning("Will use Python parser for all files")
|
|
else:
|
|
Logger.log_info(f"Rust parser found at: {RUST_PARSER_PATH}")
|
|
|
|
# Log parser strategy
|
|
if RUST_PARSER_PATH.exists():
|
|
Logger.log_info(f"Will use Rust parser for files with more than {RUST_PARSER_THRESHOLD} lines")
|
|
else:
|
|
Logger.log_warning("Using Python parser for all files")
|
|
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(description="Monitor markdown directory and convert to HTML with dynamic parser selection.")
|
|
|
|
# This stores True when passed, but means "no obfuscation"
|
|
parser.add_argument(
|
|
"--no-obfuscate",
|
|
action="store_false",
|
|
help="Disable HTML obfuscation."
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--rust-threshold",
|
|
type=int,
|
|
default=500,
|
|
help=f"Line count threshold for using Rust parser (default: {RUST_PARSER_THRESHOLD})"
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Invert it to get the obfuscate flag
|
|
obfuscate = not args.no_obfuscate
|
|
|
|
# Update threshold if specified
|
|
RUST_PARSER_THRESHOLD = args.rust_threshold
|
|
|
|
Logger.log_obfuscation_info(f"Obfuscation is {obfuscate}",obfuscate)
|
|
|
|
initial_scan(MARKDOWN_DIR)
|
|
event_handler = Handler()
|
|
observer = Observer()
|
|
observer.schedule(event_handler, str(MARKDOWN_DIR), recursive=True)
|
|
observer.start()
|
|
Logger.log_info(f"Started monitoring {MARKDOWN_DIR} for changes.")
|
|
try:
|
|
while True:
|
|
time.sleep(1)
|
|
except KeyboardInterrupt:
|
|
observer.stop()
|
|
observer.join() |