lua - now we have a plugin manager which works relativley cool!

This commit is contained in:
2025-10-01 12:25:27 +02:00
parent f50d8f9d69
commit f1bda77ce2
9 changed files with 1742 additions and 10 deletions

View File

@@ -7,6 +7,7 @@ from pathlib import Path
from jinja2 import Environment, FileSystemLoader from jinja2 import Environment, FileSystemLoader
import base64 import base64
import random import random
import time
import marko import marko
from marko.ext.gfm import GFM from marko.ext.gfm import GFM
@@ -15,6 +16,10 @@ from watchdog.observers import Observer
from log.Logger import * from log.Logger import *
from hashes.hashes import hash_list from hashes.hashes import hash_list
from htmlhandler import htmlhandler as Handler from htmlhandler import htmlhandler as Handler
from lua import plugin_manager
plugin_manager = plugin_manager.PluginManager()
plugin_manager.load_all() # load plugins
# Use absolute paths # Use absolute paths
ROOT = Path(os.path.abspath(".")) ROOT = Path(os.path.abspath("."))
@@ -109,6 +114,15 @@ def render_markdown(md_path: Path):
title = line[2:].strip() title = line[2:].strip()
break break
# Call pre_template hook properly
Logger.log_debug(f"Calling pre_template hook for {md_path}")
modified = plugin_manager.run_hook("pre_template", str(md_path), html_body)
if modified is not None:
html_body = modified
Logger.log_debug("pre_template hook modified the content")
else:
Logger.log_debug("pre_template hook returned None")
# Create clean HTML structure # Create clean HTML structure
# Pick two different hashes from hash_list # Pick two different hashes from hash_list
env = Environment(loader=FileSystemLoader("html/base")) env = Environment(loader=FileSystemLoader("html/base"))
@@ -125,6 +139,10 @@ def render_markdown(md_path: Path):
timestamp=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), timestamp=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
) )
post_mod = plugin_manager.run_hook("post_render", str(md_path), clean_jinja_html)
if post_mod is not None:
clean_jinja_html = post_mod
# Ensure html directory exists # Ensure html directory exists
HTML_DIR.mkdir(exist_ok=True) HTML_DIR.mkdir(exist_ok=True)
@@ -145,13 +163,11 @@ def remove_html(md_path: Path):
out_path.unlink() out_path.unlink()
Logger.log_debug(f"Removed: {out_path}") Logger.log_debug(f"Removed: {out_path}")
def initial_scan(markdown_dir: Path): def initial_scan(markdown_dir: Path):
Logger.log_info(f"Starting initial scan of markdown files in {markdown_dir}...") Logger.log_info(f"Starting initial scan of markdown files in {markdown_dir}...")
for md in markdown_dir.rglob("*.md"): for md in markdown_dir.rglob("*.md"):
render_markdown(md) render_markdown(md)
def build_rust_parser() -> bool: def build_rust_parser() -> bool:
fastmd_dir = ROOT / "fastmd" fastmd_dir = ROOT / "fastmd"

0
lua/__init__.py Normal file
View File

443
lua/plugin_manager.py Normal file
View File

@@ -0,0 +1,443 @@
import os
import re
import time
from pathlib import Path
from lupa import LuaRuntime, LuaError
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
from log.Logger import Logger
PLUGINS_DIR = Path(__file__).parent / "plugins"
HTML_DIR = Path(__file__).parent / "../html"
MARKDOWN_DIR = Path(__file__).parent / ".." / "markdown"
class PluginFSHandler(FileSystemEventHandler):
def __init__(self, manager):
self.manager = manager
def on_modified(self, event):
if event.is_directory:
return
if event.src_path.endswith(".lua"):
Logger.log_info(f"Plugin changed: {event.src_path}, reloading")
self.manager.reload_plugin(Path(event.src_path))
def on_created(self, event):
if event.is_directory:
return
if event.src_path.endswith(".lua"):
Logger.log_info(f"New plugin: {event.src_path}, loading")
self.manager.load_plugin(Path(event.src_path))
def on_deleted(self, event):
if event.is_directory:
return
p = Path(event.src_path)
if p.suffix == ".lua":
Logger.log_info(f"Plugin removed: {p.name}")
self.manager.unload_plugin(p.name)
class PluginManager:
def __init__(self):
self.lua = LuaRuntime(unpack_returned_tuples=True)
self.plugins = {} # name -> dict{path, lua_module, hooks, routes}
self.routes = {} # path -> (plugin_name, lua_fn)
self.hooks = {} # hook_name -> list of (plugin_name, lua_fn)
# Create directories
PLUGINS_DIR.mkdir(exist_ok=True)
HTML_DIR.mkdir(exist_ok=True)
MARKDOWN_DIR.mkdir(exist_ok=True)
# Expose helpers to Lua
self._setup_lua_globals()
self._start_watcher()
def _setup_lua_globals(self):
"""Set up all Lua global functions and guardrails"""
g = self.lua.globals()
# Route and hook registration
g.add_route = self._expose_add_route
g.register_hook = self._expose_register_hook
# Logging - using custom Logger
g.log = lambda msg: Logger.log_info(f"[lua] {msg}")
g.log_warn = lambda msg: Logger.log_warning(f"[lua] {msg}")
g.log_error = lambda msg: Logger.log_error(f"[lua] {msg}")
# Safe file operations (sandboxed)
g.read_file = self._safe_read_file
g.write_file = self._safe_write_file
# HTML manipulation
g.read_html = lambda filename: self._read_content(HTML_DIR, filename)
g.write_html = lambda filename, content: self._write_content(HTML_DIR, filename, content)
g.list_html_files = lambda: self._list_files(HTML_DIR, ".html")
# Markdown manipulation
g.read_markdown = lambda filename: self._read_content(MARKDOWN_DIR, filename)
g.write_markdown = lambda filename, content: self._write_content(MARKDOWN_DIR, filename, content)
g.list_markdown_files = lambda: self._list_files(MARKDOWN_DIR, ".md")
# HTML modifier helpers
g.html_find_tag = self._html_find_tag
g.html_replace_tag = self._html_replace_tag
g.html_insert_before = self._html_insert_before
g.html_insert_after = self._html_insert_after
g.html_wrap_content = self._html_wrap_content
# Markdown modifier helpers
g.md_add_header = self._md_add_header
g.md_replace_section = self._md_replace_section
g.md_append_content = self._md_append_content
# Utility functions
g.table_to_json = self._table_to_json
g.json_to_table = self._json_to_table
# Guardrails - predefined safe patterns
self._setup_lua_guardrails()
def _setup_lua_guardrails(self):
"""Set up Lua guardrails and safe patterns"""
guardrails_code = """
-- Guardrails and safe patterns for plugin development
-- Safe string operations
function safe_concat(...)
local result = {}
for i, v in ipairs({...}) do
if v ~= nil then
table.insert(result, tostring(v))
end
end
return table.concat(result)
end
-- Safe table operations
function table_contains(tbl, value)
for _, v in ipairs(tbl) do
if v == value then return true end
end
return false
end
function table_keys(tbl)
local keys = {}
for k, _ in pairs(tbl) do
table.insert(keys, k)
end
return keys
end
function table_values(tbl)
local values = {}
for _, v in pairs(tbl) do
table.insert(values, v)
end
return values
end
-- Safe string escaping
function escape_html(str)
if str == nil then return "" end
local s = tostring(str)
s = string.gsub(s, "&", "&")
s = string.gsub(s, "<", "&lt;")
s = string.gsub(s, ">", "&gt;")
s = string.gsub(s, '"', "&quot;")
s = string.gsub(s, "'", "&#39;")
return s
end
-- Pattern validation
function is_valid_filename(name)
if name == nil or name == "" then return false end
-- Block directory traversal
if string.match(name, "%.%.") then return false end
if string.match(name, "/") or string.match(name, "\\\\") then return false end
return true
end
-- Safe error handling wrapper
function try_catch(fn, catch_fn)
local status, err = pcall(fn)
if not status and catch_fn then
catch_fn(err)
end
return status
end
-- Request validation
function validate_request(req, required_fields)
if type(req) ~= "table" then return false, "Request must be a table" end
for _, field in ipairs(required_fields) do
if req[field] == nil then
return false, "Missing required field: " .. field
end
end
return true, nil
end
-- Rate limiting helper (simple in-memory)
_rate_limits = _rate_limits or {}
function check_rate_limit(key, max_calls, window_seconds)
local now = os.time()
if _rate_limits[key] == nil then
_rate_limits[key] = {count = 1, window_start = now}
return true
end
local rl = _rate_limits[key]
if now - rl.window_start > window_seconds then
-- Reset window
rl.count = 1
rl.window_start = now
return true
end
if rl.count >= max_calls then
return false
end
rl.count = rl.count + 1
return true
end
log("Lua guardrails initialized")
"""
try:
self.lua.execute(guardrails_code)
except LuaError as e:
Logger.log_error(f"Failed to initialize Lua guardrails: {e}")
# Safe file operations
def _safe_read_file(self, path):
"""Safe file reading with path validation"""
try:
p = Path(path)
# Prevent directory traversal
if ".." in str(p.parts):
raise ValueError("Path traversal not allowed")
return p.read_text(encoding="utf-8")
except Exception as e:
Logger.log_error(f"Error reading file {path}: {e}")
return None
def _safe_write_file(self, path, content):
"""Safe file writing with path validation"""
try:
p = Path(path)
# Prevent directory traversal
if ".." in str(p.parts):
raise ValueError("Path traversal not allowed")
p.write_text(content, encoding="utf-8")
return True
except Exception as e:
Logger.log_error(f"Error writing file {path}: {e}")
return False
# HTML/Markdown content operations
def _read_content(self, base_dir, filename):
"""Read content from HTML or Markdown directory"""
try:
path = base_dir / filename
if not path.is_relative_to(base_dir):
raise ValueError("Invalid path")
if path.exists():
return path.read_text(encoding="utf-8")
return None
except Exception as e:
Logger.log_error(f"Error reading {filename}: {e}")
return None
def _write_content(self, base_dir, filename, content):
"""Write content to HTML or Markdown directory"""
try:
path = base_dir / filename
if not path.is_relative_to(base_dir):
raise ValueError("Invalid path")
path.write_text(content, encoding="utf-8")
return True
except Exception as e:
Logger.log_error(f"Error writing {filename}: {e}")
return False
def _list_files(self, base_dir, extension):
"""List files with given extension"""
try:
return [f.name for f in base_dir.glob(f"*{extension}")]
except Exception as e:
Logger.log_error(f"Error listing files: {e}")
return []
# HTML manipulation helpers
def _html_find_tag(self, html, tag):
"""Find first occurrence of HTML tag"""
pattern = f"<{tag}[^>]*>.*?</{tag}>"
match = re.search(pattern, html, re.DOTALL | re.IGNORECASE)
return match.group(0) if match else None
def _html_replace_tag(self, html, tag, new_content):
"""Replace HTML tag content"""
pattern = f"(<{tag}[^>]*>).*?(</{tag}>)"
return re.sub(pattern, f"\\1{new_content}\\2", html, flags=re.DOTALL | re.IGNORECASE)
def _html_insert_before(self, html, marker, content):
"""Insert content before a marker"""
return html.replace(marker, content + marker)
def _html_insert_after(self, html, marker, content):
"""Insert content after a marker"""
return html.replace(marker, marker + content)
def _html_wrap_content(self, html, tag, wrapper_tag, attrs=""):
"""Wrap tag content with another tag"""
pattern = f"(<{tag}[^>]*>)(.*?)(</{tag}>)"
def replacer(match):
open_tag, content, close_tag = match.groups()
return f"{open_tag}<{wrapper_tag} {attrs}>{content}</{wrapper_tag}>{close_tag}"
return re.sub(pattern, replacer, html, flags=re.DOTALL | re.IGNORECASE)
# Markdown manipulation helpers
def _md_add_header(self, markdown, level, text):
"""Add header to markdown"""
prefix = "#" * level
return f"{prefix} {text}\n\n{markdown}"
def _md_replace_section(self, markdown, header, new_content):
"""Replace markdown section"""
# Find section starting with header
pattern = f"(#{1,6}\\s+{re.escape(header)}.*?)(?=#{1,6}\\s+|$)"
return re.sub(pattern, f"## {header}\n\n{new_content}\n\n", markdown, flags=re.DOTALL)
def _md_append_content(self, markdown, content):
"""Append content to markdown"""
return markdown.rstrip() + "\n\n" + content
# JSON conversion helpers
def _table_to_json(self, lua_table):
"""Convert Lua table to JSON string"""
import json
try:
# Convert lupa table to Python dict
py_dict = dict(lua_table)
return json.dumps(py_dict)
except Exception as e:
Logger.log_error(f"Error converting table to JSON: {e}")
return "{}"
def _json_to_table(self, json_str):
"""Convert JSON string to Lua table"""
import json
try:
return json.loads(json_str)
except Exception as e:
Logger.log_error(f"Error parsing JSON: {e}")
return {}
""" Lifecycle of Plugin """
def load_all(self):
for p in PLUGINS_DIR.glob("*.lua"):
self.load_plugin(p)
def load_plugin(self, path: Path):
name = path.name
try:
code = path.read_text(encoding="utf-8")
self._current_plugin = name
lua_module = self.lua.execute(code)
if lua_module is None:
lua_module = {}
self.plugins[name] = {"path": path, "module": lua_module}
Logger.log_info(f"Loaded plugin: {name}")
except LuaError as e:
Logger.log_error(f"Lua error while loading {name}: {e}")
except Exception as e:
Logger.log_error(f"Error loading plugin {name}: {e}")
finally:
self._current_plugin = None
def reload_plugin(self, path: Path):
name = path.name
# remove previous hooks/routes
self.unload_plugin(name)
time.sleep(0.05)
self.load_plugin(path)
def unload_plugin(self, name: str):
# Remove routes/hook registrations from this plugin
to_remove_routes = [r for r, v in self.routes.items() if v[0] == name]
for r in to_remove_routes:
del self.routes[r]
for hook, lst in list(self.hooks.items()):
self.hooks[hook] = [x for x in lst if x[0] != name]
if not self.hooks[hook]:
del self.hooks[hook]
if name in self.plugins:
del self.plugins[name]
Logger.log_info(f"Unloaded plugin {name}")
""" Expose a new API route """
def _expose_add_route(self, path, lua_fn):
"""Called from Lua as add_route(path, function(req) ... end)"""
p = str(path)
plugin_name = self._current_loading_plugin_name()
if not plugin_name:
plugin_name = "<unknown>"
self.routes[p] = (plugin_name, lua_fn)
Logger.log_info(f"Plugin {plugin_name} registered route {p}")
def _expose_register_hook(self, hook_name, lua_fn):
hook = str(hook_name)
plugin_name = self._current_loading_plugin_name() or "<unknown>"
self.hooks.setdefault(hook, []).append((plugin_name, lua_fn))
Logger.log_info(f"Plugin {plugin_name} registered hook {hook}")
def _current_loading_plugin_name(self):
return getattr(self, "_current_plugin", None)
""" Running hooks & handling routes """
def handle_request(self, path, request_info=None):
"""If a plugin registered a route for this path, call it and return (status, headers, body) or raw str."""
if path in self.routes:
plugin_name, lua_fn = self.routes[path]
try:
lua_req = request_info or {}
res = lua_fn(lua_req)
if isinstance(res, tuple) and len(res) == 3:
return res
return (200, {"Content-Type": "text/html"}, str(res))
except LuaError as e:
Logger.log_error(f"Lua error in route {path}: {e}")
return (500, {"Content-Type": "text/plain"}, f"Plugin error: {e}")
return None
def run_hook(self, hook_name: str, *args):
"""Run all registered hook functions for hook_name; return last non-None return value."""
if hook_name not in self.hooks:
return None
last = None
for plugin_name, fn in list(self.hooks[hook_name]):
try:
out = fn(*args)
if out is not None:
last = out
except LuaError as e:
Logger.log_error(f"Lua error in hook {hook_name} from {plugin_name}: {e}")
return last
""" File watcher """
def _start_watcher(self):
self.observer = Observer()
self.fs_handler = PluginFSHandler(self)
self.observer.schedule(self.fs_handler, str(PLUGINS_DIR), recursive=False)
self.observer.start()
Logger.log_info("Started plugin folder watcher")
def stop(self):
self.observer.stop()
self.observer.join()

View File

@@ -0,0 +1 @@
write_markdown("output.md", "# Generated Document\n\nContent here...")

View File

@@ -0,0 +1,6 @@
-- hello.lua
add_route("/lua/hello", function(req)
log("hello.lua handling request for " .. (req.path or "unknown"))
-- return (status, headers_table, body_string)
return 200, {["Content-Type"] = "text/html"}, "<h1>Hello from Lua plugin!</h1>"
end)

View File

@@ -0,0 +1,9 @@
register_hook("pre_template", function(md_path, html_body)
-- Check if the current markdown path is for test.md
if md_path and md_path:match("test%.md$") then
local banner = "<div class='plugin-banner' style='padding:10px;background:white;border:1px solid #f99;margin-bottom:12px;color:black;'>Note: served via Lua plugin banner</div>"
return banner .. html_body
end
-- For other pages, return the original HTML body unchanged
return html_body
end)

View File

@@ -0,0 +1,41 @@
-- Simple working endpoint using existing utilities
add_route("/admin/html-files", function(req)
-- Get files and convert to proper Lua table
local files = list_html_files()
local html = [[
<!DOCTYPE html>
<html>
<head>
<title>HTML Files</title>
<style>
body { font-family: Arial, sans-serif; margin: 40px; }
.file-item { padding: 8px; border-bottom: 1px solid #eee; }
</style>
</head>
<body>
<h1>HTML Files</h1>
<div style="background: #f5f5f5; padding: 20px; border-radius: 5px;">
]]
-- Safe iteration using a counter
local count = 0
for i = 1, 100 do -- Safe upper limit
local success, file = pcall(function() return files[i] end)
if not success or file == nil then
break
end
html = html .. "<div class='file-item'>" .. file .. "</div>"
count = count + 1
end
html = html .. [[
</div>
<p>Total files: ]] .. count .. [[</p>
<a href="/">Back</a>
</body>
</html>
]]
return html
end)

1155
lua/readme.md Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -5,14 +5,19 @@ import subprocess
from http.server import BaseHTTPRequestHandler, HTTPServer from http.server import BaseHTTPRequestHandler, HTTPServer
import mimetypes import mimetypes
from jsmin import jsmin # pip install jsmin from jsmin import jsmin # pip install jsmin
from pathlib import Path
from log.Logger import * from log.Logger import *
from lua import plugin_manager
logger = Logger() logger = Logger()
plugin_manager = plugin_manager.PluginManager()
plugin_manager.load_all() # load all plugins
PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
HTML_DIR = os.path.join(PROJECT_ROOT, "html") HTML_DIR = os.path.join(PROJECT_ROOT, "html")
MARKDOWN_DIR = os.path.join(PROJECT_ROOT, "markdown") MARKDOWN_DIR = os.path.join(PROJECT_ROOT, "markdown")
BASE_FILE = os.path.join(HTML_DIR, "base", "index.html") BASE_FILE = os.path.join(HTML_DIR, "base", "index.html")
LUA_DIR = Path(PROJECT_ROOT) / "lua" / "plugins"
def get_html_files(directory=HTML_DIR): def get_html_files(directory=HTML_DIR):
html_files = [] html_files = []
@@ -87,7 +92,7 @@ def index_footer():
class MyHandler(BaseHTTPRequestHandler): class MyHandler(BaseHTTPRequestHandler):
def do_GET(self): def do_GET(self):
req_path = self.path.lstrip("/") req_path = self.path.lstrip("/") # normalize leading /
# Handle root/index # Handle root/index
if req_path == "" or req_path == "index.html": if req_path == "" or req_path == "index.html":
@@ -98,11 +103,22 @@ class MyHandler(BaseHTTPRequestHandler):
self.wfile.write(content.encode("utf-8")) self.wfile.write(content.encode("utf-8"))
return return
# CHECK PLUGIN ROUTES FIRST
plugin_result = plugin_manager.handle_request("/" + req_path, {"path": self.path})
if plugin_result is not None:
status, headers, body = plugin_result
self.send_response(status)
for key, value in headers.items():
self.send_header(key, value)
self.end_headers()
self.wfile.write(body.encode("utf-8") if isinstance(body, str) else body)
return
# Handle markdown file downloads # Handle markdown file downloads
if req_path.startswith("markdown/"): if req_path.startswith("markdown/"):
markdown_filename = req_path[9:] # Remove "markdown/" prefix markdown_filename = req_path[9:] # Remove "markdown/" prefix
# Security check: only allow .md files and prevent directory traversal # Security check
if not markdown_filename.endswith(".md") or ".." in markdown_filename or "/" in markdown_filename: if not markdown_filename.endswith(".md") or ".." in markdown_filename or "/" in markdown_filename:
self.send_response(403) self.send_response(403)
self.end_headers() self.end_headers()
@@ -111,14 +127,12 @@ class MyHandler(BaseHTTPRequestHandler):
markdown_file_path = os.path.join(MARKDOWN_DIR, markdown_filename) markdown_file_path = os.path.join(MARKDOWN_DIR, markdown_filename)
# Check if file exists and is within markdown directory
if not os.path.exists(markdown_file_path) or not os.path.isfile(markdown_file_path): if not os.path.exists(markdown_file_path) or not os.path.isfile(markdown_file_path):
self.send_response(404) self.send_response(404)
self.end_headers() self.end_headers()
self.wfile.write(b"404 - Markdown file not found") self.wfile.write(b"404 - Markdown file not found")
return return
# Verify the resolved path is still within the markdown directory (extra security)
resolved_path = os.path.realpath(markdown_file_path) resolved_path = os.path.realpath(markdown_file_path)
resolved_markdown_dir = os.path.realpath(MARKDOWN_DIR) resolved_markdown_dir = os.path.realpath(MARKDOWN_DIR)
if not resolved_path.startswith(resolved_markdown_dir): if not resolved_path.startswith(resolved_markdown_dir):
@@ -146,6 +160,52 @@ class MyHandler(BaseHTTPRequestHandler):
self.wfile.write(b"500 - Internal Server Error") self.wfile.write(b"500 - Internal Server Error")
return return
# Handle Lua file downloads
if req_path.startswith("lua/"):
lua_filename = req_path[4:] # Remove "lua/" prefix
# Security check
if not lua_filename.endswith(".lua") or ".." in lua_filename or "/" in lua_filename:
self.send_response(403)
self.end_headers()
self.wfile.write(b"403 - Forbidden: Only .lua files allowed")
return
lua_file_path = os.path.join(LUA_DIR, lua_filename)
if not os.path.exists(lua_file_path) or not os.path.isfile(lua_file_path):
self.send_response(404)
self.end_headers()
self.wfile.write(b"404 - Lua file not found")
return
resolved_path = os.path.realpath(lua_file_path)
resolved_lua_dir = os.path.realpath(LUA_DIR)
if not resolved_path.startswith(resolved_lua_dir):
self.send_response(403)
self.end_headers()
self.wfile.write(b"403 - Forbidden")
return
try:
with open(lua_file_path, "rb") as f:
content = f.read()
self.send_response(200)
self.send_header("Content-type", "text/x-lua")
self.send_header("Content-Disposition", f'attachment; filename="{lua_filename}"')
self.end_headers()
self.wfile.write(content)
logger.log_info(f"Served Lua file: {lua_filename}")
return
except Exception as err:
logger.log_error(f"Error serving Lua file {lua_filename}: {err}")
self.send_response(500)
self.end_headers()
self.wfile.write(b"500 - Internal Server Error")
return
# Handle other files (existing functionality) # Handle other files (existing functionality)
file_path = os.path.normpath(os.path.join(PROJECT_ROOT, req_path)) file_path = os.path.normpath(os.path.join(PROJECT_ROOT, req_path))
if not file_path.startswith(PROJECT_ROOT): if not file_path.startswith(PROJECT_ROOT):
@@ -179,6 +239,7 @@ class MyHandler(BaseHTTPRequestHandler):
self.end_headers() self.end_headers()
self.wfile.write(b"404 - Not Found") self.wfile.write(b"404 - Not Found")
def run_pypost(): def run_pypost():
"""Run PyPost.py in a separate process.""" """Run PyPost.py in a separate process."""
script = os.path.join(PROJECT_ROOT, "PyPost.py") script = os.path.join(PROJECT_ROOT, "PyPost.py")