From 24ef59f0ed5ec46d431f424bb5f7152efa1d74ec Mon Sep 17 00:00:00 2001 From: rattatwinko Date: Sun, 29 Jun 2025 17:44:44 +0200 Subject: [PATCH] cleaned up and added a logging system --- markdown_backend/src/main.rs | 45 +- markdown_backend/src/markdown.rs | 541 +++++++++++----------- src/app/admin/manage/rust-status/page.tsx | 164 ++++++- src/app/api/admin/posts/route.ts | 52 +++ 4 files changed, 521 insertions(+), 281 deletions(-) diff --git a/markdown_backend/src/main.rs b/markdown_backend/src/main.rs index b6cfac1..8ceb813 100644 --- a/markdown_backend/src/main.rs +++ b/markdown_backend/src/main.rs @@ -1,7 +1,7 @@ #[warn(unused_imports)] use clap::{Parser, Subcommand}; mod markdown; -use markdown::{get_all_posts, get_post_by_slug, get_posts_by_tag, watch_posts}; +use markdown::{get_all_posts, get_post_by_slug, get_posts_by_tag, watch_posts, get_parser_logs, clear_parser_logs}; use serde_json; use std::fs; use std::io; @@ -34,6 +34,10 @@ enum Commands { Rsparseinfo, /// Check backend health Checkhealth, + /// Get parser logs + Logs, + /// Clear parser logs + ClearLogs, /// Parse markdown from file or stdin Parse { #[arg(long)] @@ -92,35 +96,36 @@ fn main() { let health = markdown::checkhealth(); println!("{}", serde_json::to_string_pretty(&health).unwrap()); } + Commands::Logs => { + let logs = get_parser_logs(); + println!("{}", serde_json::to_string_pretty(&logs).unwrap()); + } + Commands::ClearLogs => { + clear_parser_logs(); + println!("{}", serde_json::to_string(&serde_json::json!({"success": true, "message": "Logs cleared"})).unwrap()); + } Commands::Parse { file, stdin, ast } => { - let input = if let Some(file_path) = file { - match std::fs::read_to_string(file_path) { - Ok(content) => content, - Err(e) => { - eprintln!("Failed to read file: {}", e); - std::process::exit(1); - } - } - } else if *stdin { + let content = if *stdin { let mut buffer = String::new(); - if let Err(e) = io::stdin().read_to_string(&mut buffer) { - eprintln!("Failed to read from stdin: {}", e); - std::process::exit(1); - } + io::stdin().read_to_string(&mut buffer).unwrap(); buffer + } else if let Some(file_path) = file { + fs::read_to_string(file_path).unwrap() } else { - eprintln!("Please provide --file or --stdin"); + eprintln!("Either --file or --stdin must be specified"); std::process::exit(1); }; + if *ast { - // Print pulldown_cmark events as debug output - let parser = pulldown_cmark::Parser::new_ext(&input, pulldown_cmark::Options::all()); - for event in parser { + // Parse and output AST as debug format + let parser = pulldown_cmark::Parser::new_ext(&content, pulldown_cmark::Options::all()); + let events: Vec<_> = parser.collect(); + for event in events { println!("{:?}", event); } } else { - // Print HTML output - let parser = pulldown_cmark::Parser::new_ext(&input, pulldown_cmark::Options::all()); + // Parse and output HTML + let parser = pulldown_cmark::Parser::new_ext(&content, pulldown_cmark::Options::all()); let mut html_output = String::new(); pulldown_cmark::html::push_html(&mut html_output, parser); println!("{}", html_output); diff --git a/markdown_backend/src/markdown.rs b/markdown_backend/src/markdown.rs index 3cc0d2d..96c278a 100644 --- a/markdown_backend/src/markdown.rs +++ b/markdown_backend/src/markdown.rs @@ -1,41 +1,40 @@ +// // src/markdown.rs -/* +// Written by: @rattatwinko +// -This is the Rust Markdown Parser. -It supports caching of posts and is - -BLAZINGLY FAST! - -*/ - -#[warn(unused_imports)] use std::fs; use std::path::{Path, PathBuf}; +use std::collections::HashMap; +use std::sync::RwLock; +use std::time::Instant; +use std::sync::mpsc::channel; +use std::collections::VecDeque; + use chrono::{DateTime, Utc}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use pulldown_cmark::{Parser, Options, html, Event, Tag, CowStr}; use gray_matter::engine::YAML; use gray_matter::Matter; -use ammonia::clean; use slug::slugify; use notify::{RecursiveMode, RecommendedWatcher, Watcher, Config}; -use std::sync::mpsc::channel; -use std::time::{Duration, Instant}; -use syntect::highlighting::{ThemeSet, Style}; +use syntect::highlighting::ThemeSet; use syntect::parsing::SyntaxSet; -use syntect::html::{highlighted_html_for_string, IncludeBackground}; +use syntect::html::highlighted_html_for_string; use once_cell::sync::Lazy; -use std::collections::HashMap; -use std::sync::RwLock; use serde_json; -use sysinfo::{System, Pid, RefreshKind, CpuRefreshKind, ProcessRefreshKind}; -use serde::Serialize; +use sysinfo::{System, RefreshKind, CpuRefreshKind, ProcessRefreshKind}; use regex::Regex; +// Constants const POSTS_CACHE_PATH: &str = "./cache/posts_cache.json"; const POST_STATS_PATH: &str = "./cache/post_stats.json"; +const MAX_FILE_SIZE: usize = 10 * 1024 * 1024; // 10MB +const PARSING_TIMEOUT_SECS: u64 = 30; +const MAX_LOG_ENTRIES: usize = 1000; -#[derive(Debug, Deserialize, Clone, serde::Serialize)] +// Data structures +#[derive(Debug, Deserialize, Clone, Serialize)] pub struct PostFrontmatter { pub title: String, pub date: String, @@ -43,7 +42,7 @@ pub struct PostFrontmatter { pub summary: Option, } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Post { pub slug: String, pub title: String, @@ -55,21 +54,17 @@ pub struct Post { pub author: String, } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct PostStats { pub slug: String, pub cache_hits: u64, pub cache_misses: u64, pub last_interpret_time_ms: u128, pub last_compile_time_ms: u128, - pub last_cpu_usage_percent: f32, // Not f64 + pub last_cpu_usage_percent: f32, pub last_cache_status: String, // "hit" or "miss" } -static POST_CACHE: Lazy>> = Lazy::new(|| RwLock::new(HashMap::new())); -static ALL_POSTS_CACHE: Lazy>>> = Lazy::new(|| RwLock::new(None)); -static POST_STATS: Lazy>> = Lazy::new(|| RwLock::new(HashMap::new())); - #[derive(Debug, Serialize)] pub struct HealthReport { pub posts_dir_exists: bool, @@ -83,197 +78,32 @@ pub struct HealthReport { pub errors: Vec, } -fn get_posts_directory() -> PathBuf { - // Check if we're running in Docker by looking for common Docker environment indicators - let is_docker = std::env::var("DOCKER_CONTAINER").is_ok() - || std::env::var("KUBERNETES_SERVICE_HOST").is_ok() - || std::path::Path::new("/.dockerenv").exists(); - - let candidates = if is_docker { - vec![ - "/app/docker", // Docker volume mount point (highest priority in Docker) - "/app/posts", // Fallback in Docker - "./posts", - "../posts", - "/posts", - "/docker" - ] - } else { - vec![ - "./posts", - "../posts", - "/posts", - "/docker", - "/app/docker" // Lower priority for non-Docker environments - ] - }; - - for candidate in candidates.iter() { - let path = PathBuf::from(candidate); - if path.exists() && path.is_dir() { - return path; - } - } - // Fallback: default to ./posts - PathBuf::from("./posts") +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogEntry { + pub timestamp: String, + pub level: String, // "info", "warning", "error" + pub message: String, + pub slug: Option, + pub details: Option, } -// Helper function to recursively find all markdown files -fn find_markdown_files(dir: &Path) -> std::io::Result> { - let mut files = Vec::new(); - if dir.is_dir() { - for entry in fs::read_dir(dir)? { - let entry = entry?; - let path = entry.path(); - - if path.is_dir() { - // Recursively scan subdirectories - files.extend(find_markdown_files(&path)?); - } else if path.extension().map(|e| e == "md").unwrap_or(false) { - files.push(path); - } - } - } - Ok(files) -} - -// Helper function to convert a file path to a slug -fn path_to_slug(file_path: &Path, posts_dir: &Path) -> String { - // Get the relative path from posts directory - let relative_path = file_path.strip_prefix(posts_dir).unwrap_or(file_path); - // Remove the .md extension - let without_ext = relative_path.with_extension(""); - // Convert to string and replace path separators with a special separator - // Use "::" as a directory separator to avoid conflicts with hyphens in filenames - without_ext.to_string_lossy() - .replace(std::path::MAIN_SEPARATOR, "::") - .replace("/", "::") - .replace("\\", "::") -} - -// Helper function to convert a slug back to a file path -fn slug_to_path(slug: &str, posts_dir: &Path) -> PathBuf { - // Split by the special directory separator "::" - let parts: Vec<&str> = slug.split("::").collect(); - if parts.len() == 1 { - // Single part, no subdirectory - posts_dir.join(format!("{}.md", parts[0])) - } else { - // Multiple parts, all but the last are directories, last is filename - let mut path = posts_dir.to_path_buf(); - for (i, part) in parts.iter().enumerate() { - if i == parts.len() - 1 { - // Last part is the filename - path = path.join(format!("{}.md", part)); - } else { - // Other parts are directories - path = path.join(part); - } - } - path - } -} - -fn get_file_creation_date(path: &Path) -> std::io::Result> { - let metadata = fs::metadata(path)?; - // Try to get creation time, fall back to modification time if not available - match metadata.created() { - Ok(created) => Ok(DateTime::::from(created)), - Err(_) => { - // Fall back to modification time if creation time is not available - let modified = metadata.modified()?; - Ok(DateTime::::from(modified)) - } - } -} - -fn process_anchor_links(content: &str) -> String { - // Replace [text](#anchor) with slugified anchor - let re = regex::Regex::new(r"\[([^\]]+)\]\(#([^)]+)\)").unwrap(); - re.replace_all(content, |caps: ®ex::Captures| { - let link_text = &caps[1]; - let anchor = &caps[2]; - let slugified = slugify(anchor); - format!("[{}](#{})", link_text, slugified) - }).to_string() -} - -// Helper function to strip emojis from a string -// Neccesary for the slugify function to work correctly. And the ID's to work with the frontend. -fn strip_emojis(s: &str) -> String { - // Remove all characters in the Emoji Unicode ranges - // This is a simple approach and may not cover all emojis, but works for most cases - s.chars() - .filter(|c| { - let c = *c as u32; - // Basic Emoji ranges - !( (c >= 0x1F600 && c <= 0x1F64F) // Emoticons - || (c >= 0x1F300 && c <= 0x1F5FF) // Misc Symbols and Pictographs - || (c >= 0x1F680 && c <= 0x1F6FF) // Transport and Map - || (c >= 0x2600 && c <= 0x26FF) // Misc symbols - || (c >= 0x2700 && c <= 0x27BF) // Dingbats - || (c >= 0x1F900 && c <= 0x1F9FF) // Supplemental Symbols and Pictographs - || (c >= 0x1FA70 && c <= 0x1FAFF) // Symbols and Pictographs Extended-A - || (c >= 0x1F1E6 && c <= 0x1F1FF) // Regional Indicator Symbols - ) - }) - .collect() -} - -// Function to process custom tags in markdown content -fn process_custom_tags(content: &str) -> String { - let mut processed = content.to_string(); - - // Handle simple tags without parameters FIRST - let simple_tags = [ - ("", "
This is my custom tag content!
"), - ("", "
⚠️ Warning: This is a custom warning tag!
"), - ("", "
ℹ️ Info: This is a custom info tag!
"), - ("", "
✅ Success: This is a custom success tag!
"), - ("", "
❌ Error: This is a custom error tag!
"), - ]; - - for (tag, replacement) in simple_tags.iter() { - processed = processed.replace(tag, replacement); - } - - // Handle tags with parameters like - let tag_with_params = Regex::new(r"<(\w+)\s+([^>]*?[a-zA-Z0-9=])[^>]*/>").unwrap(); - processed = tag_with_params.replace_all(&processed, |caps: ®ex::Captures| { - let tag_name = &caps[1]; - let params = &caps[2]; - - match tag_name { - "mytag" => { - // Parse parameters and generate custom HTML - format!("
Custom content with params: {}
", params, params) - }, - "alert" => { - // Parse alert type from params - if params.contains("type=\"warning\"") { - "
⚠️ Warning Alert!
".to_string() - } else if params.contains("type=\"error\"") { - "
❌ Error Alert!
".to_string() - } else { - "
ℹ️ Info Alert!
".to_string() - } - }, - _ => format!("
Unknown custom tag: {}
", tag_name, tag_name) - } - }).to_string(); - - processed -} +// Static caches +static POST_CACHE: Lazy>> = Lazy::new(|| RwLock::new(HashMap::new())); +static ALL_POSTS_CACHE: Lazy>>> = Lazy::new(|| RwLock::new(None)); +static POST_STATS: Lazy>> = Lazy::new(|| RwLock::new(HashMap::new())); +static PARSER_LOGS: Lazy>> = Lazy::new(|| RwLock::new(VecDeque::new())); +// Ammonia HTML sanitizer configuration static AMMONIA: Lazy> = Lazy::new(|| { let mut builder = ammonia::Builder::default(); - // All possible HTML Tags so that you can stylize via HTML - builder.add_tag_attributes("h1", &["id", "style"]); - builder.add_tag_attributes("h2", &["id", "style"]); - builder.add_tag_attributes("h3", &["id", "style"]); - builder.add_tag_attributes("h4", &["id", "style"]); - builder.add_tag_attributes("h5", &["id", "style"]); - builder.add_tag_attributes("h6", &["id", "style"]); + + // Add allowed attributes for various HTML tags + builder.add_tag_attributes("h1", &["style", "id"]); + builder.add_tag_attributes("h2", &["style", "id"]); + builder.add_tag_attributes("h3", &["style", "id"]); + builder.add_tag_attributes("h4", &["style", "id"]); + builder.add_tag_attributes("h5", &["style", "id"]); + builder.add_tag_attributes("h6", &["style", "id"]); builder.add_tag_attributes("p", &["style"]); builder.add_tag_attributes("span", &["style"]); builder.add_tag_attributes("strong", &["style"]); @@ -290,7 +120,6 @@ static AMMONIA: Lazy> = Lazy::new(|| { builder.add_tag_attributes("pre", &["style"]); builder.add_tag_attributes("kbd", &["style"]); builder.add_tag_attributes("samp", &["style"]); - builder.add_tag_attributes("div", &["style", "class"]); builder.add_tag_attributes("section", &["style"]); builder.add_tag_attributes("article", &["style"]); builder.add_tag_attributes("header", &["style"]); @@ -335,15 +164,197 @@ static AMMONIA: Lazy> = Lazy::new(|| { builder.add_tag_attributes("fieldset", &["style"]); builder.add_tag_attributes("legend", &["style"]); builder.add_tag_attributes("blockquote", &["style"]); - builder.add_tag_attributes("font", &["style"]); // deprecated - builder.add_tag_attributes("center", &["style"]); // deprecated - builder.add_tag_attributes("big", &["style"]); // deprecated - builder.add_tag_attributes("tt", &["style"]); // deprecated + builder.add_tag_attributes("font", &["style"]); + builder.add_tag_attributes("center", &["style"]); + builder.add_tag_attributes("big", &["style"]); + builder.add_tag_attributes("tt", &["style"]); + + // Add class attribute for div + builder.add_tag_attributes("div", &["style", "class"]); + builder }); +// Helper functions +fn get_posts_directory() -> PathBuf { + let is_docker = std::env::var("DOCKER_CONTAINER").is_ok() + || std::env::var("KUBERNETES_SERVICE_HOST").is_ok() + || std::path::Path::new("/.dockerenv").exists(); + + let candidates = if is_docker { + vec![ + "/app/docker", // Docker volume mount point (highest priority in Docker) + "/app/posts", // Fallback in Docker + "./posts", + "../posts", + "/posts", + "/docker" + ] + } else { + vec![ + "./posts", + "../posts", + "/posts", + "/docker", + "/app/docker" // Lower priority for non-Docker environments + ] + }; + + for candidate in candidates.iter() { + let path = PathBuf::from(candidate); + if path.exists() && path.is_dir() { + return path; + } + } + // Fallback: default to ./posts + PathBuf::from("./posts") +} + +fn find_markdown_files(dir: &Path) -> std::io::Result> { + let mut files = Vec::new(); + if dir.is_dir() { + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + + if path.is_dir() { + files.extend(find_markdown_files(&path)?); + } else if path.extension().map(|e| e == "md").unwrap_or(false) { + files.push(path); + } + } + } + Ok(files) +} + +fn path_to_slug(file_path: &Path, posts_dir: &Path) -> String { + let relative_path = file_path.strip_prefix(posts_dir).unwrap_or(file_path); + let without_ext = relative_path.with_extension(""); + without_ext.to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "::") + .replace("/", "::") + .replace("\\", "::") +} + +fn slug_to_path(slug: &str, posts_dir: &Path) -> PathBuf { + let parts: Vec<&str> = slug.split("::").collect(); + if parts.len() == 1 { + posts_dir.join(format!("{}.md", parts[0])) + } else { + let mut path = posts_dir.to_path_buf(); + for (i, part) in parts.iter().enumerate() { + if i == parts.len() - 1 { + path = path.join(format!("{}.md", part)); + } else { + path = path.join(part); + } + } + path + } +} + +fn get_file_creation_date(path: &Path) -> std::io::Result> { + let metadata = fs::metadata(path)?; + match metadata.created() { + Ok(created) => Ok(DateTime::::from(created)), + Err(_) => { + let modified = metadata.modified()?; + Ok(DateTime::::from(modified)) + } + } +} + +fn process_anchor_links(content: &str) -> String { + let re = regex::Regex::new(r"\[([^\]]+)\]\(#([^)]+)\)").unwrap(); + re.replace_all(content, |caps: ®ex::Captures| { + let link_text = &caps[1]; + let anchor = &caps[2]; + let slugified = slugify(anchor); + format!("[{}](#{})", link_text, slugified) + }).to_string() +} + +fn strip_emojis(s: &str) -> String { + s.chars() + .filter(|c| { + let c = *c as u32; + !( (c >= 0x1F600 && c <= 0x1F64F) // Emoticons + || (c >= 0x1F300 && c <= 0x1F5FF) // Misc Symbols and Pictographs + || (c >= 0x1F680 && c <= 0x1F6FF) // Transport and Map + || (c >= 0x2600 && c <= 0x26FF) // Misc symbols + || (c >= 0x2700 && c <= 0x27BF) // Dingbats + || (c >= 0x1F900 && c <= 0x1F9FF) // Supplemental Symbols and Pictographs + || (c >= 0x1FA70 && c <= 0x1FAFF) // Symbols and Pictographs Extended-A + || (c >= 0x1F1E6 && c <= 0x1F1FF) // Regional Indicator Symbols + ) + }) + .collect() +} + +fn process_custom_tags(content: &str) -> String { + let mut processed = content.to_string(); + + // Handle simple tags without parameters + let simple_tags = [ + ("", "
This is my custom tag content!
"), + ("", "
⚠️ Warning: This is a custom warning tag!
"), + ("", "
ℹ️ Info: This is a custom info tag!
"), + ("", "
✅ Success: This is a custom success tag!
"), + ("", "
❌ Error: This is a custom error tag!
"), + ]; + + for (tag, replacement) in simple_tags.iter() { + processed = processed.replace(tag, replacement); + } + + // Handle tags with parameters + let tag_with_params = Regex::new(r"<(\w+)\s+([^>]*?[a-zA-Z0-9=])[^>]*/>").unwrap(); + processed = tag_with_params.replace_all(&processed, |caps: ®ex::Captures| { + let tag_name = &caps[1]; + let params = &caps[2]; + + match tag_name { + "mytag" => { + format!("
Custom content with params: {}
", params, params) + }, + "alert" => { + if params.contains("type=\"warning\"") { + "
⚠️ Warning Alert!
".to_string() + } else if params.contains("type=\"error\"") { + "
❌ Error Alert!
".to_string() + } else { + "
ℹ️ Info Alert!
".to_string() + } + }, + _ => format!("
Unknown custom tag: {}
", tag_name, tag_name) + } + }).to_string(); + + processed +} + +// Logging functions +fn add_log(level: &str, message: &str, slug: Option<&str>, details: Option<&str>) { + let timestamp = chrono::Utc::now().to_rfc3339(); + let log_entry = LogEntry { + timestamp, + level: level.to_string(), + message: message.to_string(), + slug: slug.map(|s| s.to_string()), + details: details.map(|s| s.to_string()), + }; + + let mut logs = PARSER_LOGS.write().unwrap(); + logs.push_back(log_entry); + + // Keep only the last MAX_LOG_ENTRIES + if logs.len() > MAX_LOG_ENTRIES { + logs.pop_front(); + } +} + +// Main public functions pub fn rsparseinfo() -> String { - // Eagerly load all posts to populate stats let _ = get_all_posts(); let stats = POST_STATS.read().unwrap(); let values: Vec<&PostStats> = stats.values().collect(); @@ -355,16 +366,20 @@ pub fn rsparseinfo() -> String { } pub fn get_post_by_slug(slug: &str) -> Result> { + add_log("info", "Starting post parsing", Some(slug), None); + let mut sys = System::new_with_specifics(RefreshKind::new().with_processes(ProcessRefreshKind::everything()).with_cpu(CpuRefreshKind::everything())); sys.refresh_processes(); let pid = sysinfo::get_current_pid()?; let before_cpu = sys.process(pid).map(|p| p.cpu_usage()).unwrap_or(0.0); let start = Instant::now(); + let mut stats = POST_STATS.write().unwrap(); let entry = stats.entry(slug.to_string()).or_insert_with(|| PostStats { slug: slug.to_string(), ..Default::default() }); + // Try cache first if let Some(post) = POST_CACHE.read().unwrap().get(slug).cloned() { entry.cache_hits += 1; @@ -373,32 +388,30 @@ pub fn get_post_by_slug(slug: &str) -> Result> entry.last_cache_status = "hit".to_string(); sys.refresh_process(pid); entry.last_cpu_usage_percent = sys.process(pid).map(|p| p.cpu_usage()).unwrap_or(0.0) - before_cpu; + add_log("info", "Cache hit", Some(slug), None); return Ok(post); } + entry.cache_misses += 1; entry.last_cache_status = "miss".to_string(); - drop(stats); // Release lock before heavy work + drop(stats); + let posts_dir = get_posts_directory(); let file_path = slug_to_path(slug, &posts_dir); - // Add debugging for file path resolution - eprintln!("[Rust Parser] Looking for file: {:?}", file_path); - eprintln!("[Rust Parser] Posts directory: {:?}", posts_dir); - eprintln!("[Rust Parser] Slug: {}", slug); - if !file_path.exists() { - eprintln!("[Rust Parser] File does not exist: {:?}", file_path); - return Err(format!("File not found: {:?}", file_path).into()); + let error_msg = format!("File not found: {:?}", file_path); + add_log("error", &error_msg, Some(slug), None); + return Err(error_msg.into()); } let file_content = fs::read_to_string(&file_path)?; - eprintln!("[Rust Parser] File size: {} bytes", file_content.len()); + add_log("info", &format!("File loaded: {} bytes", file_content.len()), Some(slug), None); - // Check file size limit (10MB) - const MAX_FILE_SIZE: usize = 10 * 1024 * 1024; // 10MB if file_content.len() > MAX_FILE_SIZE { - eprintln!("[Rust Parser] File too large: {} bytes (max: {} bytes)", file_content.len(), MAX_FILE_SIZE); - return Err(format!("File too large: {} bytes (max: {} bytes)", file_content.len(), MAX_FILE_SIZE).into()); + let error_msg = format!("File too large: {} bytes (max: {} bytes)", file_content.len(), MAX_FILE_SIZE); + add_log("error", &error_msg, Some(slug), None); + return Err(error_msg.into()); } let matter = Matter::::new(); @@ -408,20 +421,21 @@ pub fn get_post_by_slug(slug: &str) -> Result> match data.deserialize() { Ok(front) => front, Err(e) => { - eprintln!("[Rust Parser] Failed to deserialize frontmatter for post {}: {}", slug, e); - return Err(format!("Failed to deserialize frontmatter: {}", e).into()); + let error_msg = format!("Failed to deserialize frontmatter: {}", e); + add_log("error", &error_msg, Some(slug), None); + return Err(error_msg.into()); } } } else { - eprintln!("[Rust Parser] No frontmatter found for post: {}", slug); + add_log("error", "No frontmatter found", Some(slug), None); return Err("No frontmatter found".into()); }; let created_at = get_file_creation_date(&file_path)?; - let processed_markdown = process_anchor_links(&result.content); let processed_markdown = process_custom_tags(&processed_markdown); - eprintln!("[Rust Parser] Processed markdown length: {} characters", processed_markdown.len()); + + add_log("info", "Starting markdown parsing", Some(slug), Some(&format!("Content length: {} chars", processed_markdown.len()))); let parser = Parser::new_ext(&processed_markdown, Options::all()); let mut html_output = String::new(); @@ -432,22 +446,19 @@ pub fn get_post_by_slug(slug: &str) -> Result> let mut code_block_lang = String::new(); let mut code_block_content = String::new(); let mut events = Vec::new(); - let ss = SyntaxSet::load_defaults_newlines(); // SS 卐 + let ss = SyntaxSet::load_defaults_newlines(); let ts = ThemeSet::load_defaults(); let theme = &ts.themes["base16-ocean.dark"]; - // Add error handling around the parsing loop - let mut event_count = 0; let start_parsing = Instant::now(); + let mut event_count = 0; + for event in parser { event_count += 1; - if event_count % 1000 == 0 { - eprintln!("[Rust Parser] Processed {} events for slug: {}", event_count, slug); - // Check for timeout (30 seconds) - if start_parsing.elapsed().as_secs() > 30 { - eprintln!("[Rust Parser] Timeout reached for slug: {}", slug); - return Err("Parsing timeout - file too large".into()); - } + if start_parsing.elapsed().as_secs() > PARSING_TIMEOUT_SECS { + let error_msg = "Parsing timeout - file too large"; + add_log("error", error_msg, Some(slug), Some(&format!("Processed {} events", event_count))); + return Err(error_msg.into()); } match &event { @@ -458,10 +469,8 @@ pub fn get_post_by_slug(slug: &str) -> Result> }, Event::End(Tag::Heading(_, _, _)) => { in_heading = false; - // Strip emojis before slugifying for the id let heading_no_emoji = strip_emojis(&heading_text); let id = slugify(&heading_no_emoji); - // Add basic CSS style for headings let style = "color: #2d3748; margin-top: 1.5em; margin-bottom: 0.5em;"; events.push(Event::Html(CowStr::Boxed(format!("", lvl=heading_level, id=id, style=style).into_boxed_str()))); events.push(Event::Text(CowStr::Boxed(heading_text.clone().into_boxed_str()))); @@ -480,7 +489,6 @@ pub fn get_post_by_slug(slug: &str) -> Result> }, Event::End(Tag::CodeBlock(_)) => { in_code_block = false; - // Highlight code block let highlighted = if !code_block_lang.is_empty() { if let Some(syntax) = ss.find_syntax_by_token(&code_block_lang) { highlighted_html_for_string(&code_block_content, &ss, syntax, theme).unwrap_or_else(|_| format!("
{}
", html_escape::encode_text(&code_block_content))) @@ -488,7 +496,6 @@ pub fn get_post_by_slug(slug: &str) -> Result> format!("
{}
", html_escape::encode_text(&code_block_content)) } } else { - // No language specified format!("
{}
", html_escape::encode_text(&code_block_content)) }; events.push(Event::Html(CowStr::Boxed(highlighted.into_boxed_str()))); @@ -502,12 +509,11 @@ pub fn get_post_by_slug(slug: &str) -> Result> _ => {}, } } - eprintln!("[Rust Parser] Total events processed: {} for slug: {}", event_count, slug); + + add_log("info", "Markdown parsing completed", Some(slug), Some(&format!("Processed {} events", event_count))); + html::push_html(&mut html_output, events.into_iter()); - eprintln!("[Rust Parser] HTML output length: {} characters", html_output.len()); - let sanitized_html = AMMONIA.clean(&html_output).to_string(); - eprintln!("[Rust Parser] Sanitized HTML length: {} characters", sanitized_html.len()); let interpret_time = start.elapsed(); let compile_start = Instant::now(); @@ -522,8 +528,10 @@ pub fn get_post_by_slug(slug: &str) -> Result> author: std::env::var("BLOG_OWNER").unwrap_or_else(|_| "Anonymous".to_string()), }; let compile_time = compile_start.elapsed(); + // Insert into cache POST_CACHE.write().unwrap().insert(slug.to_string(), post.clone()); + // Update stats let mut stats = POST_STATS.write().unwrap(); let entry = stats.entry(slug.to_string()).or_insert_with(|| PostStats { @@ -534,6 +542,9 @@ pub fn get_post_by_slug(slug: &str) -> Result> entry.last_compile_time_ms = compile_time.as_millis(); sys.refresh_process(pid); entry.last_cpu_usage_percent = sys.process(pid).map(|p| p.cpu_usage()).unwrap_or(0.0) - before_cpu; + + add_log("info", "Post parsing completed successfully", Some(slug), Some(&format!("Interpret: {}ms, Compile: {}ms", interpret_time.as_millis(), compile_time.as_millis()))); + Ok(post) } @@ -542,6 +553,7 @@ pub fn get_all_posts() -> Result, Box> { if let Some(posts) = ALL_POSTS_CACHE.read().unwrap().clone() { return Ok(posts); } + let posts_dir = get_posts_directory(); let markdown_files = find_markdown_files(&posts_dir)?; let mut posts = Vec::new(); @@ -549,14 +561,12 @@ pub fn get_all_posts() -> Result, Box> { for file_path in markdown_files { let slug = path_to_slug(&file_path, &posts_dir); if let Ok(post) = get_post_by_slug(&slug) { - // Insert each post into the individual post cache as well POST_CACHE.write().unwrap().insert(slug.clone(), post.clone()); posts.push(post); } } posts.sort_by(|a, b| b.created_at.cmp(&a.created_at)); - // Cache the result *ALL_POSTS_CACHE.write().unwrap() = Some(posts.clone()); Ok(posts) } @@ -570,11 +580,11 @@ pub fn watch_posts(on_change: F) -> notify::Result { - // Invalidate caches on any change POST_CACHE.write().unwrap().clear(); *ALL_POSTS_CACHE.write().unwrap() = None; on_change(); @@ -618,6 +628,7 @@ pub fn checkhealth() -> HealthReport { let posts_dir = get_posts_directory(); let posts_dir_exists = posts_dir.exists() && posts_dir.is_dir(); let mut posts_count = 0; + if posts_dir_exists { match std::fs::read_dir(&posts_dir) { Ok(entries) => { @@ -630,9 +641,11 @@ pub fn checkhealth() -> HealthReport { } else { errors.push("Posts directory does not exist".to_string()); } + let cache_file_exists = Path::new(POSTS_CACHE_PATH).exists(); let cache_stats_file_exists = Path::new(POST_STATS_PATH).exists(); let (mut cache_readable, mut cache_post_count) = (false, None); + if cache_file_exists { match std::fs::read_to_string(POSTS_CACHE_PATH) { Ok(data) => { @@ -647,6 +660,7 @@ pub fn checkhealth() -> HealthReport { Err(e) => errors.push(format!("Failed to read cache file: {}", e)), } } + let (mut cache_stats_readable, mut cache_stats_count) = (false, None); if cache_stats_file_exists { match std::fs::read_to_string(POST_STATS_PATH) { @@ -662,6 +676,7 @@ pub fn checkhealth() -> HealthReport { Err(e) => errors.push(format!("Failed to read cache stats file: {}", e)), } } + HealthReport { posts_dir_exists, posts_count, @@ -674,3 +689,13 @@ pub fn checkhealth() -> HealthReport { errors, } } + +pub fn get_parser_logs() -> Vec { + let logs = PARSER_LOGS.read().unwrap(); + logs.iter().cloned().collect() +} + +pub fn clear_parser_logs() { + let mut logs = PARSER_LOGS.write().unwrap(); + logs.clear(); +} \ No newline at end of file diff --git a/src/app/admin/manage/rust-status/page.tsx b/src/app/admin/manage/rust-status/page.tsx index 964b4a7..03cc626 100644 --- a/src/app/admin/manage/rust-status/page.tsx +++ b/src/app/admin/manage/rust-status/page.tsx @@ -21,6 +21,14 @@ interface HealthReport { errors: string[]; } +interface LogEntry { + timestamp: string; + level: string; + message: string; + slug?: string; + details?: string; +} + export default function RustStatusPage() { const [stats, setStats] = useState([]); const [loading, setLoading] = useState(true); @@ -28,6 +36,11 @@ export default function RustStatusPage() { const [health, setHealth] = useState(null); const [healthLoading, setHealthLoading] = useState(true); const [healthError, setHealthError] = useState(null); + const [logs, setLogs] = useState([]); + const [logsLoading, setLogsLoading] = useState(true); + const [logsError, setLogsError] = useState(null); + const [logFilter, setLogFilter] = useState('all'); // 'all', 'info', 'warning', 'error' + const [logSearch, setLogSearch] = useState(''); // Summary calculations const totalHits = stats.reduce((sum, s) => sum + s.cache_hits, 0); @@ -65,11 +78,65 @@ export default function RustStatusPage() { } }; + const fetchLogs = async () => { + setLogsLoading(true); + setLogsError(null); + try { + const res = await fetch('/api/admin/posts?logs=1'); + if (!res.ok) throw new Error('Fehler beim Laden der Logs'); + const data = await res.json(); + setLogs(data); + } catch (e: any) { + setLogsError(e.message || 'Unbekannter Fehler'); + } finally { + setLogsLoading(false); + } + }; + + const clearLogs = async () => { + try { + const res = await fetch('/api/admin/posts?clearLogs=1', { method: 'DELETE' }); + if (!res.ok) throw new Error('Fehler beim Löschen der Logs'); + await fetchLogs(); // Refresh logs after clearing + } catch (e: any) { + console.error('Error clearing logs:', e); + } + }; + useEffect(() => { fetchStats(); fetchHealth(); + fetchLogs(); }, []); + // Filter logs based on level and search term + const filteredLogs = logs.filter(log => { + const matchesLevel = logFilter === 'all' || log.level === logFilter; + const matchesSearch = !logSearch || + log.message.toLowerCase().includes(logSearch.toLowerCase()) || + (log.slug && log.slug.toLowerCase().includes(logSearch.toLowerCase())) || + (log.details && log.details.toLowerCase().includes(logSearch.toLowerCase())); + return matchesLevel && matchesSearch; + }); + + const getLevelColor = (level: string) => { + switch (level) { + case 'error': return 'text-red-600 bg-red-50'; + case 'warning': return 'text-yellow-600 bg-yellow-50'; + case 'info': return 'text-blue-600 bg-blue-50'; + default: return 'text-gray-600 bg-gray-50'; + } + }; + + const getLevelIcon = (level: string) => { + switch (level) { + case 'error': return '❌'; + case 'warning': return '⚠️'; + case 'info': return 'ℹ️'; + default: return '📝'; + } + }; + return (
@@ -101,13 +168,17 @@ export default function RustStatusPage() { {/* Refresh button */}
+ {/* Parser Logs Section */} +
+
+

Parser Logs

+
+ +
+
+ + {/* Log Filters */} +
+
+ setLogSearch(e.target.value)} + className="w-full px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500" + /> +
+
+ +
+
+ + {/* Logs Display */} +
+ {logsLoading &&
Loading logs...
} + {logsError &&
{logsError}
} + {!logsLoading && !logsError && ( +
+ {filteredLogs.length === 0 ? ( +
No logs found
+ ) : ( + filteredLogs.map((log, index) => ( +
+
+ {getLevelIcon(log.level)} +
+
+ + {new Date(log.timestamp).toLocaleString()} + + + {log.level.toUpperCase()} + + {log.slug && ( + + {log.slug} + + )} +
+
{log.message}
+ {log.details && ( +
+ {log.details} +
+ )} +
+
+
+ )) + )} +
+ )} +
+
+ {/* Table */}

Rohdaten

diff --git a/src/app/api/admin/posts/route.ts b/src/app/api/admin/posts/route.ts index 4b2fd27..824669e 100644 --- a/src/app/api/admin/posts/route.ts +++ b/src/app/api/admin/posts/route.ts @@ -90,6 +90,26 @@ export async function GET(request: Request) { }); } } + const logs = searchParams.get('logs'); + if (logs === '1') { + // Call the Rust backend for parser logs + const rustResult = spawnSync( + process.cwd() + '/markdown_backend/target/release/markdown_backend', + ['logs'], + { encoding: 'utf-8' } + ); + if (rustResult.status === 0 && rustResult.stdout) { + return new Response(rustResult.stdout, { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } else { + return new Response(JSON.stringify({ error: rustResult.stderr || rustResult.error }), { + status: 500, + headers: { 'Content-Type': 'application/json' }, + }); + } + } // Return the current pinned.json object try { const pinnedPath = path.join(process.cwd(), 'posts', 'pinned.json'); @@ -150,4 +170,36 @@ export async function PUT(request: Request) { console.error('Error editing post:', error); return NextResponse.json({ error: 'Error editing post' }, { status: 500 }); } +} + +export async function DELETE(request: Request) { + try { + const { searchParams } = new URL(request.url); + const clearLogs = searchParams.get('clearLogs'); + + if (clearLogs === '1') { + // Call the Rust backend to clear parser logs + const rustResult = spawnSync( + process.cwd() + '/markdown_backend/target/release/markdown_backend', + ['clearLogs'], + { encoding: 'utf-8' } + ); + if (rustResult.status === 0 && rustResult.stdout) { + return new Response(rustResult.stdout, { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } else { + return new Response(JSON.stringify({ error: rustResult.stderr || rustResult.error }), { + status: 500, + headers: { 'Content-Type': 'application/json' }, + }); + } + } + + return NextResponse.json({ error: 'Invalid delete operation' }, { status: 400 }); + } catch (error) { + console.error('Error clearing logs:', error); + return NextResponse.json({ error: 'Error clearing logs' }, { status: 500 }); + } } \ No newline at end of file