diff --git a/.gitignore b/.gitignore index da35784..68ce977 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,9 @@ posts/pinned.json posts/Aquaworld/tag-1.md posts/pinned.json posts/pinned.json + +# Rust +target/ +Cargo.lock +**/*.rs.bk +*.pdb diff --git a/Dockerfile b/Dockerfile index 895faeb..75f8d5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,13 @@ +# syntax=docker/dockerfile:1 +FROM rust:latest as rust-build +WORKDIR /build +COPY ./markdown_backend ./markdown_backend +WORKDIR /build/markdown_backend +RUN rustup target add x86_64-unknown-linux-musl +RUN apt-get update && apt-get install -y musl-tools +# Build with musl target for static linking +RUN cargo build --release --target x86_64-unknown-linux-musl + FROM node:20 WORKDIR /app @@ -7,9 +17,20 @@ COPY package*.json ./ RUN npm install COPY . . +# Ensure posts directory exists and has correct permissions +RUN mkdir -p /app/posts +COPY posts/* /app/posts/ +RUN chmod -R 755 /app/posts + +# Copy the statically linked Rust binary from the build stage +COPY --from=rust-build /build/markdown_backend/target/x86_64-unknown-linux-musl/release/markdown_backend ./markdown_backend/target/release/markdown_backend +RUN chmod +x ./markdown_backend/target/release/markdown_backend RUN npm run build +# Create and set permissions for the docker volume mount point +RUN mkdir -p /app/docker && chmod 777 /app/docker + VOLUME ["/app/docker"] EXPOSE 3000 diff --git a/docker.sh b/docker.sh index 4ab52b0..a75ae64 100755 --- a/docker.sh +++ b/docker.sh @@ -2,6 +2,12 @@ set -e +# Check if Docker daemon is running +if ! docker info >/dev/null 2>&1; then + echo "Error: Docker daemon is not running" + exit 1 +fi + IMAGE_NAME="markdownblog" CONTAINER_NAME="markdownblog" VOLUME_NAME="markdownblog-posts" @@ -32,5 +38,11 @@ docker run -d \ echo "Copying built-in posts to Docker volume if empty..." docker exec $CONTAINER_NAME sh -c 'if [ -d /app/posts ] && [ -d /app/docker ] && [ "$(ls -A /app/docker)" = "" ]; then cp -r /app/posts/* /app/docker/; fi' +# Check if container is running +if ! docker ps | grep -q $CONTAINER_NAME; then + echo "Error: Container failed to start. Check logs with: docker logs $CONTAINER_NAME" + exit 1 +fi + echo "Deployment complete!" echo "App should be available at http://localhost:$PORT" diff --git a/markdown_backend/Cargo.toml b/markdown_backend/Cargo.toml new file mode 100644 index 0000000..ac8f950 --- /dev/null +++ b/markdown_backend/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "markdown_backend" +version = "0.1.0" +edition = "2021" + +[dependencies] +chrono = { version = "0.4", features = ["serde"] } +serde = { version = "1.0", features = ["derive"] } +serde_yaml = "0.9" +pulldown-cmark = "0.9" +gray_matter = "0.2.8" +ammonia = "3.1" +slug = "0.1" +notify = "6.1" +syntect = { version = "5.1", features = ["default"] } +regex = "1.10" +clap = { version = "4.4", features = ["derive"] } +serde_json = "1.0" +html-escape = "0.2.13" +once_cell = "1.18" +sysinfo = "0.30.7" diff --git a/markdown_backend/src/main.rs b/markdown_backend/src/main.rs new file mode 100644 index 0000000..176d972 --- /dev/null +++ b/markdown_backend/src/main.rs @@ -0,0 +1,77 @@ +use clap::{Parser, Subcommand}; +mod markdown; +use markdown::{get_all_posts, get_post_by_slug, get_posts_by_tag, watch_posts}; +use serde_json; +use std::fs; + +#[derive(Parser)] +#[command(name = "Markdown Backend")] +#[command(about = "A CLI for managing markdown blog posts", long_about = None)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// List all posts + List, + /// Show a post by slug + Show { + slug: String, + }, + /// List posts by tag + Tags { + tag: String, + }, + /// Watch for changes in the posts directory + Watch, + /// Show Rust parser statistics + Rsparseinfo, +} + +fn main() { + markdown::load_post_cache_from_disk(); + let cli = Cli::parse(); + match &cli.command { + Commands::List => { + let posts = get_all_posts().unwrap_or_else(|e| { + eprintln!("{}", e); + std::process::exit(1); + }); + println!("{}", serde_json::to_string(&posts).unwrap()); + } + Commands::Show { slug } => { + match get_post_by_slug(slug) { + Ok(post) => { + println!("{}", serde_json::to_string(&post).unwrap()); + markdown::save_post_cache_to_disk(); + } + Err(e) => { + eprintln!("{}", e); + std::process::exit(1); + } + } + } + Commands::Tags { tag } => { + let posts = get_posts_by_tag(tag).unwrap_or_else(|e| { + eprintln!("{}", e); + std::process::exit(1); + }); + println!("{}", serde_json::to_string(&posts).unwrap()); + } + Commands::Watch => { + println!("Watching for changes in posts directory. Press Ctrl+C to exit."); + let _ = watch_posts(|| { + println!("Posts directory changed!"); + }); + // Keep the main thread alive + loop { + std::thread::sleep(std::time::Duration::from_secs(60)); + } + } + Commands::Rsparseinfo => { + println!("{}", markdown::rsparseinfo()); + } + } +} \ No newline at end of file diff --git a/markdown_backend/src/markdown.rs b/markdown_backend/src/markdown.rs new file mode 100644 index 0000000..db40889 --- /dev/null +++ b/markdown_backend/src/markdown.rs @@ -0,0 +1,374 @@ +// src/markdown.rs +/* + +This is the Rust Markdown Parser. +It supports caching of posts and is + +BLAZINGLY FAST! + +*/ + + +use std::fs; +use std::path::{Path, PathBuf}; +use chrono::{DateTime, Utc}; +use serde::Deserialize; +use pulldown_cmark::{Parser, Options, html, Event, Tag, CowStr}; +use gray_matter::engine::YAML; +use gray_matter::Matter; +use ammonia::clean; +use slug::slugify; +use notify::{RecursiveMode, RecommendedWatcher, Watcher, Config}; +use std::sync::mpsc::channel; +use std::time::{Duration, Instant}; +use syntect::highlighting::{ThemeSet, Style}; +use syntect::parsing::SyntaxSet; +use syntect::html::{highlighted_html_for_string, IncludeBackground}; +use once_cell::sync::Lazy; +use std::collections::HashMap; +use std::sync::RwLock; +use serde_json; +use sysinfo::{System, Pid, RefreshKind, CpuRefreshKind, ProcessRefreshKind}; + +const POSTS_CACHE_PATH: &str = "./cache/posts_cache.json"; +const POST_STATS_PATH: &str = "./cache/post_stats.json"; + +#[derive(Debug, Deserialize, Clone, serde::Serialize)] +pub struct PostFrontmatter { + pub title: String, + pub date: String, + pub tags: Option>, + pub summary: Option, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct Post { + pub slug: String, + pub title: String, + pub date: String, + pub tags: Vec, + pub summary: Option, + pub content: String, + pub created_at: String, + pub author: String, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)] +pub struct PostStats { + pub slug: String, + pub cache_hits: u64, + pub cache_misses: u64, + pub last_interpret_time_ms: u128, + pub last_compile_time_ms: u128, + pub last_cpu_usage_percent: f32, // Not f64 + pub last_cache_status: String, // "hit" or "miss" +} + +static POST_CACHE: Lazy>> = Lazy::new(|| RwLock::new(HashMap::new())); +static ALL_POSTS_CACHE: Lazy>>> = Lazy::new(|| RwLock::new(None)); +static POST_STATS: Lazy>> = Lazy::new(|| RwLock::new(HashMap::new())); + +fn get_posts_directory() -> PathBuf { + let candidates = [ + "./posts", + "../posts", + "/posts", + "/docker" + ]; + for candidate in candidates.iter() { + let path = PathBuf::from(candidate); + if path.exists() && path.is_dir() { + return path; + } + } + // Fallback: default to ./posts + PathBuf::from("./posts") +} + +fn get_file_creation_date(path: &Path) -> std::io::Result> { + let metadata = fs::metadata(path)?; + // Try to get creation time, fall back to modification time if not available + match metadata.created() { + Ok(created) => Ok(DateTime::::from(created)), + Err(_) => { + // Fall back to modification time if creation time is not available + let modified = metadata.modified()?; + Ok(DateTime::::from(modified)) + } + } +} + +fn process_anchor_links(content: &str) -> String { + // Replace [text](#anchor) with slugified anchor + let re = regex::Regex::new(r"\[([^\]]+)\]\(#([^)]+)\)").unwrap(); + re.replace_all(content, |caps: ®ex::Captures| { + let link_text = &caps[1]; + let anchor = &caps[2]; + let slugified = slugify(anchor); + format!("[{}](#{})", link_text, slugified) + }).to_string() +} + +// Helper function to strip emojis from a string +// Neccesary for the slugify function to work correctly. And the ID's to work with the frontend. +fn strip_emojis(s: &str) -> String { + // Remove all characters in the Emoji Unicode ranges + // This is a simple approach and may not cover all emojis, but works for most cases + s.chars() + .filter(|c| { + let c = *c as u32; + // Basic Emoji ranges + !( (c >= 0x1F600 && c <= 0x1F64F) // Emoticons + || (c >= 0x1F300 && c <= 0x1F5FF) // Misc Symbols and Pictographs + || (c >= 0x1F680 && c <= 0x1F6FF) // Transport and Map + || (c >= 0x2600 && c <= 0x26FF) // Misc symbols + || (c >= 0x2700 && c <= 0x27BF) // Dingbats + || (c >= 0x1F900 && c <= 0x1F9FF) // Supplemental Symbols and Pictographs + || (c >= 0x1FA70 && c <= 0x1FAFF) // Symbols and Pictographs Extended-A + || (c >= 0x1F1E6 && c <= 0x1F1FF) // Regional Indicator Symbols + ) + }) + .collect() +} + +static AMMONIA: Lazy> = Lazy::new(|| { + let mut builder = ammonia::Builder::default(); + builder.add_tag_attributes("h1", &["id"]); + builder.add_tag_attributes("h2", &["id"]); + builder.add_tag_attributes("h3", &["id"]); + builder.add_tag_attributes("h4", &["id"]); + builder.add_tag_attributes("h5", &["id"]); + builder.add_tag_attributes("h6", &["id"]); + builder +}); + +pub fn rsparseinfo() -> String { + // Eagerly load all posts to populate stats + let _ = get_all_posts(); + let stats = POST_STATS.read().unwrap(); + let values: Vec<&PostStats> = stats.values().collect(); + if values.is_empty() { + "[]".to_string() + } else { + serde_json::to_string(&values).unwrap_or_else(|_| "[]".to_string()) + } +} + +pub fn get_post_by_slug(slug: &str) -> Result> { + let mut sys = System::new_with_specifics(RefreshKind::new().with_processes(ProcessRefreshKind::everything()).with_cpu(CpuRefreshKind::everything())); + sys.refresh_processes(); + let pid = sysinfo::get_current_pid()?; + let before_cpu = sys.process(pid).map(|p| p.cpu_usage()).unwrap_or(0.0); + let start = Instant::now(); + let mut stats = POST_STATS.write().unwrap(); + let entry = stats.entry(slug.to_string()).or_insert_with(|| PostStats { + slug: slug.to_string(), + ..Default::default() + }); + // Try cache first + if let Some(post) = POST_CACHE.read().unwrap().get(slug).cloned() { + entry.cache_hits += 1; + entry.last_interpret_time_ms = 0; + entry.last_compile_time_ms = 0; + entry.last_cache_status = "hit".to_string(); + sys.refresh_process(pid); + entry.last_cpu_usage_percent = sys.process(pid).map(|p| p.cpu_usage()).unwrap_or(0.0) - before_cpu; + return Ok(post); + } + entry.cache_misses += 1; + entry.last_cache_status = "miss".to_string(); + drop(stats); // Release lock before heavy work + let posts_dir = get_posts_directory(); + let file_path = posts_dir.join(format!("{}.md", slug)); + let file_content = fs::read_to_string(&file_path)?; + + let matter = Matter::::new(); + let result = matter.parse(&file_content); + + let front: PostFrontmatter = if let Some(data) = result.data { + match data.deserialize() { + Ok(front) => front, + Err(e) => { + eprintln!("Failed to deserialize frontmatter for post {}: {}", slug, e); + return Err("Failed to deserialize frontmatter".into()); + } + } + } else { + eprintln!("No frontmatter found for post: {}", slug); + return Err("No frontmatter found".into()); + }; + + let created_at = get_file_creation_date(&file_path)?; + + let processed_markdown = process_anchor_links(&result.content); + let parser = Parser::new_ext(&processed_markdown, Options::all()); + let mut html_output = String::new(); + let mut heading_text = String::new(); + let mut in_heading = false; + let mut heading_level = 0; + let mut in_code_block = false; + let mut code_block_lang = String::new(); + let mut code_block_content = String::new(); + let mut events = Vec::new(); + let ss = SyntaxSet::load_defaults_newlines(); // SS 卐 + let ts = ThemeSet::load_defaults(); + let theme = &ts.themes["base16-ocean.dark"]; + for event in parser { + match &event { + Event::Start(Tag::Heading(level, _, _)) => { + in_heading = true; + heading_level = *level as usize; + heading_text.clear(); + }, + Event::End(Tag::Heading(_, _, _)) => { + in_heading = false; + // Strip emojis before slugifying for the id + let heading_no_emoji = strip_emojis(&heading_text); + let id = slugify(&heading_no_emoji); + events.push(Event::Html(CowStr::Boxed(format!("", lvl=heading_level, id=id).into_boxed_str()))); + events.push(Event::Text(CowStr::Boxed(heading_text.clone().into_boxed_str()))); + events.push(Event::Html(CowStr::Boxed(format!("", lvl=heading_level).into_boxed_str()))); + }, + Event::Text(text) if in_heading => { + heading_text.push_str(text); + }, + Event::Start(Tag::CodeBlock(kind)) => { + in_code_block = true; + code_block_content.clear(); + code_block_lang = match kind { + pulldown_cmark::CodeBlockKind::Fenced(lang) => lang.to_string(), + pulldown_cmark::CodeBlockKind::Indented => String::new(), + }; + }, + Event::End(Tag::CodeBlock(_)) => { + in_code_block = false; + // Highlight code block + let highlighted = if !code_block_lang.is_empty() { + if let Some(syntax) = ss.find_syntax_by_token(&code_block_lang) { + highlighted_html_for_string(&code_block_content, &ss, syntax, theme).unwrap_or_else(|_| format!("
{}
", html_escape::encode_text(&code_block_content))) + } else { + format!("
{}
", html_escape::encode_text(&code_block_content)) + } + } else { + // No language specified + format!("
{}
", html_escape::encode_text(&code_block_content)) + }; + events.push(Event::Html(CowStr::Boxed(highlighted.into_boxed_str()))); + }, + Event::Text(text) if in_code_block => { + code_block_content.push_str(text); + }, + _ if !in_heading && !in_code_block => { + events.push(event); + }, + _ => {}, + } + } + html::push_html(&mut html_output, events.into_iter()); + + let sanitized_html = AMMONIA.clean(&html_output).to_string(); + + let interpret_time = start.elapsed(); + let compile_start = Instant::now(); + let post = Post { + slug: slug.to_string(), + title: front.title, + date: front.date, + tags: front.tags.unwrap_or_default(), + summary: front.summary, + content: sanitized_html, + created_at: created_at.to_rfc3339(), + author: std::env::var("BLOG_OWNER").unwrap_or_else(|_| "Anonymous".to_string()), + }; + let compile_time = compile_start.elapsed(); + // Insert into cache + POST_CACHE.write().unwrap().insert(slug.to_string(), post.clone()); + // Update stats + let mut stats = POST_STATS.write().unwrap(); + let entry = stats.entry(slug.to_string()).or_insert_with(|| PostStats { + slug: slug.to_string(), + ..Default::default() + }); + entry.last_interpret_time_ms = interpret_time.as_millis(); + entry.last_compile_time_ms = compile_time.as_millis(); + sys.refresh_process(pid); + entry.last_cpu_usage_percent = sys.process(pid).map(|p| p.cpu_usage()).unwrap_or(0.0) - before_cpu; + Ok(post) +} + +pub fn get_all_posts() -> Result, Box> { + // Try cache first + if let Some(posts) = ALL_POSTS_CACHE.read().unwrap().clone() { + return Ok(posts); + } + let posts_dir = get_posts_directory(); + let mut posts = Vec::new(); + for entry in fs::read_dir(posts_dir)? { + let entry = entry?; + let path = entry.path(); + if path.extension().map(|e| e == "md").unwrap_or(false) { + let file_stem = path.file_stem().unwrap().to_string_lossy(); + if let Ok(post) = get_post_by_slug(&file_stem) { + // Insert each post into the individual post cache as well + POST_CACHE.write().unwrap().insert(file_stem.to_string(), post.clone()); + posts.push(post); + } + } + } + posts.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + // Cache the result + *ALL_POSTS_CACHE.write().unwrap() = Some(posts.clone()); + Ok(posts) +} + +pub fn get_posts_by_tag(tag: &str) -> Result, Box> { + let all_posts = get_all_posts()?; + Ok(all_posts.into_iter().filter(|p| p.tags.contains(&tag.to_string())).collect()) +} + +pub fn watch_posts(on_change: F) -> notify::Result { + let (tx, rx) = channel(); + let mut watcher = RecommendedWatcher::new(tx, Config::default())?; + watcher.watch(get_posts_directory().as_path(), RecursiveMode::Recursive)?; + std::thread::spawn(move || { + loop { + match rx.recv() { + Ok(_event) => { + // Invalidate caches on any change + POST_CACHE.write().unwrap().clear(); + *ALL_POSTS_CACHE.write().unwrap() = None; + on_change(); + }, + Err(e) => { + eprintln!("watch error: {:?}", e); + break; + } + } + } + }); + Ok(watcher) +} + +pub fn load_post_cache_from_disk() { + if let Ok(data) = fs::read_to_string(POSTS_CACHE_PATH) { + if let Ok(map) = serde_json::from_str::>(&data) { + *POST_CACHE.write().unwrap() = map; + } + } + if let Ok(data) = fs::read_to_string(POST_STATS_PATH) { + if let Ok(map) = serde_json::from_str::>(&data) { + *POST_STATS.write().unwrap() = map; + } + } +} + +pub fn save_post_cache_to_disk() { + if let Ok(map) = serde_json::to_string(&*POST_CACHE.read().unwrap()) { + let _ = fs::create_dir_all("./cache"); + let _ = fs::write(POSTS_CACHE_PATH, map); + } + if let Ok(map) = serde_json::to_string(&*POST_STATS.read().unwrap()) { + let _ = fs::create_dir_all("./cache"); + let _ = fs::write(POST_STATS_PATH, map); + } +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 47f0e54..44edc60 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "autoprefixer": "^10.4.17", "bcrypt": "^5.0.2", "bcryptjs": "^2.4.3", + "chart.js": "^4.5.0", "chokidar": "^3.6.0", "date-fns": "^3.6.0", "dompurify": "^3.0.9", @@ -28,6 +29,7 @@ "pm2": "^6.0.8", "postcss": "^8.4.35", "react": "^18.2.0", + "react-chartjs-2": "^5.3.0", "react-dom": "^18.2.0", "tailwindcss": "^3.4.1", "typescript": "^5.3.3" @@ -502,6 +504,12 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@kurkle/color": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.4.tgz", + "integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==", + "license": "MIT" + }, "node_modules/@mapbox/node-pre-gyp": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz", @@ -2214,6 +2222,18 @@ "integrity": "sha512-syedaZ9cPe7r3hoQA9twWYKu5AIyCswN5+szkmPBe9ccdLrj4bYaCnLVPTLd2kgVRc7+zoX4tyPgRnFKCj5YjQ==", "license": "MIT/X11" }, + "node_modules/chart.js": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.0.tgz", + "integrity": "sha512-aYeC/jDgSEx8SHWZvANYMioYMZ2KX02W6f6uVfyteuCGcadDLcYVHdfdygsTQkQ4TKn5lghoojAsPj5pu0SnvQ==", + "license": "MIT", + "dependencies": { + "@kurkle/color": "^0.3.0" + }, + "engines": { + "pnpm": ">=8" + } + }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -6805,6 +6825,16 @@ "node": ">=0.10.0" } }, + "node_modules/react-chartjs-2": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/react-chartjs-2/-/react-chartjs-2-5.3.0.tgz", + "integrity": "sha512-UfZZFnDsERI3c3CZGxzvNJd02SHjaSJ8kgW1djn65H1KK8rehwTjyrRKOG3VTMG8wtHZ5rgAO5oTHtHi9GCCmw==", + "license": "MIT", + "peerDependencies": { + "chart.js": "^4.1.1", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/react-dom": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", diff --git a/package.json b/package.json index bd66501..7893274 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "autoprefixer": "^10.4.17", "bcrypt": "^5.0.2", "bcryptjs": "^2.4.3", + "chart.js": "^4.5.0", "chokidar": "^3.6.0", "date-fns": "^3.6.0", "dompurify": "^3.0.9", @@ -31,6 +32,7 @@ "pm2": "^6.0.8", "postcss": "^8.4.35", "react": "^18.2.0", + "react-chartjs-2": "^5.3.0", "react-dom": "^18.2.0", "tailwindcss": "^3.4.1", "typescript": "^5.3.3" diff --git a/posts/welcome.md b/posts/welcome.md index 22f837e..7ed70e3 100644 --- a/posts/welcome.md +++ b/posts/welcome.md @@ -29,6 +29,8 @@ author: Rattatwinko - [Support the Project ❤️](#support-the-project) - [Acknowledgments 🙏](#acknowledgments) - [Folder Emojis 🇦🇹](#folder-emoji-technical-note) +- [API 🏗️](#api) +- [ToT, and Todo](#train-of-thought-for-this-project-and-todo) --- @@ -387,6 +389,35 @@ Thanks for choosing MarkdownBlog! If you find it useful, please: --- +## API + +MarkdownBlog provides a built-in RESTful API to serve post data, handle live updates, and support integrations. The API is used internally by the frontend to fetch posts, stream updates (for live reloads), and manage features like emoji and pin assignments. You can also interact with these endpoints to build custom tools or integrations. + +Key API endpoints include: + +- `/api/posts`: Fetch all blog posts as JSON. +- `/api/posts/[slug]`: Fetch a single post by its slug. +- `/api/posts/stream`: Server-Sent Events (SSE) endpoint for real-time updates when posts change. +- `/api/posts/webhook`: Webhook endpoint to notify the app of external changes (e.g., from CI/CD or scripts). + +All API routes are implemented using Next.js API routes and are available out of the box. For more details, check the code in the `src/app/api/posts/` directory. + + +-- + +## Train of Thought for this Project and Todo + +Ok, so when I originally did this (about a week ago speaking from 24.6.25), I really had no thought of this being a huge thing. But reallistically speaking, this Repository is 2MiB large. And its bloated. But this aside it's a really cool little thing you can deploy anywhere, where Docker runs. + +If you have seen this is not very mindfull of browser resources tho. + +|IS DONE|Task| +|-------|----| +|partly / working on it|_Rewrite_ the Markdown Parser in **Rust** ; This works for local Builds but in Docker does not work due to permission error| + +--- + + I looooooove Markdown diff --git a/src/app/admin/manage/page.tsx b/src/app/admin/manage/page.tsx index 47650b4..fa39ae3 100644 --- a/src/app/admin/manage/page.tsx +++ b/src/app/admin/manage/page.tsx @@ -256,6 +256,16 @@ export default function ManagePage() { + + + + + Rust Parser Status + + + + {loading && ( +
+
+
Loading stats...
+
+ )} + {error && ( +
{error}
+ )} + {!loading && !error && ( + <> + {/* Summary Cards */} +
+
+ {totalHits} + Total Cache Hits +
+
+ {totalMisses} + Total Cache Misses +
+
+ {avgInterpret} ms + Avg Interpret Time +
+
+ {avgCompile} ms + Avg Compile Time +
+
+ + {/* Bar Chart */} +
+ +
+ + {/* Raw Data Table */} +
+ + + + + + + + + + + + {stats.length === 0 ? ( + + ) : ( + stats.map(stat => ( + + + + + + + + )) + )} + +
SlugCache HitsCache MissesLast Interpret Time (ms)Last Compile Time (ms)
No stats available.
{stat.slug}{stat.cache_hits}{stat.cache_misses}{stat.last_interpret_time_ms}{stat.last_compile_time_ms}
+
+ + )} + + ); +} \ No newline at end of file diff --git a/src/app/api/admin/posts/route.ts b/src/app/api/admin/posts/route.ts index 023b4da..b2397c7 100644 --- a/src/app/api/admin/posts/route.ts +++ b/src/app/api/admin/posts/route.ts @@ -3,6 +3,7 @@ import fs from 'fs'; import path from 'path'; import matter from 'gray-matter'; import { getPostsDirectory } from '@/lib/postsDirectory'; +import { spawnSync } from 'child_process'; const postsDirectory = getPostsDirectory(); @@ -48,6 +49,27 @@ export async function POST(request: Request) { } export async function GET(request: Request) { + const { searchParams } = new URL(request.url); + const info = searchParams.get('rsparseinfo'); + if (info === '1') { + // Call the Rust backend for parser stats + const rustResult = spawnSync( + process.cwd() + '/markdown_backend/target/release/markdown_backend', + ['rsparseinfo'], + { encoding: 'utf-8' } + ); + if (rustResult.status === 0 && rustResult.stdout) { + return new Response(rustResult.stdout, { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } else { + return new Response(JSON.stringify({ error: rustResult.stderr || rustResult.error }), { + status: 500, + headers: { 'Content-Type': 'application/json' }, + }); + } + } // Return the current pinned.json object try { const pinnedPath = path.join(process.cwd(), 'posts', 'pinned.json'); diff --git a/src/app/api/posts/[slug]/route.ts b/src/app/api/posts/[slug]/route.ts index c6e2273..16c7f21 100644 --- a/src/app/api/posts/[slug]/route.ts +++ b/src/app/api/posts/[slug]/route.ts @@ -9,6 +9,7 @@ import DOMPurify from 'dompurify'; import { JSDOM } from 'jsdom'; import hljs from 'highlight.js'; import { getPostsDirectory } from '@/lib/postsDirectory'; +import { spawnSync } from 'child_process'; const postsDirectory = getPostsDirectory(); @@ -52,6 +53,29 @@ marked.setOptions({ async function getPostBySlug(slug: string) { const realSlug = slug.replace(/\.md$/, ''); const fullPath = path.join(postsDirectory, `${realSlug}.md`); + let rustResult; + try { + // Try Rust backend first + rustResult = spawnSync( + path.resolve(process.cwd(), 'markdown_backend/target/release/markdown_backend'), + ['show', realSlug], + { encoding: 'utf-8' } + ); + if (rustResult.status === 0 && rustResult.stdout) { + // Expect Rust to output a JSON object matching the post shape + const post = JSON.parse(rustResult.stdout); + // Map snake_case to camelCase for frontend compatibility + post.createdAt = post.created_at; + delete post.created_at; + return post; + } else { + console.error('[Rust parser error]', rustResult.stderr || rustResult.error); + } + } catch (e) { + console.error('[Rust parser exception]', e); + } + + // Fallback to TypeScript parser const fileContents = fs.readFileSync(fullPath, 'utf8'); const { data, content } = matter(fileContents); const createdAt = getFileCreationDate(fullPath); @@ -60,12 +84,8 @@ async function getPostBySlug(slug: string) { try { // Convert markdown to HTML const rawHtml = marked.parse(content); - - // Create a DOM window for DOMPurify const window = new JSDOM('').window; const purify = DOMPurify(window); - - // Sanitize the HTML processedContent = purify.sanitize(rawHtml as string, { ALLOWED_TAGS: [ 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', @@ -80,11 +100,10 @@ async function getPostBySlug(slug: string) { 'src', 'alt', 'title', 'width', 'height', 'frameborder', 'allowfullscreen' ], - ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i + ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|[^a-z]|[a-z+.-]+(?:[^a-z+.-:]|$))/i }); } catch (err) { console.error(`Error processing markdown for slug "${realSlug}":`, err); - // Return a more informative error message in the content processedContent = `

Error processing markdown content. Please check the console for details.

${err instanceof Error ? err.message : 'Unknown error'}
@@ -107,11 +126,40 @@ export async function GET( request: Request, { params }: { params: { slug: string[] | string } } ) { + let parser = 'typescript'; + let rustError = ''; try { const slugArr = Array.isArray(params.slug) ? params.slug : [params.slug]; const slugPath = slugArr.join('/'); - const post = await getPostBySlug(slugPath); - return NextResponse.json(post); + let post; + try { + const rustResult = spawnSync( + path.resolve(process.cwd(), 'markdown_backend/target/release/markdown_backend'), + ['show', slugPath], + { encoding: 'utf-8' } + ); + if (rustResult.status === 0 && rustResult.stdout) { + post = JSON.parse(rustResult.stdout); + post.createdAt = post.created_at; + delete post.created_at; + parser = 'rust'; + } else { + rustError = rustResult.stderr || rustResult.error?.toString() || 'Unknown error'; + console.error('[Rust parser error]', rustError); + } + } catch (e) { + rustError = e instanceof Error ? e.message : String(e); + console.error('[Rust parser exception]', rustError); + } + if (!post) { + post = await getPostBySlug(slugPath); + } + const response = NextResponse.json(post); + response.headers.set('X-Parser', parser); + if (parser !== 'rust' && rustError) { + response.headers.set('X-Rust-Parser-Error', rustError); + } + return response; } catch (error) { console.error('Error loading post:', error); return NextResponse.json( diff --git a/src/app/page.tsx b/src/app/page.tsx index 4f32407..fc5a5e3 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -33,6 +33,7 @@ export default function Home() { const [search, setSearch] = useState(''); const [isLoading, setIsLoading] = useState(false); const [lastUpdate, setLastUpdate] = useState(null); + const [error, setError] = useState(null); // Get blog owner from env const blogOwner = process.env.NEXT_PUBLIC_BLOG_OWNER || 'Anonymous'; @@ -99,12 +100,17 @@ export default function Home() { const loadTree = async () => { try { setIsLoading(true); + setError(null); const response = await fetch('/api/posts'); + if (!response.ok) { + throw new Error(`API error: ${response.status}`); + } const data = await response.json(); setTree(data); setLastUpdate(new Date()); } catch (error) { console.error('Fehler beim Laden der Beiträge:', error); + setError(error instanceof Error ? error.message : String(error)); } finally { setIsLoading(false); } @@ -168,6 +174,12 @@ export default function Home() { return (
+ {/* Error display */} + {error && ( +
+ Fehler: {error} +
+ )} {/* Mobile-first header section */}

{blogOwner}'s Blog

diff --git a/src/app/posts/[...slug]/page.tsx b/src/app/posts/[...slug]/page.tsx index 1bc16f0..7db7cdb 100644 --- a/src/app/posts/[...slug]/page.tsx +++ b/src/app/posts/[...slug]/page.tsx @@ -14,6 +14,23 @@ interface Post { createdAt: string; } +// Runtime statistics for parser usage +const parserStats = { + rust: 0, + typescript: 0, + lastRustError: '', +}; + +// Add a slugify function that matches Rust's slug::slugify +function slugify(text: string): string { + return text + .toLowerCase() + .normalize('NFKD') + .replace(/[\u0300-\u036F]/g, '') // Remove diacritics + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); +} + export default function PostPage({ params }: { params: { slug: string[] } }) { const [post, setPost] = useState(null); // Modal state for zoomed image @@ -308,35 +325,49 @@ export default function PostPage({ params }: { params: { slug: string[] } }) { } } - // Find the element, but only consider visible ones - const allElements = document.querySelectorAll(`#${id}`); + // Try to find the element by the raw ID first + let allElements = document.querySelectorAll(`#${id}`); let element: HTMLElement | null = null; - - // Check if we're on desktop or mobile - const isDesktop = window.innerWidth >= 640; - for (const el of Array.from(allElements)) { const htmlEl = el as HTMLElement; - // Check if the element is visible (not hidden by CSS) const rect = htmlEl.getBoundingClientRect(); const isVisible = rect.width > 0 && rect.height > 0; - if (isVisible) { element = htmlEl; break; } } - if (element) { - console.log('Found target element:', element.textContent?.substring(0, 50)); + console.log('Found target element (raw id):', element.textContent?.substring(0, 50)); scrollToElement(element); - } else if (retryCount < 5) { + return; + } + // If not found, try slugified version + const slugId = slugify(id); + if (slugId !== id) { + allElements = document.querySelectorAll(`#${slugId}`); + for (const el of Array.from(allElements)) { + const htmlEl = el as HTMLElement; + const rect = htmlEl.getBoundingClientRect(); + const isVisible = rect.width > 0 && rect.height > 0; + if (isVisible) { + element = htmlEl; + break; + } + } + if (element) { + console.log('Found target element (slugified id):', element.textContent?.substring(0, 50)); + scrollToElement(element); + return; + } + } + if (retryCount < 5) { console.log(`Element not found for anchor: ${id}, retrying... (${retryCount + 1}/5)`); setTimeout(() => { findAndScrollToElement(id, retryCount + 1); }, 100); } else { - console.warn(`Element with id "${id}" not found after retries`); + console.warn(`Element with id "${id}" (or slugified "${slugId}") not found after retries`); } }; @@ -648,6 +679,20 @@ export default function PostPage({ params }: { params: { slug: string[] } }) { const loadPost = async () => { try { const response = await fetch(`/api/posts/${encodeURIComponent(slugPath)}`); + const parser = response.headers.get('X-Parser'); + const rustError = response.headers.get('X-Rust-Parser-Error'); + if (parser === 'rust') { + parserStats.rust++; + console.log('%c[Rust Parser] Used for this post.', 'color: green; font-weight: bold'); + } else { + parserStats.typescript++; + console.log('%c[TypeScript Parser] Used for this post.', 'color: orange; font-weight: bold'); + if (rustError) { + parserStats.lastRustError = rustError; + console.warn('[Rust Parser Error]', rustError); + } + } + console.info('[Parser Stats]', parserStats); const data = await response.json(); setPost(data); } catch (error) { diff --git a/src/lib/markdown.ts b/src/lib/markdown.ts index 3827190..8f36e2e 100644 --- a/src/lib/markdown.ts +++ b/src/lib/markdown.ts @@ -1,3 +1,12 @@ +// This is the frontend Markdown parser. +// It is written in TypeScript +// While I was writing this, only I and God knew how it works. +// Now, only God knows. +// +// If you are trying to understand how it works , and optimize it. Please increse the counter +// +// Hours wasted here: 12 + import fs from 'fs'; import path from 'path'; import matter from 'gray-matter'; @@ -221,8 +230,25 @@ export function watchPosts(callback: () => void) { onChangeCallback = callback; watcher = chokidar.watch(postsDirectory, { - ignored: /(^|[\/\\])\../, // ignore dotfiles - persistent: true + ignored: [ + /(^|[\/\\])\../, // ignore dotfiles + /node_modules/, + /\.git/, + /\.next/, + /\.cache/, + /\.DS_Store/, + /Thumbs\.db/, + /\.tmp$/, + /\.temp$/ + ], + persistent: true, + ignoreInitial: true, // Don't trigger on initial scan + awaitWriteFinish: { + stabilityThreshold: 1000, // Wait 1 second after file changes + pollInterval: 100 // Check every 100ms + }, + usePolling: false, // Use native file system events when possible + interval: 1000 // Fallback polling interval (only used if native events fail) }); watcher @@ -235,20 +261,6 @@ function handleFileChange() { if (onChangeCallback) { onChangeCallback(); } - - // Also notify via webhook if available - try { - fetch('/api/posts/webhook', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ type: 'update', timestamp: new Date().toISOString() }) - }).catch(error => { - // Webhook is optional, so we don't need to handle this as a critical error - console.debug('Webhook notification failed:', error); - }); - } catch (error) { - // Ignore webhook errors - } } export function stopWatching() {