diff --git a/.gitignore b/.gitignore index 37b09be..4997ab3 100644 --- a/.gitignore +++ b/.gitignore @@ -674,4 +674,8 @@ fabric.properties .env lvenv -wvenv \ No newline at end of file +wvenv + +## html doc + +*.html \ No newline at end of file diff --git a/TODO.md b/TODO.md index 724c656..06dd919 100644 --- a/TODO.md +++ b/TODO.md @@ -198,6 +198,27 @@ This is what the programm outputted to the stdout file (inserted into out.html): - - ``` + +### Fix (22/04/26): + +rewrite of sanizizer.py + +now output looks like this: + +```html + + + + + + +Document + + +

Test

+

This is a test

+ + +``` + diff --git a/src/profiles.py b/src/profiles.py index 7eb6ac9..359d202 100644 --- a/src/profiles.py +++ b/src/profiles.py @@ -71,7 +71,7 @@ def build_profiles_from_cfg(cfg: Optional[Dict[str, Any]] = None) -> Dict[str, D # like daddy :3 STRICT = { "tags": [ - "html", "body", "header", "main", "footer", "section", "article", "nav", + "html", "body", "header", "head", "title", "meta", "link", "body", "main", "footer", "section", "article", "nav", "aside", "h1", "h2", "h3", "h4", "h5", "h6", "p", "pre", "code", "blockquote", "ul", "ol", "li", "dl", "dt", "dd", @@ -90,6 +90,8 @@ def build_profiles_from_cfg(cfg: Optional[Dict[str, Any]] = None) -> Dict[str, D "th": ["scope", "colspan", "rowspan"], "td": ["colspan", "rowspan"], "time": ["datetime"], + "meta": ["charset", "name", "content"], + "html": ["lang"], "*": ["class", "id"] # limit }, "protocols": ["http", "https", "mailto"], @@ -99,7 +101,7 @@ def build_profiles_from_cfg(cfg: Optional[Dict[str, Any]] = None) -> Dict[str, D # like my ass :3 LOOSE = { "tags": [ - "html", "body", "header", "main", "footer", "section", "article", "nav", + "html", "body", "header", "head", "title", "meta", "link", "body", "main", "footer", "section", "article", "nav", "aside", "h1", "h2", "h3", "h4", "h5", "h6", "p", "pre", "code", "blockquote", "q", "ul", "ol", "li", "dl", "dt", "dd", @@ -127,6 +129,8 @@ def build_profiles_from_cfg(cfg: Optional[Dict[str, Any]] = None) -> Dict[str, D "span": ["class", "id", "style"], "pre": ["class"], "code": ["class"], + "meta": ["charset", "name", "content"], + "html": ["lang"], "*": ["class", "id", "style"] }, "protocols": ["http", "https", "mailto", "data"], diff --git a/src/sanitizer.py b/src/sanitizer.py index d2b9a33..79b10bd 100644 --- a/src/sanitizer.py +++ b/src/sanitizer.py @@ -4,66 +4,88 @@ # import bleach -from bs4 import BeautifulSoup, Tag import datetime -from log.Logger import * +from bs4 import BeautifulSoup +from log.Logger import Logger logger = Logger() + def sanitize(html: str, mode: str, profiles: dict) -> str: + """ + Arguments: + html : str -> input html + mode : str -> the defined mode which is from profiles.py ; or PROFILES variable + + Functionality: + This function takes in HTML sanitzies it with bleach and returns clean HTML + """ + if mode == "passthrough": logger.log_warning("You are about to not care about sanitizing ANY HTML. You are in passthrough mode.") - return _wrap_document(html) # still wrap passthrough output + return _wrap_document(html) + config = profiles.get(mode) if not config: - raise ValueError(f"Unknown mode! : {mode}") - allowed = set(config.get("tags", [])) - good_soup = BeautifulSoup(html, "html.parser") - def has_allowed_child(node: Tag) -> bool: - for child in node.descendants: - if isinstance(child, Tag) and child.name.lower() in allowed: - return True - return False - for child in list(good_soup.find_all()): - name = child.name.lower() - if name in allowed: - continue - if has_allowed_child(child): - child.unwrap() - else: - child.decompose() - bs_parsed = str(good_soup) - logger.log_debug( - "called sanitizer! using this as args: \n" + - str(bs_parsed) + - str(config.get("tags")) + "\n" + - str(config.get("protocols")) + "\n" + - str(config.get("strip")) + raise ValueError(f"Unknown mode: {mode}") + + soup = BeautifulSoup(html, "html.parser") + + original_head = soup.find("head") + original_body = soup.find("body") + + head_html = str(original_head) if original_head else "" + body_html = str(original_body) if original_body else html + + head_tags = {"title", "meta", "link", "base", "style"} + head_allowed = set(config.get("tags", [])) & head_tags + + cleaned_head_content = bleach.clean( + head_html, + tags=head_allowed, + attributes=config.get("attributes"), + protocols=config.get("protocols"), + strip=config.get("strip"), ) - cleaned = bleach.clean( - bs_parsed, + + cleaned_body_content = bleach.clean( + body_html, tags=config.get("tags"), attributes=config.get("attributes"), protocols=config.get("protocols"), strip=config.get("strip"), ) - return _wrap_document(cleaned) + + pretty_body = BeautifulSoup(cleaned_body_content, "html.parser").prettify() + pretty_head = BeautifulSoup(cleaned_head_content, "html.parser").prettify() + + return f""" + + + +{pretty_head} +{pretty_body} + +""" def _wrap_document(content: str) -> str: + logger.log_debug("Content:" + content) soup = BeautifulSoup(content, "html.parser") existing_head = soup.find("head") + logger.log_debug("Existing Head?:" + str(existing_head)) existing_body = soup.find("body") + logger.log_debug("Existing Body?:" + str(existing_body)) head_content = str(existing_head) if existing_head else "" body_content = str(existing_body) if existing_body else f"{content}" return f""" - \n - \n - - {head_content}\n - {body_content}\n - + + + +{head_content} +{body_content} + """