diff --git a/TODO.md b/TODO.md index 16d1880..724c656 100644 --- a/TODO.md +++ b/TODO.md @@ -7,6 +7,7 @@ - loose - passthrough - [x] profiles in a seperate .py file +- Ok you see, I am too lazy to write the stuff which I completed here, so i just italizized it. ## To implement: @@ -27,15 +28,15 @@ NOTE: That things which are in italic text are already implemented! --- ### Expland strict mode: -- [ ] Allow these: - - html, head, body - - title - - meta (with restrictions) - - link (for css only) - - table, tr, td, th - - any of crucuial html tags -- [ ] restrict metas (allow): - - allow only charset, name, content +- *[x] Allow these:* + - *html, head, body* + - *title* + - *meta (with restrictions)* + - *link (for css only)* + - *table, tr, td, th* + - *any of crucuial html tags* +- [x] *restrict metas (allow):* + - *allow only charset, name, content* - [ ] restrict metas (dont allow): - base - meta http-equiv @@ -79,4 +80,124 @@ Lets maybe use bs4 for this, its fitting cause it can decompose HTML. Fixed! ### Around 30 minutes later: -This issue is no longer relevant, it is fixed! using bs4. \ No newline at end of file +This issue is no longer relevant, it is fixed! using bs4. + +## Development brach additions (overview of functionallity): + +As a example of the functionallity we use a command to call our sanizizer using the strict mode and we specifiy a configuration living in examples/config_example.json: + +```bash +cat ~/Documents/INF6B/html/wm/startseite_wm.html | python src/main.py --mode strict > out.html +``` + +We output to a file called out.html, this is what the stdout of our programm looks like. if we grad stderr we get any debugging information aswell as errors / warnings / info + +### command output: + +```bash +(venv) sebastian@kubuntu:~/Documents/sanigi-html$ cat ~/Documents/INF6B/html/wm/startseite_wm.html | python src/main.py --mode strict > out.html +[ DEBUG@2026-04-20 12:32:50 ]: called sanitizer! using this as args: + + + + +

Die Fußball-Weltmeisterschaft 2026

+

vom 11. Juni bis 19. Juli 2026 in Kanada, Mexiko und den USA

+ + + +['html', 'body', 'header', 'main', 'footer', 'section', 'article', 'nav', 'aside', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'pre', 'code', 'blockquote', 'ul', 'ol', 'li', 'dl', 'dt', 'dd', 'br', 'hr', 'a', 'img', 'figure', 'figcaption', 'table', 'thead', 'tbody', 'tfoot', 'tr', 'th', 'td', 'strong', 'em', 'b', 'i', 'u', 'small', 'details', 'summary', 'time'] +['http', 'https', 'mailto'] +True +``` + +This is what the programm outputted to the stdout file (inserted into out.html): + +```html + + + + + + + + + + + + +

Die Fußball-Weltmeisterschaft 2026

+

vom 11. Juni bis 19. Juli 2026 in Kanada, Mexiko und den USA

+ + + + + + + + +``` diff --git a/out.html b/out.html new file mode 100644 index 0000000..309d3b9 --- /dev/null +++ b/out.html @@ -0,0 +1,53 @@ + + + + + + + + + + + + +

Die Fußball-Weltmeisterschaft 2026

+

vom 11. Juni bis 19. Juli 2026 in Kanada, Mexiko und den USA

+ + + + + + + diff --git a/src/configuration.py b/src/configuration.py new file mode 100644 index 0000000..5533f46 --- /dev/null +++ b/src/configuration.py @@ -0,0 +1,63 @@ +# +# configuration.py +# AGPL-3.0 - sanigi-html +# json config loader +# + +from pathlib import Path +import json + +def load_json_cfg(path: Path): + if path == None: + return {} + + if not path.exists(): + raise FileNotFoundError(path) + + with path.open("r", encoding="utf-8") as f: + raw = json.load(f) + + def to_bool(variable, default=False): + """ + basically just like a match case, but functionalized + so we can call it more efficiently. + """ + if isinstance(variable, bool): return variable + if isinstance(variable, str): + return variable.lower() in ("1", "true", "yes", "no") + return default + + def to_list(variable): + if variable is None: + return [] + if isinstance(variable, str): + return [ + # very cheeky logic here! + # this is a oneliner to split at , + s.strip() for s in variable.split(",") if s.strip() + ] + + if isinstance(variable, list): + return [ + # this is another example of cheeky logic! + # its fun to figure out yourself, trust me + str(x).strip() for x in variable if str(x).strip() + ] + + return [] + + configuration = { + "allow_relative": to_bool(raw.get("ALLOW_RELATIVE_URLS"), True), + "allowed_domains_default": to_list(raw.get("ALLOWED_DOMAINS")), + "allowed_domains_img": to_list(raw.get("ALLOWED_DOMAINS_IMG")), + "allowed_domains_link": to_list(raw.get("ALLOWED_DOMAINS_LINK")), + "STRICT_overrides": raw.get("STRICT"), + "LOOSE_overrides": raw.get("LOOSE"), + } + + if not configuration["allowed_domains_img"]: + configuration["allowed_domains_img"] = configuration["allowed_domains_default"] + if not configuration["allowed_domains_link"]: + configuration["allowed_domains_link"] = configuration["allowed_domains_default"] + + return configuration \ No newline at end of file diff --git a/src/example/config_example.json b/src/example/config_example.json new file mode 100644 index 0000000..f825d40 --- /dev/null +++ b/src/example/config_example.json @@ -0,0 +1,18 @@ +{ + "ALLOW_RELATIVE_URLS": true, + "ALLOWED_DOMAINS": "example.com, static.example.com", + "ALLOWED_DOMAINS_IMG": "images.example.com", + "ALLOWED_DOMAINS_LINK": "example.com, partner.example.org", + + "STRICT": { + "tags": ["p","b","i","ul","ol","li","br","h1","h2"], + "protocols": [], + "strip": true + }, + + "LOOSE": { + "tags": ["p","b","i","u","em","strong","ul","ol","li","br","hr","span","div","a","img","h1","h2","h3"], + "protocols": ["http","https"], + "strip": true + } +} \ No newline at end of file diff --git a/src/main.py b/src/main.py index da5a536..5b85ebe 100644 --- a/src/main.py +++ b/src/main.py @@ -7,24 +7,43 @@ import sys import argparse +from pathlib import Path + +from configuration import load_json_cfg +from profiles import build_profiles_from_cfg from sanitizer import sanitize def entry(): parser = argparse.ArgumentParser(description="saningi-html sanitizer") + parser.add_argument( + "--mode", + choices=["strict","loose","passthrough"], + default="strict", + help="strictness" + ) parser.add_argument( - "--mode", - choices=["strict","loose","passthrough"], - default="strict", - help="Strictness of Tags or HTML" + "--config", + type=Path, + help="path to config json" ) args = parser.parse_args() + cfg = load_json_cfg(args.config) + profile_dict = build_profiles_from_cfg(cfg) + + # cause i dont hate myself + PROFILES = { + "strict": profile_dict["STRICT"], + "loose": profile_dict["LOOSE"], + } + + # read from stdin and putout to stdout raw_html = sys.stdin.read() - clean_html = sanitize(raw_html, args.mode) + clean_html = sanitize(raw_html, args.mode, PROFILES) print(clean_html) if __name__ == "__main__": - entry() \ No newline at end of file + entry() diff --git a/src/profiles.py b/src/profiles.py index 64aceb8..7eb6ac9 100644 --- a/src/profiles.py +++ b/src/profiles.py @@ -2,43 +2,36 @@ # src/profiles.py # AGPL-3 - saningi-html # -# presets for sanitization of HTML +# presets for sanitization of HTML; +# and newly the building of configurations # import os -from dotenv import load_dotenv from urllib.parse import urlparse +from typing import Dict, Any, Optional -# loadenv init -load_dotenv() - -def is_safe_url(url : str, allowed_domain=None, allow_relative : bool = True) -> bool: - """ - Determine wether or not a URL is safe or not. - Safe URL's are only local so /this/url - Unsafe URL's are external ones like https://pornhub.com - """ +def is_safe_url(url: str, allowed_domain=None, allow_relative: bool = True) -> bool: if not url: return False if allow_relative: if url.startswith(("#", "/", "./", "../")): return True - + if url.startswith("//"): if not allowed_domain: return False - + parsed = urlparse("http:" + url) domain = parsed.netloc or parsed.path.split("/")[0] return domain in allowed_domain - + parsed = urlparse(url) if parsed.scheme in ("http", "https"): if not allowed_domain: return False return parsed.netloc in allowed_domain - + return False def create_url_filter(allowed_attrs, allowed_domains=None, allow_relative=True): @@ -52,62 +45,99 @@ def create_url_filter(allowed_attrs, allowed_domains=None, allow_relative=True): return value return filter_func -def get_domains(variable): - value = os.getenv(variable, "") - return [ - # return a list of domains, and split by comma - d.strip() for d in value.split(",") if d.strip() - ] +def build_profiles_from_cfg(cfg: Optional[Dict[str, Any]] = None) -> Dict[str, Dict]: + """ + This method esentially builds any configuration from a json config + """ + def get_domains_from_env(variable): + value = os.getenv(variable, "") + return [d.strip() for d in value.split(",") if d.strip()] -# Setup any .env variables for python to use + if cfg is None: + allow_relative = os.getenv("ALLOW_RELATIVE_URLS", "true").lower() == "true" + allowed_domains_default = get_domains_from_env("ALLOWED_DOMAINS") + allowed_domains_img = get_domains_from_env("ALLOWED_DOMAINS_IMG") or allowed_domains_default + allowed_domains_link = get_domains_from_env("ALLOWED_DOMAINS_LINK") or allowed_domains_default + strict_overrides = None + loose_overrides = None + else: + allow_relative = cfg.get("allow_relative", True) + allowed_domains_default = cfg.get("allowed_domains_default", []) + allowed_domains_img = cfg.get("allowed_domains_img", allowed_domains_default) + allowed_domains_link = cfg.get("allowed_domains_link", allowed_domains_default) + strict_overrides = cfg.get("STRICT_overrides") + loose_overrides = cfg.get("LOOSE_overrides") -allow_relative = os.getenv("ALLOW_RELATIVE_URLS", "true").lower() == "true" + # like daddy :3 + STRICT = { + "tags": [ + "html", "body", "header", "main", "footer", "section", "article", "nav", + "aside", "h1", "h2", "h3", "h4", "h5", "h6", + "p", "pre", "code", "blockquote", + "ul", "ol", "li", "dl", "dt", "dd", + "br", "hr", + "a", "img", "figure", "figcaption", + "table", "thead", "tbody", "tfoot", "tr", "th", "td", + "strong", "em", "b", "i", "u", "small", + "details", "summary", "time" + ], + "attributes": { + "a": create_url_filter(["href", "title", "rel", "target"], allowed_domains=allowed_domains_link, allow_relative=allow_relative), + "img": create_url_filter(["src", "alt", "width", "height", "loading"], allowed_domains=allowed_domains_img, allow_relative=allow_relative), + "figure": ["class"], + "figcaption": ["class"], + "table": ["summary", "class"], + "th": ["scope", "colspan", "rowspan"], + "td": ["colspan", "rowspan"], + "time": ["datetime"], + "*": ["class", "id"] # limit + }, + "protocols": ["http", "https", "mailto"], + "strip": True, + } -allowed_domains_default = get_domains("ALLOWED_DOMAINS") -allowed_domains_img = get_domains("ALLOWED_DOMAINS_IMG") or allowed_domains_default -allowed_domains_link = get_domains("ALLOWED_DOMAINS_LINK") or allowed_domains_default + # like my ass :3 + LOOSE = { + "tags": [ + "html", "body", "header", "main", "footer", "section", "article", "nav", + "aside", "h1", "h2", "h3", "h4", "h5", "h6", + "p", "pre", "code", "blockquote", "q", + "ul", "ol", "li", "dl", "dt", "dd", + "br", "hr", "span", "div", + "a", "img", "figure", "figcaption", + "table", "thead", "tbody", "tfoot", "tr", "th", "td", + "strong", "em", "b", "i", "u", "small", + "details", "summary", "time", + "video", "audio", "source", "picture", "source", + "caption" + ], + "attributes": { + "a": ["href", "title", "rel", "target"], + "img": ["src", "alt", "width", "height", "loading", "decoding", "style"], + "figure": ["class", "style"], + "figcaption": ["class", "style"], + "table": ["summary", "class", "style"], + "th": ["scope", "colspan", "rowspan", "style"], + "td": ["colspan", "rowspan", "style"], + "time": ["datetime"], + "video": ["src", "controls", "width", "height", "poster"], + "audio": ["src", "controls"], + "source": ["src", "type"], + "div": ["class", "id", "style"], + "span": ["class", "id", "style"], + "pre": ["class"], + "code": ["class"], + "*": ["class", "id", "style"] + }, + "protocols": ["http", "https", "mailto", "data"], + "strip": True, + } -STRICT = { - "tags" : [ - "p", "b", "i", "u", "em", "strong", - "ul", "ol", "li", - "br", "hr", - "h1", "h2", "h3", "h4", "h5", "h6" - ], - "attributes": { - "a": create_url_filter( - ["href"], - allowed_domains=allowed_domains_link, - allow_relative=allow_relative - ), - "img": create_url_filter( - ["src", "alt"], - allowed_domains=allowed_domains_img, - allow_relative=allow_relative - ), - "link": create_url_filter( - ["href"], - allowed_domains=allowed_domains_default, - allow_relative=allow_relative - ), - }, - "protocols": [], - "strip": True, -} -# like my ass :3 -LOOSE = { - "tags": [ - "p", "b", "i", "u", "em", "strong", - "ul", "ol", "li", - "br", "hr", "span", "div", - "a", "img", - "h1", "h2", "h3", "h4", "h5", "h6" - ], - "attributes": { - "a": ["href", "title"], - "img": {"src","alt","style","loading"} - }, - "protocols": ["http", "https"], - "strip": True, -} \ No newline at end of file + # overrides + if isinstance(strict_overrides, dict): + STRICT.update(strict_overrides) + if isinstance(loose_overrides, dict): + LOOSE.update(loose_overrides) + + return {"STRICT": STRICT, "LOOSE": LOOSE} diff --git a/src/sanitizer.py b/src/sanitizer.py index 1a9d95a..d2b9a33 100644 --- a/src/sanitizer.py +++ b/src/sanitizer.py @@ -3,50 +3,27 @@ # AGPL-3.0 - saningi-html # - import bleach from bs4 import BeautifulSoup, Tag -from profiles import STRICT, LOOSE +import datetime from log.Logger import * + logger = Logger() -PROFILES = { - "strict": STRICT, - "loose": LOOSE, -} - - -def sanitize(html : str, mode : str) -> str: - """ - Arguments: - html : str -> input html - mode : str -> the defined mode which is from profiles.py ; or PROFILES variable - - Functionality: - This function takes in HTML sanitzies it with bleach and returns clean HTML - """ - - # this should be used with caution! it is for trusted - # individuals only +def sanitize(html: str, mode: str, profiles: dict) -> str: if mode == "passthrough": logger.log_warning("You are about to not care about sanitizing ANY HTML. You are in passthrough mode.") - return html - - config = PROFILES.get(mode) + return _wrap_document(html) # still wrap passthrough output + config = profiles.get(mode) if not config: raise ValueError(f"Unknown mode! : {mode}") - allowed = set(config.get("tags", [])) good_soup = BeautifulSoup(html, "html.parser") - def has_allowed_child(node: Tag) -> bool: - # check if shit is allowed for child in node.descendants: if isinstance(child, Tag) and child.name.lower() in allowed: return True return False - - for child in list(good_soup.find_all()): name = child.name.lower() if name in allowed: @@ -55,21 +32,38 @@ def sanitize(html : str, mode : str) -> str: child.unwrap() else: child.decompose() - bs_parsed = str(good_soup) - logger.log_debug( - "called sanitizer! using this as args: \n" + + "called sanitizer! using this as args: \n" + str(bs_parsed) + - str(config["tags"]) + "\n" + - str(config["protocols"]) + "\n" + - str(config["strip"]) + str(config.get("tags")) + "\n" + + str(config.get("protocols")) + "\n" + + str(config.get("strip")) ) - - return bleach.clean( + cleaned = bleach.clean( bs_parsed, - tags=config["tags"], - attributes=config["attributes"], - protocols=config["protocols"], - strip=config["strip"], - ) \ No newline at end of file + tags=config.get("tags"), + attributes=config.get("attributes"), + protocols=config.get("protocols"), + strip=config.get("strip"), + ) + return _wrap_document(cleaned) + + +def _wrap_document(content: str) -> str: + soup = BeautifulSoup(content, "html.parser") + + existing_head = soup.find("head") + existing_body = soup.find("body") + + head_content = str(existing_head) if existing_head else "" + body_content = str(existing_body) if existing_body else f"{content}" + + return f""" + \n + \n + + {head_content}\n + {body_content}\n + + """