vom 11. Juni bis 19. Juli 2026 in Kanada, Mexiko und den USA
+
+vom 11. Juni bis 19. Juli 2026 in Kanada, Mexiko und den USA
+This is a test
+ + +``` + diff --git a/src/configuration.py b/src/configuration.py new file mode 100644 index 0000000..5533f46 --- /dev/null +++ b/src/configuration.py @@ -0,0 +1,63 @@ +# +# configuration.py +# AGPL-3.0 - sanigi-html +# json config loader +# + +from pathlib import Path +import json + +def load_json_cfg(path: Path): + if path == None: + return {} + + if not path.exists(): + raise FileNotFoundError(path) + + with path.open("r", encoding="utf-8") as f: + raw = json.load(f) + + def to_bool(variable, default=False): + """ + basically just like a match case, but functionalized + so we can call it more efficiently. + """ + if isinstance(variable, bool): return variable + if isinstance(variable, str): + return variable.lower() in ("1", "true", "yes", "no") + return default + + def to_list(variable): + if variable is None: + return [] + if isinstance(variable, str): + return [ + # very cheeky logic here! + # this is a oneliner to split at , + s.strip() for s in variable.split(",") if s.strip() + ] + + if isinstance(variable, list): + return [ + # this is another example of cheeky logic! + # its fun to figure out yourself, trust me + str(x).strip() for x in variable if str(x).strip() + ] + + return [] + + configuration = { + "allow_relative": to_bool(raw.get("ALLOW_RELATIVE_URLS"), True), + "allowed_domains_default": to_list(raw.get("ALLOWED_DOMAINS")), + "allowed_domains_img": to_list(raw.get("ALLOWED_DOMAINS_IMG")), + "allowed_domains_link": to_list(raw.get("ALLOWED_DOMAINS_LINK")), + "STRICT_overrides": raw.get("STRICT"), + "LOOSE_overrides": raw.get("LOOSE"), + } + + if not configuration["allowed_domains_img"]: + configuration["allowed_domains_img"] = configuration["allowed_domains_default"] + if not configuration["allowed_domains_link"]: + configuration["allowed_domains_link"] = configuration["allowed_domains_default"] + + return configuration \ No newline at end of file diff --git a/src/example/config_example.json b/src/example/config_example.json new file mode 100644 index 0000000..f825d40 --- /dev/null +++ b/src/example/config_example.json @@ -0,0 +1,18 @@ +{ + "ALLOW_RELATIVE_URLS": true, + "ALLOWED_DOMAINS": "example.com, static.example.com", + "ALLOWED_DOMAINS_IMG": "images.example.com", + "ALLOWED_DOMAINS_LINK": "example.com, partner.example.org", + + "STRICT": { + "tags": ["p","b","i","ul","ol","li","br","h1","h2"], + "protocols": [], + "strip": true + }, + + "LOOSE": { + "tags": ["p","b","i","u","em","strong","ul","ol","li","br","hr","span","div","a","img","h1","h2","h3"], + "protocols": ["http","https"], + "strip": true + } +} \ No newline at end of file diff --git a/src/main.py b/src/main.py index da5a536..5b85ebe 100644 --- a/src/main.py +++ b/src/main.py @@ -7,24 +7,43 @@ import sys import argparse +from pathlib import Path + +from configuration import load_json_cfg +from profiles import build_profiles_from_cfg from sanitizer import sanitize def entry(): parser = argparse.ArgumentParser(description="saningi-html sanitizer") + parser.add_argument( + "--mode", + choices=["strict","loose","passthrough"], + default="strict", + help="strictness" + ) parser.add_argument( - "--mode", - choices=["strict","loose","passthrough"], - default="strict", - help="Strictness of Tags or HTML" + "--config", + type=Path, + help="path to config json" ) args = parser.parse_args() + cfg = load_json_cfg(args.config) + profile_dict = build_profiles_from_cfg(cfg) + + # cause i dont hate myself + PROFILES = { + "strict": profile_dict["STRICT"], + "loose": profile_dict["LOOSE"], + } + + # read from stdin and putout to stdout raw_html = sys.stdin.read() - clean_html = sanitize(raw_html, args.mode) + clean_html = sanitize(raw_html, args.mode, PROFILES) print(clean_html) if __name__ == "__main__": - entry() \ No newline at end of file + entry() diff --git a/src/profiles.py b/src/profiles.py index 64aceb8..01d2576 100644 --- a/src/profiles.py +++ b/src/profiles.py @@ -2,43 +2,36 @@ # src/profiles.py # AGPL-3 - saningi-html # -# presets for sanitization of HTML +# presets for sanitization of HTML; +# and newly the building of configurations # import os -from dotenv import load_dotenv from urllib.parse import urlparse +from typing import Dict, Any, Optional -# loadenv init -load_dotenv() - -def is_safe_url(url : str, allowed_domain=None, allow_relative : bool = True) -> bool: - """ - Determine wether or not a URL is safe or not. - Safe URL's are only local so /this/url - Unsafe URL's are external ones like https://pornhub.com - """ +def is_safe_url(url: str, allowed_domain=None, allow_relative: bool = True) -> bool: if not url: return False if allow_relative: if url.startswith(("#", "/", "./", "../")): return True - + if url.startswith("//"): if not allowed_domain: return False - + parsed = urlparse("http:" + url) domain = parsed.netloc or parsed.path.split("/")[0] return domain in allowed_domain - + parsed = urlparse(url) if parsed.scheme in ("http", "https"): if not allowed_domain: return False return parsed.netloc in allowed_domain - + return False def create_url_filter(allowed_attrs, allowed_domains=None, allow_relative=True): @@ -52,62 +45,105 @@ def create_url_filter(allowed_attrs, allowed_domains=None, allow_relative=True): return value return filter_func -def get_domains(variable): - value = os.getenv(variable, "") - return [ - # return a list of domains, and split by comma - d.strip() for d in value.split(",") if d.strip() - ] +def build_profiles_from_cfg(cfg: Optional[Dict[str, Any]] = None) -> Dict[str, Dict]: + """ + This method esentially builds any configuration from a json config + """ + def get_domains_from_env(variable): + value = os.getenv(variable, "") + return [d.strip() for d in value.split(",") if d.strip()] -# Setup any .env variables for python to use + if cfg is None: + allow_relative = os.getenv("ALLOW_RELATIVE_URLS", "true").lower() == "true" + allowed_domains_default = get_domains_from_env("ALLOWED_DOMAINS") + allowed_domains_img = get_domains_from_env("ALLOWED_DOMAINS_IMG") or allowed_domains_default + allowed_domains_link = get_domains_from_env("ALLOWED_DOMAINS_LINK") or allowed_domains_default + strict_overrides = None + loose_overrides = None + else: + allow_relative = cfg.get("allow_relative", True) + allowed_domains_default = cfg.get("allowed_domains_default", []) + allowed_domains_img = cfg.get("allowed_domains_img", allowed_domains_default) + allowed_domains_link = cfg.get("allowed_domains_link", allowed_domains_default) + strict_overrides = cfg.get("STRICT_overrides") + loose_overrides = cfg.get("LOOSE_overrides") -allow_relative = os.getenv("ALLOW_RELATIVE_URLS", "true").lower() == "true" + # like daddy :3 + STRICT = { + "tags": [ + "html", "body", "header", "head", "title", "meta", "link", "body", "main", "footer", "section", "article", "nav", + "aside", "h1", "h2", "h3", "h4", "h5", "h6", + "p", "pre", "code", "blockquote", + "ul", "ol", "li", "dl", "dt", "dd", + "br", "hr", + "a", "img", "figure", "figcaption", + "table", "thead", "tbody", "tfoot", "tr", "th", "td", + "strong", "em", "b", "i", "u", "small", + "details", "summary", "time" + ], + "attributes": { + "a": create_url_filter(["href", "title", "rel", "target"], allowed_domains=allowed_domains_link, allow_relative=allow_relative), + "img": create_url_filter(["src", "alt", "width", "height", "loading"], allowed_domains=allowed_domains_img, allow_relative=allow_relative), + "figure": ["class"], + "figcaption": ["class"], + "table": ["summary", "class"], + "th": ["scope", "colspan", "rowspan"], + "td": ["colspan", "rowspan"], + "link": ["href", "rel"], + "time": ["datetime"], + "meta": ["charset", "name", "content"], + "html": ["lang"], + "*": ["class", "id"] # limit + }, + "protocols": ["http", "https", "mailto"], + "strip": True, + } -allowed_domains_default = get_domains("ALLOWED_DOMAINS") -allowed_domains_img = get_domains("ALLOWED_DOMAINS_IMG") or allowed_domains_default -allowed_domains_link = get_domains("ALLOWED_DOMAINS_LINK") or allowed_domains_default + # like my ass :3 + LOOSE = { + "tags": [ + "html", "body", "header", "head", "title", "meta", "link", "body", "main", "footer", "section", "article", "nav", + "aside", "h1", "h2", "h3", "h4", "h5", "h6", + "p", "pre", "code", "blockquote", "q", + "ul", "ol", "li", "dl", "dt", "dd", + "br", "hr", "span", "div", + "a", "img", "figure", "figcaption", + "table", "thead", "tbody", "tfoot", "tr", "th", "td", + "strong", "em", "b", "i", "u", "small", + "details", "summary", "time", + "video", "audio", "source", "picture", "source", + "caption" + ], + "attributes": { + "a": ["href", "title", "rel", "target"], + "img": ["src", "alt", "width", "height", "loading", "decoding", "style"], + "figure": ["class", "style"], + "figcaption": ["class", "style"], + "table": ["summary", "class", "style"], + "th": ["scope", "colspan", "rowspan", "style"], + "td": ["colspan", "rowspan", "style"], + "time": ["datetime"], + "video": ["src", "controls", "width", "height", "poster"], + "audio": ["src", "controls"], + "source": ["src", "type"], + "div": ["class", "id", "style"], + "span": ["class", "id", "style"], + "pre": ["class"], + "code": ["class"], + "link" : ["href", "rel"], + "meta": ["charset", "name", "content"], + "html": ["lang"], + "*": ["class", "id", "style"] + }, + "protocols": ["http", "https", "mailto", "data"], + "strip": True, + } -STRICT = { - "tags" : [ - "p", "b", "i", "u", "em", "strong", - "ul", "ol", "li", - "br", "hr", - "h1", "h2", "h3", "h4", "h5", "h6" - ], - "attributes": { - "a": create_url_filter( - ["href"], - allowed_domains=allowed_domains_link, - allow_relative=allow_relative - ), - "img": create_url_filter( - ["src", "alt"], - allowed_domains=allowed_domains_img, - allow_relative=allow_relative - ), - "link": create_url_filter( - ["href"], - allowed_domains=allowed_domains_default, - allow_relative=allow_relative - ), - }, - "protocols": [], - "strip": True, -} -# like my ass :3 -LOOSE = { - "tags": [ - "p", "b", "i", "u", "em", "strong", - "ul", "ol", "li", - "br", "hr", "span", "div", - "a", "img", - "h1", "h2", "h3", "h4", "h5", "h6" - ], - "attributes": { - "a": ["href", "title"], - "img": {"src","alt","style","loading"} - }, - "protocols": ["http", "https"], - "strip": True, -} \ No newline at end of file + # overrides + if isinstance(strict_overrides, dict): + STRICT.update(strict_overrides) + if isinstance(loose_overrides, dict): + LOOSE.update(loose_overrides) + + return {"STRICT": STRICT, "LOOSE": LOOSE} diff --git a/src/sanitizer.py b/src/sanitizer.py index 1a9d95a..6f982bf 100644 --- a/src/sanitizer.py +++ b/src/sanitizer.py @@ -3,20 +3,15 @@ # AGPL-3.0 - saningi-html # - import bleach -from bs4 import BeautifulSoup, Tag -from profiles import STRICT, LOOSE -from log.Logger import * +import datetime +from bs4 import BeautifulSoup +from log.Logger import Logger + logger = Logger() -PROFILES = { - "strict": STRICT, - "loose": LOOSE, -} - -def sanitize(html : str, mode : str) -> str: +def sanitize(html: str, mode: str, profiles: dict) -> str: """ Arguments: html : str -> input html @@ -26,50 +21,84 @@ def sanitize(html : str, mode : str) -> str: This function takes in HTML sanitzies it with bleach and returns clean HTML """ - # this should be used with caution! it is for trusted - # individuals only if mode == "passthrough": logger.log_warning("You are about to not care about sanitizing ANY HTML. You are in passthrough mode.") - return html + return _wrap_document(html) - config = PROFILES.get(mode) + config = profiles.get(mode) if not config: - raise ValueError(f"Unknown mode! : {mode}") - - allowed = set(config.get("tags", [])) - good_soup = BeautifulSoup(html, "html.parser") + raise ValueError(f"Unknown mode: {mode}") - def has_allowed_child(node: Tag) -> bool: - # check if shit is allowed - for child in node.descendants: - if isinstance(child, Tag) and child.name.lower() in allowed: - return True - return False + soup = BeautifulSoup(html, "html.parser") - - for child in list(good_soup.find_all()): - name = child.name.lower() - if name in allowed: - continue - if has_allowed_child(child): - child.unwrap() - else: - child.decompose() - - bs_parsed = str(good_soup) - - logger.log_debug( - "called sanitizer! using this as args: \n" + - str(bs_parsed) + - str(config["tags"]) + "\n" + - str(config["protocols"]) + "\n" + - str(config["strip"]) + original_head = soup.find("head") + original_body = soup.find("body") + + head_html = str(original_head) if original_head else "" + body_html = str(original_body) if original_body else html + + head_tags = {"title", "meta", "link", "base", "style"} + head_allowed = set(config.get("tags", [])) & head_tags + + cleaned_head_content = bleach.clean( + head_html, + tags=head_allowed, + attributes=config.get("attributes"), + protocols=config.get("protocols"), + strip=config.get("strip"), ) - return bleach.clean( - bs_parsed, - tags=config["tags"], - attributes=config["attributes"], - protocols=config["protocols"], - strip=config["strip"], - ) \ No newline at end of file + cleaned_body_content = bleach.clean( + body_html, + tags=config.get("tags"), + attributes=config.get("attributes"), + protocols=config.get("protocols"), + strip=config.get("strip"), + ) + + pretty_body = BeautifulSoup(cleaned_body_content, "html.parser").prettify() + pretty_head = BeautifulSoup(cleaned_head_content, "html.parser").prettify() + + logger.log_debug( + # very ugly string concationation for logging + "parser args: " + str( + "Tags: " + str(config.get("tags")) + "\n" + + "Attributes: " + str(config.get("attributes")) + "\n" + + "Protocols: " + str(config.get("protocols")) + "\n" + + "Strip: " + str(config.get("strip")) + "\n" + ) + + "input html: " + str(html) + + "clean head: " + str(pretty_head) + + "clean body: " + str(pretty_body) + ) + + return f""" + + + +{pretty_head} +{pretty_body} + +""" + + +def _wrap_document(content: str) -> str: + logger.log_debug("Content:" + content) + soup = BeautifulSoup(content, "html.parser") + + existing_head = soup.find("head") + logger.log_debug("Existing Head?:" + str(existing_head)) + existing_body = soup.find("body") + logger.log_debug("Existing Body?:" + str(existing_body)) + + head_content = str(existing_head) if existing_head else "" + body_content = str(existing_body) if existing_body else f"{content}" + + return f""" + + + +{head_content} +{body_content} + + """