import codecs import json import re import string from urllib.parse import quote ESCAPE_SEQUENCE_RE = re.compile( r""" ( \\U........ # 8-digit hex escapes | \\u.... # 4-digit hex escapes | \\x.. # 2-digit hex escapes | \\[0-7]{1,3} # Octal escapes | \\N\{[^}]+\} # Unicode characters by name | \\[\\'"abfnrtv] # Single-character escapes )""", re.UNICODE | re.VERBOSE, ) def decode_escapes(s: str) -> str: def decode_match(match: re.Match) -> str: return codecs.decode(match.group(0), "unicode-escape") return ESCAPE_SEQUENCE_RE.sub(decode_match, s) def make_url_compatible(s: str) -> str: s_with_underscores = s.replace(" ", "_") return quote(s_with_underscores, safe="") def has_unescaped_quote(s: str) -> bool: pattern = r'(? str: return re.sub(r"(? str: return re.sub(r"\s", " ", s) def extract_embedded_json(s: str) -> dict: first_brace_index = s.find("{") last_brace_index = s.rfind("}") if first_brace_index == -1 or last_brace_index == -1: raise ValueError("No valid json found") return json.loads(s[first_brace_index : last_brace_index + 1], strict=False) def clean_up_code_blocks(model_out_raw: str) -> str: return model_out_raw.strip().strip("```").strip().replace("\\xa0", "") def clean_model_quote(quote: str, trim_length: int) -> str: quote_clean = quote.strip() if quote_clean[0] == '"': quote_clean = quote_clean[1:] if quote_clean[-1] == '"': quote_clean = quote_clean[:-1] if trim_length > 0: quote_clean = quote_clean[:trim_length] return quote_clean def shared_precompare_cleanup(text: str) -> str: """LLMs models sometime restructure whitespaces or edits special characters to fit a more likely distribution of characters found in its training data, but this hurts exact quote matching """ text = text.lower() # \s: matches any whitespace character (spaces, tabs, newlines, etc.) # |: acts as an OR. # \*: matches the asterisk character. # \\": matches the \" sequence. # [.,:`"#-]: matches any character inside the square brackets. text = re.sub(r'\s|\*|\\"|[.,:`"#-]', "", text) return text def is_valid_email(text: str) -> bool: """Can use a library instead if more detailed checks are needed""" regex = r"^[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$" if re.match(regex, text): return True else: return False def count_punctuation(text: str) -> int: return sum(1 for char in text if char in string.punctuation)