""" This module will parse the JSON file following the BNF definition: ::= ::= | | ; Where: ; is a valid real number expressed in one of a number of given formats ; is a string of valid characters enclosed in quotes ; is one of the literal strings 'true', 'false', or 'null' (unquoted) ::= | ::= '[' [ *(', ' ) ] ']' ; A sequence of JSON values separated by commas ::= '{' [ *(', ' ) ] '}' ; A sequence of 'members' ::= ': ' ; A pair consisting of a name, and a JSON value If something is wrong (a missing parentheses or quotes for example) it will use a few simple heuristics to fix the JSON string: - Add the missing parentheses if the parser believes that the array or object should be closed - Quote strings or add missing single quotes - Adjust whitespaces and remove line breaks All supported use cases are in the unit tests """ import argparse import json import sys from typing import Any, Literal, TextIO, overload from .json_parser import JSONParser from .utils.constants import JSONReturnType @overload def repair_json( json_str: str = "", return_objects: Literal[False] = False, skip_json_loads: bool = False, logging: bool = False, json_fd: TextIO | None = None, chunk_length: int = 0, stream_stable: bool = False, strict: bool = False, **json_dumps_args: Any, ) -> str: ... @overload def repair_json( json_str: str = "", return_objects: Literal[True] = True, skip_json_loads: bool = False, logging: bool = False, json_fd: TextIO | None = None, chunk_length: int = 0, stream_stable: bool = False, strict: bool = False, **json_dumps_args: Any, ) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]: ... def repair_json( json_str: str = "", return_objects: bool = False, skip_json_loads: bool = False, logging: bool = False, json_fd: TextIO | None = None, chunk_length: int = 0, stream_stable: bool = False, strict: bool = False, **json_dumps_args: Any, ) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]: """ Given a json formatted string, it will try to decode it and, if it fails, it will try to fix it. Args: json_str (str, optional): The JSON string to repair. Defaults to an empty string. return_objects (bool, optional): If True, return the decoded data structure. Defaults to False. skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False. logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False. When no repairs were required, the repair log will be an empty list. json_fd (Optional[TextIO], optional): File descriptor for JSON input. Do not use! Use `from_file` or `load` instead. Defaults to None. ensure_ascii (bool, optional): Set to False to avoid converting non-latin characters to ascii (for example when using chinese characters). Defaults to True. Ignored if `skip_json_loads` is True. chunk_length (int, optional): Size in bytes of the file chunks to read at once. Ignored if `json_fd` is None. Do not use! Use `from_file` or `load` instead. Defaults to 1MB. stream_stable (bool, optional): When the json to be repaired is the accumulation of streaming json at a certain moment.If this parameter to True will keep the repair results stable. strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them. Returns: Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON or a tuple with the repaired JSON and repair log when logging is True. """ parser = JSONParser(json_str, json_fd, logging, chunk_length, stream_stable, strict) if skip_json_loads: parsed_json = parser.parse() else: try: parsed_json = json.load(json_fd) if json_fd else json.loads(json_str) except json.JSONDecodeError: parsed_json = parser.parse() # It's useful to return the actual object instead of the json string, # it allows this lib to be a replacement of the json library if return_objects or logging: # If logging is True, the user should expect a tuple. # If json.load(s) worked, the repair log list is empty if logging and not isinstance(parsed_json, tuple): return parsed_json, [] return parsed_json # Avoid returning only a pair of quotes if it's an empty string elif parsed_json == "": return "" return json.dumps(parsed_json, **json_dumps_args) def loads( json_str: str, skip_json_loads: bool = False, logging: bool = False, stream_stable: bool = False, strict: bool = False, ) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]] | str: """ This function works like `json.loads()` except that it will fix your JSON in the process. It is a wrapper around the `repair_json()` function with `return_objects=True`. Args: json_str (str): The JSON string to load and repair. skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False. logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False. strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them. Returns: Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]], str]: The repaired JSON object or a tuple with the repaired JSON object and repair log. """ return repair_json( json_str=json_str, return_objects=True, skip_json_loads=skip_json_loads, logging=logging, stream_stable=stream_stable, strict=strict, ) def load( fd: TextIO, skip_json_loads: bool = False, logging: bool = False, chunk_length: int = 0, strict: bool = False, ) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]: """ This function works like `json.load()` except that it will fix your JSON in the process. It is a wrapper around the `repair_json()` function with `json_fd=fd` and `return_objects=True`. Args: fd (TextIO): File descriptor for JSON input. skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False. logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False. chunk_length (int, optional): Size in bytes of the file chunks to read at once. Defaults to 1MB. strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them. Returns: Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON object or a tuple with the repaired JSON object and repair log. """ return repair_json( json_fd=fd, chunk_length=chunk_length, return_objects=True, skip_json_loads=skip_json_loads, logging=logging, strict=strict, ) def from_file( filename: str, skip_json_loads: bool = False, logging: bool = False, chunk_length: int = 0, strict: bool = False, ) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]: """ This function is a wrapper around `load()` so you can pass the filename as string Args: filename (str): The name of the file containing JSON data to load and repair. skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False. logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False. chunk_length (int, optional): Size in bytes of the file chunks to read at once. Defaults to 1MB. strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them. Returns: Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON object or a tuple with the repaired JSON object and repair log. """ with open(filename) as fd: jsonobj = load( fd=fd, skip_json_loads=skip_json_loads, logging=logging, chunk_length=chunk_length, strict=strict, ) return jsonobj def cli(inline_args: list[str] | None = None) -> int: """ Command-line interface for repairing and parsing JSON files. Args: inline_args (Optional[List[str]]): List of command-line arguments for testing purposes. Defaults to None. - filename (str): The JSON file to repair. If omitted, the JSON is read from stdin. - -i, --inline (bool): Replace the file inline instead of returning the output to stdout. - -o, --output TARGET (str): If specified, the output will be written to TARGET filename instead of stdout. - --ensure_ascii (bool): Pass ensure_ascii=True to json.dumps(). Will pass False otherwise. - --indent INDENT (int): Number of spaces for indentation (Default 2). Returns: int: Exit code of the CLI operation. Raises: Exception: Any exception that occurs during file processing. Example: >>> cli(['example.json', '--indent', '4']) >>> cat json.txt | json_repair """ parser = argparse.ArgumentParser(description="Repair and parse JSON files.") # Make the filename argument optional; if omitted, we will read from stdin. parser.add_argument( "filename", nargs="?", help="The JSON file to repair (if omitted, reads from stdin)", ) parser.add_argument( "-i", "--inline", action="store_true", help="Replace the file inline instead of returning the output to stdout", ) parser.add_argument( "-o", "--output", metavar="TARGET", help="If specified, the output will be written to TARGET filename instead of stdout", ) parser.add_argument( "--ensure_ascii", action="store_true", help="Pass ensure_ascii=True to json.dumps()", ) parser.add_argument( "--indent", type=int, default=2, help="Number of spaces for indentation (Default 2)", ) parser.add_argument( "--strict", action="store_true", help="Raise on duplicate keys, missing separators, empty keys/values, and other unrecoverable structures instead of repairing them", ) args = parser.parse_args() if inline_args is None else parser.parse_args(inline_args) # Inline mode requires a filename, so error out if none was provided. if args.inline and not args.filename: # pragma: no cover print("Error: Inline mode requires a filename", file=sys.stderr) sys.exit(1) if args.inline and args.output: # pragma: no cover print("Error: You cannot pass both --inline and --output", file=sys.stderr) sys.exit(1) ensure_ascii = False if args.ensure_ascii: ensure_ascii = True try: # Use from_file if a filename is provided; otherwise read from stdin. if args.filename: result = from_file(args.filename, strict=args.strict) else: data = sys.stdin.read() result = loads(data, strict=args.strict) if args.inline or args.output: with open(args.output or args.filename, mode="w") as fd: json.dump(result, fd, indent=args.indent, ensure_ascii=ensure_ascii) else: print(json.dumps(result, indent=args.indent, ensure_ascii=ensure_ascii)) except Exception as e: # pragma: no cover print(f"Error: {str(e)}", file=sys.stderr) return 1 return 0 # Success if __name__ == "__main__": # pragma: no cover sys.exit(cli())