298 lines
12 KiB
Python
298 lines
12 KiB
Python
"""
|
|
This module will parse the JSON file following the BNF definition:
|
|
|
|
<json> ::= <container>
|
|
|
|
<primitive> ::= <number> | <string> | <boolean>
|
|
; Where:
|
|
; <number> is a valid real number expressed in one of a number of given formats
|
|
; <string> is a string of valid characters enclosed in quotes
|
|
; <boolean> is one of the literal strings 'true', 'false', or 'null' (unquoted)
|
|
|
|
<container> ::= <object> | <array>
|
|
<array> ::= '[' [ <json> *(', ' <json>) ] ']' ; A sequence of JSON values separated by commas
|
|
<object> ::= '{' [ <member> *(', ' <member>) ] '}' ; A sequence of 'members'
|
|
<member> ::= <string> ': ' <json> ; A pair consisting of a name, and a JSON value
|
|
|
|
If something is wrong (a missing parentheses or quotes for example) it will use a few simple heuristics to fix the JSON string:
|
|
- Add the missing parentheses if the parser believes that the array or object should be closed
|
|
- Quote strings or add missing single quotes
|
|
- Adjust whitespaces and remove line breaks
|
|
|
|
All supported use cases are in the unit tests
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from typing import Any, Literal, TextIO, overload
|
|
|
|
from .json_parser import JSONParser
|
|
from .utils.constants import JSONReturnType
|
|
|
|
|
|
@overload
|
|
def repair_json(
|
|
json_str: str = "",
|
|
return_objects: Literal[False] = False,
|
|
skip_json_loads: bool = False,
|
|
logging: bool = False,
|
|
json_fd: TextIO | None = None,
|
|
chunk_length: int = 0,
|
|
stream_stable: bool = False,
|
|
strict: bool = False,
|
|
**json_dumps_args: Any,
|
|
) -> str: ...
|
|
|
|
|
|
@overload
|
|
def repair_json(
|
|
json_str: str = "",
|
|
return_objects: Literal[True] = True,
|
|
skip_json_loads: bool = False,
|
|
logging: bool = False,
|
|
json_fd: TextIO | None = None,
|
|
chunk_length: int = 0,
|
|
stream_stable: bool = False,
|
|
strict: bool = False,
|
|
**json_dumps_args: Any,
|
|
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]: ...
|
|
|
|
|
|
def repair_json(
|
|
json_str: str = "",
|
|
return_objects: bool = False,
|
|
skip_json_loads: bool = False,
|
|
logging: bool = False,
|
|
json_fd: TextIO | None = None,
|
|
chunk_length: int = 0,
|
|
stream_stable: bool = False,
|
|
strict: bool = False,
|
|
**json_dumps_args: Any,
|
|
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
|
|
"""
|
|
Given a json formatted string, it will try to decode it and, if it fails, it will try to fix it.
|
|
|
|
Args:
|
|
json_str (str, optional): The JSON string to repair. Defaults to an empty string.
|
|
return_objects (bool, optional): If True, return the decoded data structure. Defaults to False.
|
|
skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
|
|
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False. When no repairs were required, the repair log will be an empty list.
|
|
json_fd (Optional[TextIO], optional): File descriptor for JSON input. Do not use! Use `from_file` or `load` instead. Defaults to None.
|
|
ensure_ascii (bool, optional): Set to False to avoid converting non-latin characters to ascii (for example when using chinese characters). Defaults to True. Ignored if `skip_json_loads` is True.
|
|
chunk_length (int, optional): Size in bytes of the file chunks to read at once. Ignored if `json_fd` is None. Do not use! Use `from_file` or `load` instead. Defaults to 1MB.
|
|
stream_stable (bool, optional): When the json to be repaired is the accumulation of streaming json at a certain moment.If this parameter to True will keep the repair results stable.
|
|
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
|
|
Returns:
|
|
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON or a tuple with the repaired JSON and repair log when logging is True.
|
|
"""
|
|
parser = JSONParser(json_str, json_fd, logging, chunk_length, stream_stable, strict)
|
|
if skip_json_loads:
|
|
parsed_json = parser.parse()
|
|
else:
|
|
try:
|
|
parsed_json = json.load(json_fd) if json_fd else json.loads(json_str)
|
|
except json.JSONDecodeError:
|
|
parsed_json = parser.parse()
|
|
# It's useful to return the actual object instead of the json string,
|
|
# it allows this lib to be a replacement of the json library
|
|
if return_objects or logging:
|
|
# If logging is True, the user should expect a tuple.
|
|
# If json.load(s) worked, the repair log list is empty
|
|
if logging and not isinstance(parsed_json, tuple):
|
|
return parsed_json, []
|
|
return parsed_json
|
|
# Avoid returning only a pair of quotes if it's an empty string
|
|
elif parsed_json == "":
|
|
return ""
|
|
return json.dumps(parsed_json, **json_dumps_args)
|
|
|
|
|
|
def loads(
|
|
json_str: str,
|
|
skip_json_loads: bool = False,
|
|
logging: bool = False,
|
|
stream_stable: bool = False,
|
|
strict: bool = False,
|
|
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]] | str:
|
|
"""
|
|
This function works like `json.loads()` except that it will fix your JSON in the process.
|
|
It is a wrapper around the `repair_json()` function with `return_objects=True`.
|
|
|
|
Args:
|
|
json_str (str): The JSON string to load and repair.
|
|
skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
|
|
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
|
|
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
|
|
|
|
Returns:
|
|
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]], str]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
|
|
"""
|
|
return repair_json(
|
|
json_str=json_str,
|
|
return_objects=True,
|
|
skip_json_loads=skip_json_loads,
|
|
logging=logging,
|
|
stream_stable=stream_stable,
|
|
strict=strict,
|
|
)
|
|
|
|
|
|
def load(
|
|
fd: TextIO,
|
|
skip_json_loads: bool = False,
|
|
logging: bool = False,
|
|
chunk_length: int = 0,
|
|
strict: bool = False,
|
|
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
|
|
"""
|
|
This function works like `json.load()` except that it will fix your JSON in the process.
|
|
It is a wrapper around the `repair_json()` function with `json_fd=fd` and `return_objects=True`.
|
|
|
|
Args:
|
|
fd (TextIO): File descriptor for JSON input.
|
|
skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
|
|
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
|
|
chunk_length (int, optional): Size in bytes of the file chunks to read at once. Defaults to 1MB.
|
|
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
|
|
|
|
Returns:
|
|
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
|
|
"""
|
|
return repair_json(
|
|
json_fd=fd,
|
|
chunk_length=chunk_length,
|
|
return_objects=True,
|
|
skip_json_loads=skip_json_loads,
|
|
logging=logging,
|
|
strict=strict,
|
|
)
|
|
|
|
|
|
def from_file(
|
|
filename: str,
|
|
skip_json_loads: bool = False,
|
|
logging: bool = False,
|
|
chunk_length: int = 0,
|
|
strict: bool = False,
|
|
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
|
|
"""
|
|
This function is a wrapper around `load()` so you can pass the filename as string
|
|
|
|
Args:
|
|
filename (str): The name of the file containing JSON data to load and repair.
|
|
skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
|
|
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
|
|
chunk_length (int, optional): Size in bytes of the file chunks to read at once. Defaults to 1MB.
|
|
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
|
|
|
|
Returns:
|
|
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
|
|
"""
|
|
with open(filename) as fd:
|
|
jsonobj = load(
|
|
fd=fd,
|
|
skip_json_loads=skip_json_loads,
|
|
logging=logging,
|
|
chunk_length=chunk_length,
|
|
strict=strict,
|
|
)
|
|
|
|
return jsonobj
|
|
|
|
|
|
def cli(inline_args: list[str] | None = None) -> int:
|
|
"""
|
|
Command-line interface for repairing and parsing JSON files.
|
|
|
|
Args:
|
|
inline_args (Optional[List[str]]): List of command-line arguments for testing purposes. Defaults to None.
|
|
- filename (str): The JSON file to repair. If omitted, the JSON is read from stdin.
|
|
- -i, --inline (bool): Replace the file inline instead of returning the output to stdout.
|
|
- -o, --output TARGET (str): If specified, the output will be written to TARGET filename instead of stdout.
|
|
- --ensure_ascii (bool): Pass ensure_ascii=True to json.dumps(). Will pass False otherwise.
|
|
- --indent INDENT (int): Number of spaces for indentation (Default 2).
|
|
|
|
Returns:
|
|
int: Exit code of the CLI operation.
|
|
|
|
Raises:
|
|
Exception: Any exception that occurs during file processing.
|
|
|
|
Example:
|
|
>>> cli(['example.json', '--indent', '4'])
|
|
>>> cat json.txt | json_repair
|
|
"""
|
|
parser = argparse.ArgumentParser(description="Repair and parse JSON files.")
|
|
# Make the filename argument optional; if omitted, we will read from stdin.
|
|
parser.add_argument(
|
|
"filename",
|
|
nargs="?",
|
|
help="The JSON file to repair (if omitted, reads from stdin)",
|
|
)
|
|
parser.add_argument(
|
|
"-i",
|
|
"--inline",
|
|
action="store_true",
|
|
help="Replace the file inline instead of returning the output to stdout",
|
|
)
|
|
parser.add_argument(
|
|
"-o",
|
|
"--output",
|
|
metavar="TARGET",
|
|
help="If specified, the output will be written to TARGET filename instead of stdout",
|
|
)
|
|
parser.add_argument(
|
|
"--ensure_ascii",
|
|
action="store_true",
|
|
help="Pass ensure_ascii=True to json.dumps()",
|
|
)
|
|
parser.add_argument(
|
|
"--indent",
|
|
type=int,
|
|
default=2,
|
|
help="Number of spaces for indentation (Default 2)",
|
|
)
|
|
parser.add_argument(
|
|
"--strict",
|
|
action="store_true",
|
|
help="Raise on duplicate keys, missing separators, empty keys/values, and other unrecoverable structures instead of repairing them",
|
|
)
|
|
|
|
args = parser.parse_args() if inline_args is None else parser.parse_args(inline_args)
|
|
|
|
# Inline mode requires a filename, so error out if none was provided.
|
|
if args.inline and not args.filename: # pragma: no cover
|
|
print("Error: Inline mode requires a filename", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
if args.inline and args.output: # pragma: no cover
|
|
print("Error: You cannot pass both --inline and --output", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
ensure_ascii = False
|
|
if args.ensure_ascii:
|
|
ensure_ascii = True
|
|
|
|
try:
|
|
# Use from_file if a filename is provided; otherwise read from stdin.
|
|
if args.filename:
|
|
result = from_file(args.filename, strict=args.strict)
|
|
else:
|
|
data = sys.stdin.read()
|
|
result = loads(data, strict=args.strict)
|
|
if args.inline or args.output:
|
|
with open(args.output or args.filename, mode="w") as fd:
|
|
json.dump(result, fd, indent=args.indent, ensure_ascii=ensure_ascii)
|
|
else:
|
|
print(json.dumps(result, indent=args.indent, ensure_ascii=ensure_ascii))
|
|
except Exception as e: # pragma: no cover
|
|
print(f"Error: {str(e)}", file=sys.stderr)
|
|
return 1
|
|
|
|
return 0 # Success
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover
|
|
sys.exit(cli())
|