diff --git a/README.md b/README.md index b8a7f48..58745bd 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,13 @@ or TOML. Binary fields are converted between CBOR, MessagePack, and YAML. be converted to CBOR. The Local Date type can only be converted to JSON and YAML. The Local Time type can not be converted to any other format. Offset Date-Time and its equivalents can be converted between CBOR, MessagePack, -TOML, and YAML. +TOML, and YAML. Keys of any date-time type are converted to string TOML +keys. * Date and time types are converted to JSON strings. They can not be safely roundtripped through JSON. -* A YAML timestamp with only a date becomes a TOML Local Date-Time for the -midnight of that date. +* A YAML timestamp with only a date becomes a YAML timestamp or a TOML Local +Date-Time for the midnight of that date. This means you can not roundtrip +every YAML document through Remarshal. ## Installation diff --git a/pyproject.toml b/pyproject.toml index 71e7ce2..48b401f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "Remarshal" -version = "0.16.1" +version = "0.17.0" description = "Convert between CBOR, JSON, MessagePack, TOML, and YAML" authors = ["D. Bohdan "] license = "MIT" diff --git a/remarshal.py b/remarshal.py index 2d261bf..aba6659 100755 --- a/remarshal.py +++ b/remarshal.py @@ -24,7 +24,7 @@ import yaml import yaml.parser import yaml.scanner -__version__ = "0.16.1" +__version__ = "0.17.0" FORMATS = ["cbor", "json", "msgpack", "toml", "yaml"] @@ -61,16 +61,6 @@ for loader in loaders: loader.add_constructor("tag:yaml.org,2002:timestamp", timestamp_constructor) -# === JSON === - - -def json_default(obj: Any) -> str: - if isinstance(obj, datetime.datetime): - return obj.isoformat() - msg = f"{obj!r} is not JSON serializable" - raise TypeError(msg) - - # === CLI === @@ -96,6 +86,7 @@ def parse_command_line(argv: List[str]) -> argparse.Namespace: # noqa: C901. defaults: Dict[str, Any] = { "json_indent": None, "ordered": True, + "stringify_keys": False, "yaml_options": {}, } @@ -160,6 +151,18 @@ def parse_command_line(argv: List[str]) -> argparse.Namespace: # noqa: C901. help="JSON indentation", ) + if not format_from_argv0 or argv0_to in {"json", "toml"}: + parser.add_argument( + "-k", + "--stringify-keys", + dest="stringify_keys", + action="store_true", + help=( + "stringify boolean, datetime, null keys when converting to " + "JSON and TOML" + ), + ) + if not format_from_argv0 or argv0_to == "yaml": parser.add_argument( "--yaml-indent", @@ -422,25 +425,57 @@ def decode(input_format: str, input_data: bytes) -> Document: return decoder[input_format](input_data) +def reject_special_keys(key: Any) -> Any: + if isinstance(key, bool): + msg = "boolean key" + raise TypeError(msg) + if isinstance(key, datetime.datetime): + msg = "datetime key" + raise TypeError(msg) + if key is None: + msg = "null key" + raise TypeError(msg) + + return key + + +def stringify_special_keys(key: Any) -> Any: + if isinstance(key, bool): + return "true" if key else "false" + if isinstance(key, datetime.datetime): + return key.isoformat() + if key is None: + return "null" + + return key + + +def json_default(obj: Any) -> str: + if isinstance(obj, datetime.datetime): + return obj.isoformat() + msg = f"{obj!r} is not JSON-serializable" + raise TypeError(msg) + + def encode_json( - data: Document, ordered: bool, indent: Union[bool, int, None] # noqa: FBT001 + data: Document, + *, + ordered: bool, + indent: Union[bool, int, None], + stringify_keys: bool, ) -> str: if indent is True: indent = 2 separators = (",", ": " if indent else ":") - - def stringify_key(key: Any) -> Any: - if isinstance(key, bool): - return "true" if key else "false" - return "null" if key is None else key + key_callback = stringify_special_keys if stringify_keys else reject_special_keys try: return ( json.dumps( traverse( data, - key_callback=stringify_key, + key_callback=key_callback, ), default=json_default, ensure_ascii=False, @@ -450,7 +485,7 @@ def encode_json( ) + "\n" ) - except TypeError as e: + except (TypeError, ValueError) as e: msg = f"Cannot convert data to JSON ({e})" raise ValueError(msg) @@ -471,9 +506,22 @@ def encode_cbor(data: Document) -> bytes: raise ValueError(msg) -def encode_toml(data: Mapping[Any, Any], ordered: bool) -> str: # noqa: FBT001 +def encode_toml( + data: Mapping[Any, Any], + *, + ordered: bool, + stringify_keys: bool, +) -> str: + key_callback = stringify_special_keys if stringify_keys else reject_special_keys + try: - return tomlkit.dumps(data, sort_keys=not ordered) + return tomlkit.dumps( + traverse( + data, + key_callback=key_callback, + ), + sort_keys=not ordered, + ) except AttributeError as e: if str(e) == "'list' object has no attribute 'as_string'": msg = ( @@ -488,9 +536,7 @@ def encode_toml(data: Mapping[Any, Any], ordered: bool) -> str: # noqa: FBT001 raise ValueError(msg) -def encode_yaml( - data: Document, ordered: bool, yaml_options: Dict[Any, Any] # noqa: FBT001 -) -> str: +def encode_yaml(data: Document, *, ordered: bool, yaml_options: Dict[Any, Any]) -> str: dumper = OrderedDumper if ordered else yaml.SafeDumper try: return yaml.dump( @@ -513,10 +559,16 @@ def encode( *, json_indent: Union[int, None], ordered: bool, + stringify_keys: bool, yaml_options: Dict[Any, Any], ) -> bytes: if output_format == "json": - encoded = encode_json(data, ordered, json_indent).encode("utf-8") + encoded = encode_json( + data, + indent=json_indent, + ordered=ordered, + stringify_keys=stringify_keys, + ).encode("utf-8") elif output_format == "msgpack": encoded = encode_msgpack(data) elif output_format == "toml": @@ -526,9 +578,13 @@ def encode( "be encoded as TOML" ) raise TypeError(msg) - encoded = encode_toml(data, ordered).encode("utf-8") + encoded = encode_toml( + data, ordered=ordered, stringify_keys=stringify_keys + ).encode("utf-8") elif output_format == "yaml": - encoded = encode_yaml(data, ordered, yaml_options).encode("utf-8") + encoded = encode_yaml(data, ordered=ordered, yaml_options=yaml_options).encode( + "utf-8" + ) elif output_format == "cbor": encoded = encode_cbor(data) else: @@ -548,11 +604,12 @@ def run(argv: List[str]) -> None: args.output, args.input_format, args.output_format, - args.wrap, - args.unwrap, - args.json_indent, - args.yaml_options, - args.ordered, + json_indent=args.json_indent, + ordered=args.ordered, + stringify_keys=args.stringify_keys, + unwrap=args.unwrap, + wrap=args.wrap, + yaml_options=args.yaml_options, ) @@ -561,12 +618,14 @@ def remarshal( output: str, input_format: str, output_format: str, - wrap: Union[str, None] = None, - unwrap: Union[str, None] = None, + *, json_indent: Union[int, None] = None, - yaml_options: Dict[Any, Any] = {}, - ordered: bool = True, # noqa: FBT001 + ordered: bool = True, + stringify_keys: bool = False, transform: Union[Callable[[Document], Document], None] = None, + unwrap: Union[str, None] = None, + wrap: Union[str, None] = None, + yaml_options: Dict[Any, Any] = {}, ) -> None: input_file = None output_file = None @@ -603,6 +662,7 @@ def remarshal( parsed, json_indent=json_indent, ordered=ordered, + stringify_keys=stringify_keys, yaml_options=yaml_options, ) diff --git a/tests/bool-null-key.toml b/tests/bool-null-key.toml new file mode 100644 index 0000000..2bda468 --- /dev/null +++ b/tests/bool-null-key.toml @@ -0,0 +1,4 @@ +true = "foo" +false = "oof" +another = "bar" +null = "nothin'" diff --git a/tests/empty-mapping.yaml b/tests/empty-mapping.yaml new file mode 100644 index 0000000..0152a79 --- /dev/null +++ b/tests/empty-mapping.yaml @@ -0,0 +1 @@ +foo: diff --git a/tests/test_remarshal.py b/tests/test_remarshal.py index 3d3325b..b370e75 100755 --- a/tests/test_remarshal.py +++ b/tests/test_remarshal.py @@ -83,14 +83,16 @@ class TestRemarshal(unittest.TestCase): input: str, input_format: str, output_format: str, - wrap: Union[str, None] = None, - unwrap: Union[str, None] = None, + *, json_indent: Union[int, None] = 2, - yaml_options: Dict[Any, Any] = {}, - ordered: bool = False, # noqa: FBT001 + ordered: bool = False, + stringify_keys: bool = False, transform: Union[ Callable[[remarshal.Document], remarshal.Document], None ] = None, + unwrap: Union[str, None] = None, + wrap: Union[str, None] = None, + yaml_options: Dict[Any, Any] = {}, ) -> bytes: output_filename = self.temp_filename() remarshal.remarshal( @@ -98,12 +100,13 @@ class TestRemarshal(unittest.TestCase): output_filename, input_format, output_format, - wrap=wrap, - unwrap=unwrap, json_indent=json_indent, - yaml_options=yaml_options, ordered=ordered, + stringify_keys=stringify_keys, transform=transform, + unwrap=unwrap, + wrap=wrap, + yaml_options=yaml_options, ) return read_file(output_filename) @@ -506,17 +509,40 @@ class TestRemarshal(unittest.TestCase): reference = read_file("example.yaml") assert output == reference - def test_bool_null_key_yaml2json(self) -> None: + def test_yaml2json_bool_null_key(self) -> None: output = self.convert_and_read( "bool-null-key.yaml", "yaml", "json", json_indent=0, ordered=True, + stringify_keys=True, ) reference = read_file("bool-null-key.json") assert output == reference + def test_yaml2toml_bool_null_key(self) -> None: + output = self.convert_and_read( + "bool-null-key.yaml", + "yaml", + "toml", + ordered=True, + stringify_keys=True, + ) + reference = read_file("bool-null-key.toml") + assert output == reference + + def test_yaml2toml_timestamp_key(self) -> None: + output = self.convert_and_read( + "timestamp-key.yaml", + "yaml", + "toml", + ordered=True, + stringify_keys=True, + ) + reference = read_file("timestamp-key.toml") + assert output == reference + def test_yaml_width_default(self) -> None: output = self.convert_and_read( "long-line.json", @@ -542,3 +568,7 @@ class TestRemarshal(unittest.TestCase): "long-line.json", "json", "yaml", yaml_options={"indent": 5} ).decode("utf-8") assert set(re.findall(r"\n +", output)) == {"\n ", "\n "} + + def test_yaml2toml_empty_mapping(self) -> None: + with pytest.raises(ValueError): + self.convert_and_read("empty-mapping.yaml", "yaml", "toml") diff --git a/tests/timestamp-key.toml b/tests/timestamp-key.toml new file mode 100644 index 0000000..173481d --- /dev/null +++ b/tests/timestamp-key.toml @@ -0,0 +1 @@ +"2023-08-09T00:00:00" = true diff --git a/tests/timestamp-key.yaml b/tests/timestamp-key.yaml new file mode 100644 index 0000000..f4eb30d --- /dev/null +++ b/tests/timestamp-key.yaml @@ -0,0 +1 @@ +2023-08-09: true