diff --git a/.gitignore b/.gitignore index 239d948..96f7c55 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ -remarshal +/build +/dist +/venv* +*.egg-info +*.pyc +__pycache__ diff --git a/.travis.yml b/.travis.yml index 53dfd3b..f7a210e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,9 @@ -language: go +language: python sudo: false -go: - - 1.2 - - 1.3 - - 1.4 - - 1.5 - - 1.6 - - 1.7 - - release -install: - - go get github.com/BurntSushi/toml - - go get gopkg.in/yaml.v2 -before_script: - - chmod +x tests.sh -script: - - go build remarshal.go - - ./tests.sh +python: + - 2.7 + - 3.3 + - 3.4 + - 3.5 +install: python setup.py install +script: python setup.py test diff --git a/LICENSE b/LICENSE index b69569c..4f87d47 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2014 Danyil Bohdan +Copyright (c) 2014, 2015, 2016 dbohdan Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 5b57182..c62ea3b 100644 --- a/README.md +++ b/README.md @@ -10,97 +10,83 @@ commands `toml2yaml`, `toml2json`, `yaml2toml`, `yaml2json`. `json2toml` and # Usage ``` -remarshal -if inputformat -of outputformat [-indent-json=(true|false)] - [-i inputfile] [-o outputfile] [-wrap wrapper] [-unwrap wrapper] - +usage: remarshal.py [-h] [-i INPUT] [-o OUTPUT] -if {json,toml,yaml} -of + {json,toml,yaml} [--indent-json] [--wrap WRAP] + [--unwrap UNWRAP] + [inputfile] ``` -where `inputformat` and `outputformat` can each be `toml`, `yaml` or -`json`. +``` +usage: {json,toml,yaml}2{toml,yaml} [-h] [-i INPUT] [-o OUTPUT] [--wrap WRAP] + [--unwrap UNWRAP] [inputfile] +``` ``` -toml2toml [-wrap wrapper] [-unwrap wrapper] [-o outputfile] [[-i] inputfile] -yaml2toml [-wrap wrapper] [-unwrap wrapper] [-o outputfile] [[-i] inputfile] -json2toml [-wrap wrapper] [-unwrap wrapper] [-o outputfile] [[-i] inputfile] -toml2yaml [-wrap wrapper] [-unwrap wrapper] [-o outputfile] [[-i] inputfile] -yaml2yaml [-wrap wrapper] [-unwrap wrapper] [-o outputfile] [[-i] inputfile] -json2yaml [-wrap wrapper] [-unwrap wrapper] [-o outputfile] [[-i] inputfile] -toml2json [-indent-json=(true|false)] [-wrap wrapper] [-unwrap wrapper] - [-o outputfile] [[-i] inputfile] -yaml2json [-indent-json=(true|false)] [-wrap wrapper] [-unwrap wrapper] - [-o outputfile] [[-i] inputfile] -json2json [-indent-json=(true|false)] [-wrap wrapper] [-unwrap wrapper] - [-o outputfile] [[-i] inputfile] +usage: {json,toml,yaml}2json [-h] [-i INPUT] [-o OUTPUT] [--indent-json] + [-wrap WRAP] [-unwrap UNWRAP] [inputfile] ``` All of the commands above exit with status 0 on success and 1 on failure. -If no `inputfile` is given or it is `-` or a blank string the data to convert is -read from standard input. If no `outputfile` is given or it is `-` or a blank -string the result of the conversion is written to standard output. +If no `inputfile` or `-i INPUT` is given or it is `-` or a blank string the data +to convert is read from standard input. If no `-o OUTPUT` is given or it is `-` +or a blank string the result of the conversion is written to standard output. For the short commands (`x2y`) the flag `-i` before `inputfile` can be omitted if `inputfile` is the last argument. ## Wrappers -The flags `-wrap` and `-unwrap` are there to solve the problem of converting +The flags `--wrap` and `--unwrap` are there to solve the problem of converting JSON and YAML data to TOML if the topmost element of that data is not of a map type (i.e., not an object in JSON or an associative array in YAML) but a list, a string or a number. Such data can not be represented as TOML directly; it needs -to wrapped in a map type first. Passing the flag `-wrap someKey` to `remarshal` +to wrapped in a map type first. Passing the flag `--wrap someKey` to `remarshal` or one of its short commands wraps the input data in a "wrapper" map with one -key, "someKey", with the input data as its value. The flag `-unwrap someKey` +key, "someKey", with the input data as its value. The flag `--unwrap someKey` does the opposite: if it is specified only the value stored under the key "someKey" in the top-level map element of the input data is converted to the target format and output; all other data is skipped. If the top-level element is -not a map or does not have the key `someKey` then `-unwrap someKey` returns an +not a map or does not have the key `someKey` then `--unwrap someKey` returns an error. -The following shell transcript demonstrates the problem and how `-wrap` and -`-unwrap` solve it: +The following shell transcript demonstrates the problem and how `--wrap` and +`--unwrap` solve it: ``` -$ echo '[{"a":"b"},{"c":[1,2,3]}]' | ./remarshal -if json -of toml -cannot convert data: top-level values must be a Go map or struct +$ echo '[{"a":"b"},{"c":[1,2,3]}]' | ./remarshal.py -if json -of toml +Error: cannot convert non-dictionary data to TOML; use "wrap" to wrap it in a dictionary $ echo '[{"a":"b"},{"c":[1,2,3]}]' | \ -./remarshal -if json -of toml -wrap main +./remarshal.py -if json -of toml --wrap main [[main]] - a = "b" +a = "b" [[main]] - c = [1, 2, 3] +c = [1, 2, 3] $ echo '[{"a":"b"},{"c":[1,2,3]}]' | \ -./remarshal -if json -of toml -wrap main > test.toml +./remarshal.py -if json -of toml --wrap main > test.toml -$ ./remarshal -if toml -of json -indent-json=0 < test.toml +$ ./remarshal.py -if toml -of json < test.toml {"main":[{"a":"b"},{"c":[1,2,3]}]} -$ ./remarshal -if toml -of json -indent-json=0 -unwrap main < test.toml +$ ./remarshal.py -if toml -of json --unwrap main < test.toml [{"a":"b"},{"c":[1,2,3]}] ``` -# Building and installation +# Installation -Tested with `go version go1.2.2 linux/amd64`. Do the following to install -`remarshal`: +You will need Python 2.7 or Python 3.3 or later. ```sh -go get github.com/BurntSushi/toml -go get gopkg.in/yaml.v2 -git clone https://github.com/dbohdan/remarshal.git -cd remarshal -go build remarshal.go -sh tests.sh -sudo sh install.sh # install into /usr/local/bin +sudo python setup.py install ``` # Examples ``` -$ ./remarshal -i example.toml -if toml -of yaml +$ ./remarshal.py -i example.toml -if toml -of yaml clients: data: - - gamma @@ -119,10 +105,10 @@ database: - 8002 server: 192.168.1.1 owner: - bio: |- - GitHub Cofounder & CEO - Likes tater tots and beer. - dob: 1979-05-27T07:32:00Z + bio: 'GitHub Cofounder & CEO + + Likes tater tots and beer.' + dob: 1979-05-27 07:32:00+00:00 name: Tom Preston-Werner organization: GitHub products: @@ -142,7 +128,7 @@ servers: title: TOML Example $ curl -s http://api.openweathermap.org/data/2.5/weather\?q\=Kiev,ua | \ -./remarshal -if json -of toml +./remarshal.py -if json -of toml base = "cmc stations" cod = 200 dt = 1412532000 @@ -150,43 +136,42 @@ id = 703448 name = "Kiev" [clouds] - all = 44 +all = 44 [coord] - lat = 50.43 - lon = 30.52 +lat = 50.42999999999999972 +lon = 30.51999999999999957 [main] - humidity = 66 - pressure = 1026 - temp = 283.49 - temp_max = 284.15 - temp_min = 283.15 +humidity = 66 +pressure = 1026 +temp = 283.49000000000000909 +temp_max = 284.14999999999997726 +temp_min = 283.14999999999997726 [sys] - country = "UA" - id = 7358 - message = 0.2437 - sunrise = 1412481902 - sunset = 1412522846 - type = 1 +country = "UA" +id = 7358 +message = 0.24370000000000000 +sunrise = 1412481902 +sunset = 1412522846 +type = 1 [[weather]] - description = "scattered clouds" - icon = "03n" - id = 802 - main = "Clouds" +description = "scattered clouds" +icon = "03n" +id = 802 +main = "Clouds" [wind] - deg = 80 - speed = 2 +deg = 80 +speed = 2 ``` # Known bugs and limitations * Converting data with floating point values to YAML may cause a loss of precision. -* `remarshal` only supports TOML [v0.2.0](https://github.com/toml-lang/toml/tree/v0.2.0). # License diff --git a/VERSION b/VERSION deleted file mode 100644 index 0d91a54..0000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.3.0 diff --git a/array.json b/array.json new file mode 100644 index 0000000..91760a2 --- /dev/null +++ b/array.json @@ -0,0 +1 @@ +[{"a":"b"},{"c":[1,2,3]}] diff --git a/array.toml b/array.toml new file mode 100644 index 0000000..ea95a95 --- /dev/null +++ b/array.toml @@ -0,0 +1,6 @@ + +[[data]] +a = "b" + +[[data]] +c = [1, 2, 3] diff --git a/example.json b/example.json index 659442e..37bead3 100644 --- a/example.json +++ b/example.json @@ -26,8 +26,8 @@ "server": "192.168.1.1" }, "owner": { - "bio": "GitHub Cofounder \u0026 CEO\nLikes tater tots and beer.", - "dob": "1979-05-27T07:32:00Z", + "bio": "GitHub Cofounder & CEO\nLikes tater tots and beer.", + "dob": "1979-05-27T07:32:00+00:00", "name": "Tom Preston-Werner", "organization": "GitHub" }, @@ -54,4 +54,4 @@ } }, "title": "TOML Example" -} \ No newline at end of file +} diff --git a/example.yaml b/example.yaml index 4ff0bef..372f3f4 100644 --- a/example.yaml +++ b/example.yaml @@ -16,10 +16,10 @@ database: - 8002 server: 192.168.1.1 owner: - bio: |- - GitHub Cofounder & CEO - Likes tater tots and beer. - dob: 1979-05-27T07:32:00Z + bio: 'GitHub Cofounder & CEO + + Likes tater tots and beer.' + dob: 1979-05-27 07:32:00+00:00 name: Tom Preston-Werner organization: GitHub products: diff --git a/install.sh b/install.sh deleted file mode 100755 index 2644ab9..0000000 --- a/install.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -targetDir=/usr/local/bin -cp remarshal $targetDir -for if in toml yaml json; do - for of in toml yaml json; do - ln -s "$targetDir/remarshal" "$targetDir/${if}2${of}" - done -done diff --git a/remarshal.go b/remarshal.go deleted file mode 100644 index 8bfc458..0000000 --- a/remarshal.go +++ /dev/null @@ -1,330 +0,0 @@ -// remarshal, a utility to convert between serialization formats. -// Copyright (C) 2014 Danyil Bohdan -// License: MIT -package main - -import ( - "bytes" - "encoding/json" - "errors" - "flag" - "fmt" - "github.com/BurntSushi/toml" - "gopkg.in/yaml.v2" - "io/ioutil" - "os" - "path/filepath" - "strings" -) - -type format int - -const ( - fTOML format = iota - fYAML - fJSON - fPlaceholder - fUnknown -) - -const ( - defaultFormatFlagValue = "unspecified" - defaultWrapFlagValue = "key" -) - -// convertMapsToStringMaps recursively converts values of type -// map[interface{}]interface{} contained in item to map[string]interface{}. This -// is needed before the encoders for TOML and JSON can accept data returned by -// the YAML decoder. -func convertMapsToStringMaps(item interface{}) (res interface{}, err error) { - switch item.(type) { - case map[interface{}]interface{}: - res := make(map[string]interface{}) - for k, v := range item.(map[interface{}]interface{}) { - res[k.(string)], err = convertMapsToStringMaps(v) - if err != nil { - return nil, err - } - } - return res, nil - case []interface{}: - res := make([]interface{}, len(item.([]interface{}))) - for i, v := range item.([]interface{}) { - res[i], err = convertMapsToStringMaps(v) - if err != nil { - return nil, err - } - } - return res, nil - default: - return item, nil - } -} - -// convertNumbersToInt64 recursively walks the structures contained in item -// converting values of the type json.Number to int64 or, failing that, float64. -// This approach is meant to prevent encoders from putting numbers stored as -// json.Number in quotes or encoding large intergers in scientific notation. -func convertNumbersToInt64(item interface{}) (res interface{}, err error) { - switch item.(type) { - case map[string]interface{}: - res := make(map[string]interface{}) - for k, v := range item.(map[string]interface{}) { - res[k], err = convertNumbersToInt64(v) - if err != nil { - return nil, err - } - } - return res, nil - case []interface{}: - res := make([]interface{}, len(item.([]interface{}))) - for i, v := range item.([]interface{}) { - res[i], err = convertNumbersToInt64(v) - if err != nil { - return nil, err - } - } - return res, nil - case json.Number: - n, err := item.(json.Number).Int64() - if err != nil { - f, err := item.(json.Number).Float64() - if err != nil { - // Can't convert to Int64. - return item, nil - } - return f, nil - } - return n, nil - default: - return item, nil - } -} - -func stringToFormat(s string) (f format, err error) { - switch strings.ToLower(s) { - case "toml": - return fTOML, nil - case "yaml": - return fYAML, nil - case "json": - return fJSON, nil - case defaultFormatFlagValue: - return fPlaceholder, errors.New("placeholder format") - default: - return fUnknown, errors.New("cannot convert string to format: '" + - s + "'") - } -} - -// filenameToFormat tries to parse string s as "2". -// It returns both formats as type format if successful. -func filenameToFormat(s string) (inputf format, outputf format, err error) { - filenameParts := strings.Split(filepath.Base(s), "2") - if len(filenameParts) != 2 { - return fUnknown, fUnknown, errors.New( - "cannot determine format from filename") - } - prefix, err := stringToFormat(filenameParts[0]) - if err != nil { - return fUnknown, fUnknown, err - } - suffix, err := stringToFormat(filenameParts[1]) - if err != nil { - return fUnknown, fUnknown, err - } - return prefix, suffix, nil -} - -// unmarshal decodes serialized data in the format inputFormat into a structure -// of nested maps and slices. -func unmarshal(input []byte, inputFormat format) (data interface{}, - err error) { - switch inputFormat { - case fTOML: - _, err = toml.Decode(string(input), &data) - case fYAML: - err = yaml.Unmarshal(input, &data) - if err == nil { - data, err = convertMapsToStringMaps(data) - } - case fJSON: - decoder := json.NewDecoder(bytes.NewReader(input)) - decoder.UseNumber() - err = decoder.Decode(&data) - if err == nil { - data, err = convertNumbersToInt64(data) - } - } - if err != nil { - return nil, err - } - return -} - -// marshal encodes data stored in nested maps and slices in the format -// outputFormat. -func marshal(data interface{}, outputFormat format, - indentJSON bool) (result []byte, err error) { - switch outputFormat { - case fTOML: - buf := new(bytes.Buffer) - err = toml.NewEncoder(buf).Encode(data) - result = buf.Bytes() - case fYAML: - result, err = yaml.Marshal(&data) - case fJSON: - result, err = json.Marshal(&data) - if err == nil && indentJSON { - buf := new(bytes.Buffer) - err = json.Indent(buf, result, "", " ") - result = buf.Bytes() - } - } - if err != nil { - return nil, err - } - return -} - -// processCommandLine parses the command line arguments (including os.Args[0], -// the program name) and sets the input and the output file names as well as -// other conversion options based on them. -func processCommandLine() (inputFile string, outputFile string, - inputFormat format, outputFormat format, - indentJSON bool, wrap string, unwrap string) { - var inputFormatStr, outputFormatStr string - - flag.StringVar(&inputFile, "i", "-", "input file") - flag.StringVar(&outputFile, "o", "-", "output file") - flag.StringVar(&wrap, "wrap", defaultWrapFlagValue, - "wrap the data in a map type with the given key") - flag.StringVar(&unwrap, "unwrap", defaultWrapFlagValue, - "only output the data stored under the given key") - - // See if our program is named, e.g., "json2yaml" (normally due to having - // been started through a symlink). - inputFormat, outputFormat, err := filenameToFormat(os.Args[0]) - formatFromProgramName := err == nil - if !formatFromProgramName { - // Only give the user an option to specify the input and the output - // format with flags when it is mandatory, i.e., when we are *not* being - // run as "json2yaml" or similar. This makes the usage messages for the - // "x2y" commands more accurate as well. - flag.StringVar(&inputFormatStr, "if", defaultFormatFlagValue, - "input format ('toml', 'yaml' or 'json')") - flag.StringVar(&outputFormatStr, "of", defaultFormatFlagValue, - "input format ('toml', 'yaml' or 'json')") - } - if !formatFromProgramName || outputFormat == fJSON { - flag.BoolVar(&indentJSON, "indent-json", true, "indent JSON output") - } - flag.Parse() - if !formatFromProgramName { - // Try to parse the format options we were given through the command - // line flags. - if inputFormat, err = stringToFormat(inputFormatStr); err != nil { - if inputFormat == fPlaceholder { - fmt.Println("please specify the input format") - } else { - fmt.Printf("please specify a valid input format ('%s' given)\n", - inputFormatStr) - } - os.Exit(1) - } - if outputFormat, err = stringToFormat(outputFormatStr); err != nil { - if outputFormat == fPlaceholder { - fmt.Println("please specify the output format") - } else { - fmt.Printf( - "please specify a valid output format ('%s' given)\n", - outputFormatStr) - } - os.Exit(1) - } - } - - // Check for extraneous command line arguments. If we are running as "x2y" - // set inputFile if given on the command line without the -i flag. - tail := flag.Args() - if len(tail) > 0 { - if formatFromProgramName && len(tail) == 1 && - (inputFile == "" || inputFile == "-") { - inputFile = flag.Arg(0) - } else { - if len(tail) == 1 { - fmt.Print("extraneous command line argument:") - } else { - fmt.Print("extraneous command line arguments:") - } - for _, a := range tail { - fmt.Printf(" '%s'", a) - } - fmt.Printf("\n") - os.Exit(1) - } - } - return -} - -func main() { - inputFile, outputFile, inputFormat, outputFormat, - indentJSON, wrap, unwrap := processCommandLine() - - // Read the input data from either standard input or a file. - var input []byte - var err error - if inputFile == "" || inputFile == "-" { - input, err = ioutil.ReadAll(os.Stdin) - } else { - if _, err := os.Stat(inputFile); os.IsNotExist(err) { - fmt.Printf("no such file or directory: '%s'\n", inputFile) - os.Exit(1) - } - input, err = ioutil.ReadFile(inputFile) - } - if err != nil { - fmt.Println(err) - os.Exit(1) - - } - - // Convert the input data from inputFormat to outputFormat. - data, err := unmarshal(input, inputFormat) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - // Unwrap and/or wrap the data in a map if we were told to. - if unwrap != defaultWrapFlagValue { - temp, ok := data.(map[string]interface{}) - if !ok { - fmt.Printf("cannot unwrap data: top-level value not a map\n") - os.Exit(1) - } - data, ok = temp[unwrap] - if !ok { - fmt.Printf("cannot unwrap data: no key '%s'\n", unwrap) - os.Exit(1) - } - } - if wrap != defaultWrapFlagValue { - data = map[string]interface{}{wrap: data} - } - output, err := marshal(data, outputFormat, indentJSON) - if err != nil { - fmt.Printf("cannot convert data: %v\n", err) - os.Exit(1) - } - - // Print the result to either standard output or a file. - if outputFile == "" || outputFile == "-" { - fmt.Printf("%s\n", string(output)) - } else { - err = ioutil.WriteFile(outputFile, output, 0644) - if err != nil { - fmt.Printf("cannot write to file %s\n", outputFile) - os.Exit(1) - } - } -} diff --git a/remarshal.py b/remarshal.py new file mode 100755 index 0000000..07c06ff --- /dev/null +++ b/remarshal.py @@ -0,0 +1,168 @@ +#! /usr/bin/env python +# remarshal, a utility to convert between serialization formats. +# Copyright (C) 2014, 2015, 2016 dbohdan +# License: MIT + +from __future__ import print_function + +import argparse +import datetime +import dateutil.parser +import io +import json +import os.path +import re +import string +import sys +import pytoml +import yaml + +FORMATS = ['json', 'toml', 'yaml'] +__version__ = '0.4.0' + +def filename2format(filename): + try: + from_, to = filename.split('2', 1) + except ValueError: + return False, None, None + if from_ in FORMATS and to in FORMATS: + return True, from_, to + else: + return False, None, None + + +def json_serialize(obj): + if isinstance(obj, datetime.datetime): + return obj.isoformat() + raise TypeError("{0} is not JSON serializable".format(repr(obj))) + + +# Fix loss of time zone inforation. +# http://stackoverflow.com/questions/13294186/can-pyyaml-parse-iso8601-dates +def timestamp_constructor(loader, node): + return dateutil.parser.parse(node.value) +yaml.add_constructor(u'tag:yaml.org,2002:timestamp', timestamp_constructor) + + +def parse_command_line(argv): + me = os.path.basename(argv[0]) + format_from_filename, from_, to = filename2format(me) + + parser = argparse.ArgumentParser(description='Convert between JSON, TOML ' + + 'and YAML.') + + group = parser.add_mutually_exclusive_group() + group.add_argument('-i', '--input', dest='input_flag', metavar='INPUT', + default=None, help='input file') + group.add_argument('inputfile', nargs='?', default='-', help='input file') + + parser.add_argument('-o', '--output', dest='output', default='-', + help='output file') + if not format_from_filename: + parser.add_argument('-if', '--input-format', dest='input_format', + required=True, help="input format", choices=FORMATS) + parser.add_argument('-of', '--output-format', dest='output_format', + required=True, help="output format", + choices=FORMATS) + if not format_from_filename or to == 'json': + parser.add_argument('--indent-json', dest='indent_json', + action='store_const', const=2, default=None, + help='indent JSON output') + parser.add_argument('--wrap', dest='wrap', default=None, + help='wrap the data in a map type with the given key') + parser.add_argument('--unwrap', dest='unwrap', default=None, + help='only output the data stored under the given key') + + args = parser.parse_args(args=argv[1:]) + + if args.input_flag is not None: + args.input = args.input_flag + else: + args.input = args.inputfile + if format_from_filename: + args.input_format = from_ + args.output_format = to + if to != 'json': + args.__dict__['indent_json'] = None + + return args + + +def run(argv): + args = parse_command_line(argv) + remarshal(args.input, args.output, args.input_format, args.output_format, + args.indent_json, args.wrap, args.unwrap) + + +def remarshal(input, output, input_format, output_format, indent_json=None, + wrap=None, unwrap=None): + if input == '-': + input_file = sys.stdin + else: + input_file = open(input, 'rb') + + if output == '-': + output_file = getattr(sys.stdout, 'buffer', sys.stdout) + else: + output_file = open(output, 'wb') + + input_data = input_file.read() + + if input_format == 'json': + parsed = json.loads(input_data.decode('utf-8')) + elif input_format == 'toml': + parsed = pytoml.loads(input_data) + elif input_format == 'yaml': + parsed = yaml.load(input_data) + else: + raise ValueError('Unknown input format: {0}'.format(input_format)) + + if unwrap is not None: + parsed = parsed[unwrap] + if wrap is not None: + temp = {} + temp[wrap] = parsed + parsed = temp + + if output_format == 'json': + if indent_json == True: + indent_json = 2 + if indent_json: + separators=(',', ': ') + else: + separators=(',', ':') + output_data = json.dumps(parsed, default=json_serialize, + ensure_ascii=False, indent=indent_json, + separators=separators, sort_keys=True) + "\n" + elif output_format == 'toml': + try: + output_data = pytoml.dumps(parsed, sort_keys=True) + except AttributeError as e: + if str(e) == "'list' object has no attribute 'keys'": + raise ValueError('cannot convert non-dictionary data to TOML;' + + ' use "wrap" to wrap it in a dictionary') + else: + raise e + elif output_format == 'yaml': + output_data = yaml.safe_dump(parsed, allow_unicode=True, + default_flow_style=False, + encoding=None) + else: + raise ValueError('Unknown output format: {0}'. + format(output_format)) + output_file.write(output_data.encode('utf-8')) + + input_file.close() + output_file.close() + + +def main(): + try: + run(sys.argv) + except ValueError as e: + print('Error: {0}'.format(e), file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c4295db --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +python-dateutil==2.5.3 +pytoml==0.1.11 +PyYAML==3.12 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..5ffbef0 --- /dev/null +++ b/setup.py @@ -0,0 +1,27 @@ +#! /usr/bin/env python + +import re +from setuptools import find_packages, setup + +with open('remarshal.py', 'rb') as f: + content = f.read().decode('utf-8') + version = re.search(r"__version__ = '(\d+\.\d+\.\d+)", + content, re.MULTILINE).group(1) + +setup(name='remarshal', + version=version, + description='Convert between TOML, YAML and JSON', + author='dbohdan', + url='https://github.com/dbohdan/remarshal', + license='MIT', + py_modules=['remarshal'], + test_suite='tests', + install_requires=[ + 'python-dateutil >= 2.5.0', + 'pytoml >= 0.1.11', + 'PyYAML >= 3.12', + ], + entry_points = { + 'console_scripts': ['remarshal = remarshal:main'], + }, +) diff --git a/tests.sh b/tests.sh deleted file mode 100755 index 41ad151..0000000 --- a/tests.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -set -e - -cp example.toml /tmp/toml -cp example.yaml /tmp/yaml -cp example.json /tmp/json - -for if in toml yaml json; do - for of in toml yaml json; do - echo "--- $if -> $of" - ./remarshal -i "/tmp/$if" -o "/tmp/$of.2" -if $if -of $of - if test "$1" = "-v"; then - cat "/tmp/$of.2" - fi - if test "$of" != "toml"; then - diff "/tmp/$of" "/tmp/$of.2" - fi - done -done - -for filename in toml yaml json; do - rm "/tmp/$filename" "/tmp/$filename.2" -done diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/context.py b/tests/context.py new file mode 100644 index 0000000..ec3388d --- /dev/null +++ b/tests/context.py @@ -0,0 +1,5 @@ +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + +import remarshal diff --git a/tests/test_remarshal.py b/tests/test_remarshal.py new file mode 100755 index 0000000..69c2f7d --- /dev/null +++ b/tests/test_remarshal.py @@ -0,0 +1,122 @@ +#! /usr/bin/env python + +from .context import remarshal +import os +import os.path +import re +import tempfile +import unittest + + +TEST_DATA_PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + +def test_file_path(filename): + return os.path.join(TEST_DATA_PATH, filename) + + +def readFile(filename): + with open(test_file_path(filename), 'rb') as f: + content = f.read().decode('utf-8') + return content + + +def tomlSignature(data): + '''A lossy representation for TOML example data for comparison.''' + def strip_more(line): + return re.sub(r' *#.*$', '', line.strip()).replace(' ', '') + def f(lst): + def q(line): + return line.startswith('#') or line == u'' or line == u']' or \ + re.match(r'^".*",?$', line) or re.match(r'^hosts', line) + return sorted([strip_more(line) for line in lst if + not q(strip_more(line))]) + return f(data.split("\n")) + + +class TestRemarshal(unittest.TestCase): + + def tempFilename(self): + temp_filename = tempfile.mkstemp()[1] + self.temp_files.append(temp_filename) + return temp_filename + + def convertAndRead(self, input, input_format, output_format, + indent_json=True, wrap=None, unwrap=None): + output_filename = self.tempFilename() + remarshal.remarshal(test_file_path(input), output_filename, + input_format, output_format, + indent_json=indent_json, wrap=wrap, unwrap=unwrap) + return readFile(output_filename) + + def setUp(self): + self.temp_files = [] + + def tearDown(self): + for filename in self.temp_files: + os.remove(filename) + + def test_json2json(self): + output = self.convertAndRead('example.json', 'json', 'json') + reference = readFile('example.json') + self.assertEqual(output, reference) + + def test_toml2toml(self): + output = self.convertAndRead('example.toml', 'toml', 'toml') + reference = readFile('example.toml') + self.assertEqual(tomlSignature(output), tomlSignature(reference)) + + def test_yaml2yaml(self): + output = self.convertAndRead('example.yaml', 'yaml', 'yaml') + reference = readFile('example.yaml') + self.assertEqual(output, reference) + + def test_json2toml(self): + output = self.convertAndRead('example.json', 'json', 'toml') + reference = readFile('example.toml') + output_sig = tomlSignature(output) + # The date in 'example.json' is a string. + reference_sig = tomlSignature(reference. + replace('1979-05-27T07:32:00Z', '"1979-05-27T07:32:00+00:00"')) + self.assertEqual(output_sig, reference_sig) + + def test_json2yaml(self): + output = self.convertAndRead('example.json', 'json', 'yaml') + reference = readFile('example.yaml') + # The date in 'example.json' is a string. + reference_patched = reference.replace('1979-05-27 07:32:00+00:00', + "'1979-05-27T07:32:00+00:00'") + self.assertEqual(output, reference_patched) + + def test_toml2json(self): + output = self.convertAndRead('example.toml', 'toml', 'json') + reference = readFile('example.json') + self.assertEqual(output, reference) + + def test_toml2yaml(self): + output = self.convertAndRead('example.toml', 'toml', 'yaml') + reference = readFile('example.yaml') + self.assertEqual(output, reference) + + def test_yaml2json(self): + output = self.convertAndRead('example.yaml', 'yaml', 'json') + reference = readFile('example.json') + self.assertEqual(output, reference) + + def test_yaml2toml(self): + output = self.convertAndRead('example.yaml', 'yaml', 'toml') + reference = readFile('example.toml') + self.assertEqual(tomlSignature(output), tomlSignature(reference)) + + def test_wrap(self): + output = self.convertAndRead('array.json', 'json', 'toml', wrap='data') + reference = readFile('array.toml') + self.assertEqual(output, reference) + + def test_unwrap(self): + output = self.convertAndRead('array.toml', 'toml', 'json', + unwrap='data', indent_json=None) + reference = readFile('array.json') + self.assertEqual(output, reference) + +if __name__ == '__main__': + unittest.main()