From 6b6e14274780a26d11881941713309955b2ae1ca Mon Sep 17 00:00:00 2001 From: Jake Lishman Date: Wed, 18 Aug 2021 17:14:36 +0100 Subject: [PATCH] Create Python package for reference parser This is just an exercise in packaging; it doesn't actually change any behaviour, or add anything to the parser. It's just for simplicity in managing the tool, and the package path. --- .github/workflows/tests-grammar.yml | 42 +++++++++++------ .gitignore | 1 + source/grammar/.gitignore | 6 +-- source/grammar/README.md | 45 +++++++++++++------ .../openqasm_reference_parser/__init__.py | 4 ++ .../openqasm_reference_parser/exceptions.py | 5 +++ .../openqasm_reference_parser/tools.py | 43 ++++++++++++++++++ source/grammar/pyproject.toml | 3 ++ source/grammar/setup.cfg | 36 +++++++++++++++ 9 files changed, 154 insertions(+), 31 deletions(-) create mode 100644 source/grammar/openqasm_reference_parser/__init__.py create mode 100644 source/grammar/openqasm_reference_parser/exceptions.py create mode 100644 source/grammar/openqasm_reference_parser/tools.py create mode 100644 source/grammar/pyproject.toml create mode 100644 source/grammar/setup.cfg diff --git a/.github/workflows/tests-grammar.yml b/.github/workflows/tests-grammar.yml index dd1756a..1b8cdb0 100644 --- a/.github/workflows/tests-grammar.yml +++ b/.github/workflows/tests-grammar.yml @@ -7,26 +7,40 @@ jobs: tests: name: ANTLR grammar tests runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + antlr-version: ['4.9.2'] + defaults: + run: + working-directory: source/grammar steps: - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 with: python-version: '3.9' + + - uses: actions/setup-java@v2 + with: + java-version: '15' + distribution: 'adopt' + + - name: Update pip + run: python -mpip install --upgrade pip + - name: Install ANTLR4 - run: | - sudo apt install antlr4 - - name: Install Python dependencies - working-directory: source/grammar - run: | - set -e - python -mpip install --upgrade pip - python -mpip install --upgrade -r requirements.txt -r requirements-dev.txt + run: curl -O https://www.antlr.org/download/antlr-${{ matrix.antlr-version }}-complete.jar + + - name: Install ANTLR4 Python runtime + run: python -mpip install antlr4-python3-runtime==${{ matrix.antlr-version }} + - name: Generate grammar - working-directory: source/grammar - run: | - antlr4 -Dlanguage=Python3 qasm3.g4 + run: java -Xmx500M -jar antlr-${{ matrix.antlr-version }}-complete.jar -o openqasm_reference_parser -Dlanguage=Python3 qasm3.g4 + + - name: Install Python package + run: python -mpip install -e .[all] + - name: Run tests - working-directory: source/grammar - run: | - pytest -vv --color=yes tests + run: pytest -vv --color=yes tests diff --git a/.gitignore b/.gitignore index 8fed69b..3fc162d 100644 --- a/.gitignore +++ b/.gitignore @@ -192,6 +192,7 @@ __pycache__ *$py.class pip-log.txt pip-delete-this-directory.txt +*.egg-info # OS specific .DS_Store diff --git a/source/grammar/.gitignore b/source/grammar/.gitignore index 1f2a4c9..dedc56a 100644 --- a/source/grammar/.gitignore +++ b/source/grammar/.gitignore @@ -1,7 +1,5 @@ -qasm3.interp -qasm3.tokens -qasm3Lexer.interp +qasm3*.interp +qasm3*.tokens qasm3Lexer.py -qasm3Lexer.tokens qasm3Listener.py qasm3Parser.py diff --git a/source/grammar/README.md b/source/grammar/README.md index 397eb99..b838ca8 100644 --- a/source/grammar/README.md +++ b/source/grammar/README.md @@ -1,17 +1,36 @@ -# OpenQasm 3.0 Grammar +# OpenQASM 3.0 Grammar Reference -Grammar specification in [ANTLR](https://www.antlr.org/). See `source/grammar/qasm3.g4`. +The file [qasm3.g4](./qasm3.g4) is the reference grammar, written in [ANTLR](https://www.antlr.org/). + +This directory also contains a very basic Python parser, which is simply built from the reference grammar and used to test against the examples. + + +## Requisites + +Building the grammar only requires ANTLR 4. +You can likely get a copy of ANTLR using your system package manager if you are on Unix, or from `brew` if you are on macOS. +You could also follow [these instructions](https://github.com/antlr/antlr4/blob/master/doc/getting-started.md). + +Running the Python version of the parser also requires the ANTLR Python runtime. +You can install this with `pip` by +```bash +$ pip install antlr4-python3-runtime== +``` +where `` should exactly match the version of ANTLR 4 you installed. +If you let `pip` do this automatically when it installs the reference parser, it will likely pull the wrong version, and produce errors during use. + + +## Building the Python Parser + +1. Build the grammar files into the package directory with ` -o openqasm_reference_parser -Dlanguage=Python3 qasm3.g4`. + `` is however you invoke ANTLR. + If you used a package manager, it is likely `antlr4` or `antlr`. + If you followed the "Getting Started" instructions, it is likely just the `antlr4` alias, or it might be `java -jar `. +2. Install the Python package with `pip install -e .`. -## Working with ANTLR -To get up and running with ANTLR, follow these steps. -1. Install ANTLR locally following these [instructions](https://github.com/antlr/antlr4/blob/master/doc/getting-started.md). -2. Install the ANTLR Python runtime: `pip install antlr4-python3-runtime`. -3. Generate the ANTLR parser files in Python: `antlr4 -Dlanguage=Python3 MYPATH/qasm3.g4` - - Note: This assumes you set the `antlr4` alias on installation. -4. You can now use the generated files to parse qasm3 code! See, for instance, the method `build_parse_tree()` in `source/grammar/tests/test_grammar`. ## Run the Tests -1. Make sure you are set up with ANTLR by following the steps above. -2. From the root of the repository, run the test suite: `pytest source/grammar/tests/test_grammar.py` (or just `pytest .`) - - Reference files at `source/grammar/tests/outputs/` - - Example files at `examples/` + +1. Make sure the Python parser is built and available on the Python path. +2. Install the testing requirements with `pip install -e .[tests]` or `pip install -r requirements-dev.txt`. +3. Run `pytest`. diff --git a/source/grammar/openqasm_reference_parser/__init__.py b/source/grammar/openqasm_reference_parser/__init__.py new file mode 100644 index 0000000..fe825ca --- /dev/null +++ b/source/grammar/openqasm_reference_parser/__init__.py @@ -0,0 +1,4 @@ +from .exceptions import * +from .tools import * +from .qasm3Lexer import qasm3Lexer +from .qasm3Parser import qasm3Parser diff --git a/source/grammar/openqasm_reference_parser/exceptions.py b/source/grammar/openqasm_reference_parser/exceptions.py new file mode 100644 index 0000000..c97b6ed --- /dev/null +++ b/source/grammar/openqasm_reference_parser/exceptions.py @@ -0,0 +1,5 @@ +__all__ = ["Qasm3ParserError"] + + +class Qasm3ParserError(Exception): + pass diff --git a/source/grammar/openqasm_reference_parser/tools.py b/source/grammar/openqasm_reference_parser/tools.py new file mode 100644 index 0000000..6405b72 --- /dev/null +++ b/source/grammar/openqasm_reference_parser/tools.py @@ -0,0 +1,43 @@ +import contextlib +import io + +import antlr4 +from antlr4.tree.Trees import Trees, ParseTree + +from . import Qasm3ParserError +from .qasm3Lexer import qasm3Lexer +from .qasm3Parser import qasm3Parser + +__all__ = ["pretty_tree"] + + +def pretty_tree(program: str = None, file: str = None) -> str: + if program is not None and file is not None: + raise ValueError("Must supply only one of 'program' and 'file'.") + if program is not None: + input_stream = antlr4.InputStream(program) + elif file is not None: + input_stream = antlr4.FileStream(file, encoding="utf-8") + else: + raise TypeError("One of 'program' and 'file' must be supplied.") + + # ANTLR errors (lexing and parsing) are sent to stderr, which we redirect + # to the variable `err`. + with io.StringIO() as err, contextlib.redirect_stderr(err): + lexer = qasm3Lexer(input_stream) + token_stream = antlr4.CommonTokenStream(lexer) + parser = qasm3Parser(token_stream) + tree = _pretty_tree_inner(parser.program(), parser.ruleNames, 0) + error = err.getvalue() + if error: + raise Qasm3ParserError("Parse tree build failed. Error:\n" + error) + return tree + + +def _pretty_tree_inner(parse_tree: ParseTree, rule_names: list, level: int) -> str: + indent = " " * level + tree = indent + Trees.getNodeText(parse_tree, rule_names) + "\n" + return tree + "".join( + _pretty_tree_inner(parse_tree.getChild(i), rule_names, level + 1) + for i in range(parse_tree.getChildCount()) + ) diff --git a/source/grammar/pyproject.toml b/source/grammar/pyproject.toml new file mode 100644 index 0000000..9787c3b --- /dev/null +++ b/source/grammar/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/source/grammar/setup.cfg b/source/grammar/setup.cfg new file mode 100644 index 0000000..1865ed4 --- /dev/null +++ b/source/grammar/setup.cfg @@ -0,0 +1,36 @@ +[metadata] +name = openqasm_reference_parser +url = https://github.com/Qiskit/openqasm +author = OpenQASM Contributors +description = Reference parser for OpenQASM files +long_description = file: README.md +long_description_content_type = text/markdown; variant=GFM +license = Apache 2.0 Software License +license_files = + ../../LICENSE +keywords = openqasm quantum +classifiers = + License :: OSI Approved :: Apache Software License + Intended Audience :: Developers + Intended Audience :: Science/Research + Operating System :: Microsoft :: Windows + Operating System :: MacOS + Operating System :: POSIX :: Linux + Programming Language :: Python :: 3 :: Only + Topic :: Scientific/Engineering + +[options] +packages = find: +include_package_data = True +install_requires = + antlr4-python3-runtime + +[options.packages.find] +exclude = tests* + +[options.extras_require] +tests = + pytest>=6.0 + pyyaml +all = + %(tests)s