Merge pull request #32 from jmanuel1/parser

Replace parsy with my own parser combinator library

.gitignore

@@ -9,7 +9,7 @@ __pycache__/
 
 # Distribution / packaging
 .Python
-env/
+env*/
 build/
 develop-eggs/
 dist/

LICENSE.md

@@ -35,8 +35,8 @@ above.
 
 All files are under the MIT license unless otherwise specified.
 
-**Our `desc_cumulatively` parser combinator is based on the source code of
-`parsy.desc`. [Parsy is under its own
+**Our `desc` parser combinator is based on the source code of `parsy.desc`. [Parsy
+is under its own
 license.](https://github.com/python-parsy/parsy/blob/master/LICENSE)**
 
 ---

concat/__main__.py

@@ -2,17 +2,19 @@
 
 
 import argparse
-from concat.transpile import transpile
+from concat.transpile import parse, transpile_ast, typecheck
 import concat.astutils
 import concat.execute
 import concat.lex
+import concat.parser_combinators
 import concat.stdlib.repl
 import concat.typecheck
 import io
 import json
 import os.path
 import sys
-from typing import Callable, IO, AnyStr, TextIO
+import textwrap
+from typing import Callable, IO, AnyStr, Sequence, TextIO
 
 
 filename = '<stdin>'
@@ -35,6 +37,23 @@ def get_line_at(file: TextIO, location: concat.astutils.Location) -> str:
     return lines[location[0] - 1]
 
 
+def create_parsing_failure_message(
+    file: TextIO,
+    stream: Sequence[concat.lex.Token],
+    failure: concat.parser_combinators.FailureTree,
+) -> str:
+    location = stream[failure.furthest_index].start
+    line = get_line_at(file, location)
+    message = f'Expected {failure.expected} at line {location[0]}, column {location[1] + 1}:\n{line.rstrip()}\n{" " * location[1] + "^"}'
+    if failure.children:
+        message += '\nbecause:'
+        for f in failure.children:
+            message += '\n' + textwrap.indent(
+                create_parsing_failure_message(file, stream, f), '  '
+            )
+    return message
+
+
 arg_parser = argparse.ArgumentParser(description='Run a Concat program.')
 arg_parser.add_argument(
     'file',
@@ -74,16 +93,28 @@ def get_line_at(file: TextIO, location: concat.astutils.Location) -> str:
     concat.stdlib.repl.repl([], [], args.debug)
 else:
     try:
-        python_ast = transpile(args.file.read(), os.path.dirname(filename))
+        tokens = concat.lex.tokenize(args.file.read())
+        concat_ast = parse(tokens)
+        recovered_parsing_failures = concat_ast.parsing_failures
+        for failure in recovered_parsing_failures:
+            print('Parse Error:')
+            print(create_parsing_failure_message(args.file, tokens, failure))
+        source_dir = os.path.dirname(filename)
+        typecheck(concat_ast, source_dir)
+        python_ast = transpile_ast(concat_ast)
     except concat.typecheck.StaticAnalysisError as e:
         print('Static Analysis Error:\n')
         print(e, 'in line:')
         if e.location:
             print(get_line_at(args.file, e.location), end='')
             print(' ' * e.location[1] + '^')
-    except concat.parse.ParseError as e:
+    except concat.parser_combinators.ParseError as e:
         print('Parse Error:')
-        print(e)
+        print(
+            create_parsing_failure_message(
+                args.file, tokens, e.args[0].failures
+            )
+        )
     except Exception:
         print('An internal error has occurred.')
         print('This is a bug in Concat.')
@@ -94,7 +125,9 @@ def get_line_at(file: TextIO, location: concat.astutils.Location) -> str:
             python_ast,
             {},
             should_log_stacks=args.debug,
-            import_resolution_start_directory=os.path.dirname(filename),
+            import_resolution_start_directory=source_dir,
         )
+        if list(concat_ast.parsing_failures):
+            sys.exit(1)
     finally:
         args.file.close()

concat/execute.py

@@ -14,7 +14,8 @@
 import contextlib
 import sys
 import types
-from typing import Callable, Dict, Iterator, List, Optional
+from typing import Callable, Dict, Iterator, List, Optional, Union, cast
+from typing_extensions import SupportsIndex
 import os
 
 
@@ -39,7 +40,7 @@ def append(self, val: object) -> None:
             self._name, ':: after push (.append):', self
         )
 
-    def pop(self, i: int = -1) -> object:
+    def pop(self, i: SupportsIndex = -1) -> object:
         r = super().pop(i)
         self._should_log and print(self._name, ':: after pop (.pop):', self)
         return r
@@ -59,7 +60,7 @@ def _compile(filename: str, ast_: ast.Module) -> types.CodeType:
 
 def _run(
     prog: types.CodeType,
-    import_resolution_start_directory: os.PathLike,
+    import_resolution_start_directory: Union[os.PathLike, str],
     globals: Optional[Dict[str, object]] = None,
     locals: Optional[Dict[str, object]] = None,
 ) -> None:
@@ -72,12 +73,15 @@ def _run(
     except Exception as e:
         # TODO: throw away all of the traceback outside the code, but have an
         # option to keep the traceback.
-        raise ConcatRuntimeError(globals['stack'], globals['stash']) from e
+        raise ConcatRuntimeError(
+            cast(List[object], globals['stack']),
+            cast(List[object], globals['stash']),
+        ) from e
 
 
 @contextlib.contextmanager
 def _override_import_resolution_start_directory(
-    import_resolution_start_directory: os.PathLike,
+    import_resolution_start_directory: Union[os.PathLike, str],
 ) -> Iterator[None]:
     # This is similar to how Python sets sys.path[0] when running just a file
     # (like `python blah.py`, not `python -m blah`). This should probably be
@@ -180,6 +184,11 @@ def push_func(stack: List[object], _: List[object]):
     globals.setdefault('-', lambda s, _: s.append(s.pop(-2) - s.pop()))
     globals.setdefault('+', lambda s, _: s.append(s.pop(-2) + s.pop()))
 
+    def hole(stack: List[object], stash: List[object]) -> None:
+        raise Exception('Hole generated from parse error reached')
+
+    globals.setdefault('@@concat_parse_error_hole', hole)
+
 
 def execute(
     filename: str,
@@ -188,7 +197,7 @@ def execute(
     locals: Optional[Dict[str, object]] = None,
     should_log_stacks=False,
     # By default, sys.path[0] is '' when executing a package.
-    import_resolution_start_directory: os.PathLike = '',
+    import_resolution_start_directory: Union[os.PathLike, str] = '',
 ) -> None:
     _do_preamble(globals, should_log_stacks)