diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 00000000..b2e707a9 --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,20 @@ +module.exports = { + 'env': { + 'commonjs': true, + 'es2021': true, + }, + 'extends': 'google', + 'overrides': [ + ], + 'parserOptions': { + 'ecmaVersion': 'latest', + 'sourceType': 'module', + }, + 'rules': { + 'indent': ['error', 2, {'SwitchCase': 1}], + 'max-len': [ + 'error', + {'code': 120, 'ignoreComments': true, 'ignoreUrls': true, 'ignoreStrings': true}, + ], + }, +}; diff --git a/.gitattributes b/.gitattributes index f60d7b97..49e20c96 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,7 @@ -/src/** linguist-vendored +/src/parser.c linguist-vendored +/src/*.json linguist-vendored /examples/* linguist-vendored + +src/grammar.json -diff +src/node-types.json -diff +src/parser.c -diff diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c460fa8..a5a381eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: Build/test +name: CI on: pull_request: branches: @@ -14,18 +14,18 @@ jobs: matrix: os: [macos-latest, ubuntu-latest] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 - run: npm install - run: npm test test_windows: runs-on: windows-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 - run: npm install - run: npm run-script test-windows diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 00000000..f503b443 --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,22 @@ +name: Fuzz Parser + +on: + push: + paths: + - src/scanner.c + pull_request: + paths: + - src/scanner.c + workflow_dispatch: + +jobs: + test: + name: Parser fuzzing + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: vigoux/tree-sitter-fuzz-action@v1 + with: + language: python + external-scanner: src/scanner.c + time: 60 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..d94f7f39 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,19 @@ +name: Lint + +on: + push: + branches: + - master + pull_request: + branches: + - "**" + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install modules + run: npm install + - name: Run ESLint + run: npm run lint diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..c2020e92 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,103 @@ +name: Release + +on: + workflow_run: + workflows: ["CI"] + branches: + - master + types: + - completed + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get previous commit SHA + id: get_previous_commit + run: | + LATEST_TAG=$(git describe --tags --abbrev=0) + if [[ -z "$LATEST_TAG" ]]; then + echo "No tag found. Failing..." + exit 1 + fi + echo "latest_tag=${LATEST_TAG#v}" >> "$GITHUB_ENV" # Remove 'v' prefix from the tag + + - name: Check if version changed and is greater than the previous + id: version_check + run: | + # Compare the current version with the version from the previous commit + PREVIOUS_NPM_VERSION=${{ env.latest_tag }} + CURRENT_NPM_VERSION=$(jq -r '.version' package.json) + CURRENT_CARGO_VERSION=$(awk -F '"' '/^version/ {print $2}' Cargo.toml) + if [[ "$CURRENT_NPM_VERSION" != "$CURRENT_CARGO_VERSION" ]]; then # Cargo.toml and package.json versions must match + echo "Mismatch: NPM version ($CURRENT_NPM_VERSION) and Cargo.toml version ($CURRENT_CARGO_VERSION)" + echo "version_changed=false" >> "$GITHUB_ENV" + else + if [[ "$PREVIOUS_NPM_VERSION" == "$CURRENT_NPM_VERSION" ]]; then + echo "version_changed=" >> "$GITHUB_ENV" + else + IFS='.' read -ra PREVIOUS_VERSION_PARTS <<< "$PREVIOUS_NPM_VERSION" + IFS='.' read -ra CURRENT_VERSION_PARTS <<< "$CURRENT_NPM_VERSION" + VERSION_CHANGED=false + for i in "${!PREVIOUS_VERSION_PARTS[@]}"; do + if [[ ${CURRENT_VERSION_PARTS[i]} -gt ${PREVIOUS_VERSION_PARTS[i]} ]]; then + VERSION_CHANGED=true + break + elif [[ ${CURRENT_VERSION_PARTS[i]} -lt ${PREVIOUS_VERSION_PARTS[i]} ]]; then + break + fi + done + + echo "version_changed=$VERSION_CHANGED" >> "$GITHUB_ENV" + echo "current_version=${CURRENT_NPM_VERSION}" >> "$GITHUB_ENV" + fi + fi + + - name: Display result + run: | + echo "Version bump detected: ${{ env.version_changed }}" + + - name: Fail if version is lower + if: env.version_changed == 'false' + run: exit 1 + + - name: Setup Node + if: env.version_changed == 'true' + uses: actions/setup-node@v3 + with: + node-version: 18 + registry-url: "https://registry.npmjs.org" + - name: Publish to NPM + if: env.version_changed == 'true' + env: + NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} + run: npm publish + + - name: Setup Rust + if: env.version_changed == 'true' + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - name: Publish to Crates.io + if: env.version_changed == 'true' + uses: katyo/publish-crates@v2 + with: + registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }} + + - name: Tag versions + if: env.version_changed == 'true' + run: | + git checkout master + git config user.name github-actions[bot] + git config user.email github-actions[bot]@users.noreply.github.com + git tag -d "v${{ env.current_version }}" || true + git push origin --delete "v${{ env.current_version }}" || true + git tag -a "v${{ env.current_version }}" -m "Version ${{ env.current_version }}" + git push origin "v${{ env.current_version }}" diff --git a/.npmignore b/.npmignore index 8e142f74..0f438b55 100644 --- a/.npmignore +++ b/.npmignore @@ -1,6 +1,6 @@ -corpus -examples -build -script -target +/test +/examples +/build +/script +/target bindings/rust diff --git a/README.md b/README.md index a7cb44c7..ea884e26 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -tree-sitter-python -================== +# tree-sitter-python [![build](https://github.com/tree-sitter/tree-sitter-python/actions/workflows/ci.yml/badge.svg)](https://github.com/tree-sitter/tree-sitter-python/actions/workflows/ci.yml) @@ -7,7 +6,7 @@ Python grammar for [tree-sitter][]. [tree-sitter]: https://github.com/tree-sitter/tree-sitter -#### References +## References -* [Python 2 Grammar](https://docs.python.org/2/reference/grammar.html) -* [Python 3 Grammar](https://docs.python.org/3/reference/grammar.html) +- [Python 2 Grammar](https://docs.python.org/2/reference/grammar.html) +- [Python 3 Grammar](https://docs.python.org/3/reference/grammar.html) diff --git a/bindings/rust/README.md b/bindings/rust/README.md index 66976d46..6f10918e 100644 --- a/bindings/rust/README.md +++ b/bindings/rust/README.md @@ -2,11 +2,11 @@ This crate provides a Python grammar for the [tree-sitter][] parsing library. To use this crate, add it to the `[dependencies]` section of your `Cargo.toml` -file. (Note that you will probably also need to depend on the +file. (Note that you will probably also need to depend on the [`tree-sitter`][tree-sitter crate] crate to use the parsed result in any useful way.) -``` toml +```toml [dependencies] tree-sitter = "0.17" tree-sitter-python = "0.17" @@ -15,7 +15,7 @@ tree-sitter-python = "0.17" Typically, you will use the [language][language func] function to add this grammar to a tree-sitter [Parser][], and then use the parser to parse some code: -``` rust +```rust let code = r#" def double(x): return x * 2 diff --git a/bindings/rust/lib.rs b/bindings/rust/lib.rs index 7a58509e..71bc80bd 100644 --- a/bindings/rust/lib.rs +++ b/bindings/rust/lib.rs @@ -43,18 +43,18 @@ pub fn language() -> Language { } /// The source of the Python tree-sitter grammar description. -pub const GRAMMAR: &'static str = include_str!("../../grammar.js"); +pub const GRAMMAR: &str = include_str!("../../grammar.js"); /// The syntax highlighting query for this language. -pub const HIGHLIGHT_QUERY: &'static str = include_str!("../../queries/highlights.scm"); +pub const HIGHLIGHT_QUERY: &str = include_str!("../../queries/highlights.scm"); /// The content of the [`node-types.json`][] file for this grammar. /// /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types -pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json"); +pub const NODE_TYPES: &str = include_str!("../../src/node-types.json"); /// The symbol tagging query for this language. -pub const TAGGING_QUERY: &'static str = include_str!("../../queries/tags.scm"); +pub const TAGGING_QUERY: &str = include_str!("../../queries/tags.scm"); #[cfg(test)] mod tests { diff --git a/grammar.js b/grammar.js index a2c2d1a1..6566200e 100644 --- a/grammar.js +++ b/grammar.js @@ -1,3 +1,17 @@ +/** + * @file Python grammar for tree-sitter + * @author Max Brunsfeld + * @license MIT + * @see {@link https://docs.python.org/2/reference/grammar.html|Python 2 grammar} + * @see {@link https://docs.python.org/3/reference/grammar.html|Python 3 grammar} + */ + +/* eslint-disable arrow-parens */ +/* eslint-disable camelcase */ +/* eslint-disable-next-line spaced-comment */ +/// +// @ts-check + const PREC = { // this resolves a conflict between the usage of ':' in a lambda vs in a // typed parameter. In the case of a lambda, we don't allow typed parameters. @@ -20,9 +34,9 @@ const PREC = { unary: 20, power: 21, call: 22, -} +}; -const SEMICOLON = ';' +const SEMICOLON = ';'; module.exports = grammar({ name: 'python', @@ -91,7 +105,7 @@ module.exports = grammar({ _statement: $ => choice( $._simple_statements, - $._compound_statement + $._compound_statement, ), // Simple statements @@ -99,7 +113,7 @@ module.exports = grammar({ _simple_statements: $ => seq( sep1($._simple_statement, SEMICOLON), optional(SEMICOLON), - $._newline + $._newline, ), _simple_statement: $ => choice( @@ -117,19 +131,19 @@ module.exports = grammar({ $.continue_statement, $.global_statement, $.nonlocal_statement, - $.exec_statement + $.exec_statement, ), import_statement: $ => seq( 'import', - $._import_list + $._import_list, ), - import_prefix: $ => repeat1('.'), + import_prefix: _ => repeat1('.'), relative_import: $ => seq( $.import_prefix, - optional($.dotted_name) + optional($.dotted_name), ), future_import_statement: $ => seq( @@ -139,61 +153,61 @@ module.exports = grammar({ choice( $._import_list, seq('(', $._import_list, ')'), - ) + ), ), import_from_statement: $ => seq( 'from', field('module_name', choice( $.relative_import, - $.dotted_name + $.dotted_name, )), 'import', choice( $.wildcard_import, $._import_list, - seq('(', $._import_list, ')') - ) + seq('(', $._import_list, ')'), + ), ), _import_list: $ => seq( commaSep1(field('name', choice( $.dotted_name, - $.aliased_import + $.aliased_import, ))), - optional(',') + optional(','), ), aliased_import: $ => seq( field('name', $.dotted_name), 'as', - field('alias', $.identifier) + field('alias', $.identifier), ), - wildcard_import: $ => '*', + wildcard_import: _ => '*', print_statement: $ => choice( prec(1, seq( 'print', $.chevron, repeat(seq(',', field('argument', $.expression))), - optional(',')) + optional(',')), ), prec(-3, prec.dynamic(-1, seq( 'print', commaSep1(field('argument', $.expression)), - optional(',') + optional(','), ))), ), chevron: $ => seq( '>>', - $.expression + $.expression, ), assert_statement: $ => seq( 'assert', - commaSep1($.expression) + commaSep1($.expression), ), expression_statement: $ => choice( @@ -201,44 +215,44 @@ module.exports = grammar({ seq(commaSep1($.expression), optional(',')), $.assignment, $.augmented_assignment, - $.yield + $.yield, ), named_expression: $ => seq( field('name', $._named_expression_lhs), ':=', - field('value', $.expression) + field('value', $.expression), ), _named_expression_lhs: $ => choice( $.identifier, - $.keyword_identifier + $.keyword_identifier, ), return_statement: $ => seq( 'return', - optional($._expressions) + optional($._expressions), ), delete_statement: $ => seq( 'del', - $._expressions + $._expressions, ), _expressions: $ => choice( $.expression, - $.expression_list + $.expression_list, ), raise_statement: $ => seq( 'raise', optional($._expressions), - optional(seq('from', field('cause', $.expression))) + optional(seq('from', field('cause', $.expression))), ), - pass_statement: $ => prec.left('pass'), - break_statement: $ => prec.left('break'), - continue_statement: $ => prec.left('continue'), + pass_statement: _ => prec.left('pass'), + break_statement: _ => prec.left('break'), + continue_statement: _ => prec.left('continue'), // Compound statements @@ -260,20 +274,20 @@ module.exports = grammar({ ':', field('consequence', $._suite), repeat(field('alternative', $.elif_clause)), - optional(field('alternative', $.else_clause)) + optional(field('alternative', $.else_clause)), ), elif_clause: $ => seq( 'elif', field('condition', $.expression), ':', - field('consequence', $._suite) + field('consequence', $._suite), ), else_clause: $ => seq( 'else', ':', - field('body', $._suite) + field('body', $._suite), ), match_statement: $ => seq( @@ -299,12 +313,12 @@ module.exports = grammar({ field( 'pattern', alias($.expression, $.case_pattern), - ) + ), ), optional(','), optional(field('guard', $.if_clause)), ':', - field('consequence', $._suite) + field('consequence', $._suite), ), for_statement: $ => seq( @@ -315,7 +329,7 @@ module.exports = grammar({ field('right', $._expressions), ':', field('body', $._suite), - field('alternative', optional($.else_clause)) + field('alternative', optional($.else_clause)), ), while_statement: $ => seq( @@ -323,7 +337,7 @@ module.exports = grammar({ field('condition', $.expression), ':', field('body', $._suite), - optional(field('alternative', $.else_clause)) + optional(field('alternative', $.else_clause)), ), try_statement: $ => seq( @@ -334,15 +348,15 @@ module.exports = grammar({ seq( repeat1($.except_clause), optional($.else_clause), - optional($.finally_clause) + optional($.finally_clause), ), seq( repeat1($.except_group_clause), optional($.else_clause), - optional($.finally_clause) + optional($.finally_clause), ), - $.finally_clause - ) + $.finally_clause, + ), ), except_clause: $ => seq( @@ -351,11 +365,11 @@ module.exports = grammar({ $.expression, optional(seq( choice('as', ','), - $.expression - )) + $.expression, + )), )), ':', - $._suite + $._suite, ), except_group_clause: $ => seq( @@ -364,17 +378,17 @@ module.exports = grammar({ $.expression, optional(seq( 'as', - $.expression - )) + $.expression, + )), ), ':', - $._suite + $._suite, ), finally_clause: $ => seq( 'finally', ':', - $._suite + $._suite, ), with_statement: $ => seq( @@ -382,12 +396,12 @@ module.exports = grammar({ 'with', $.with_clause, ':', - field('body', $._suite) + field('body', $._suite), ), with_clause: $ => choice( seq(commaSep1($.with_item), optional(',')), - seq('(', commaSep1($.with_item), optional(','), ')') + seq('(', commaSep1($.with_item), optional(','), ')'), ), with_item: $ => prec.dynamic(1, seq( @@ -402,17 +416,17 @@ module.exports = grammar({ optional( seq( '->', - field('return_type', $.type) - ) + field('return_type', $.type), + ), ), ':', - field('body', $._suite) + field('body', $._suite), ), parameters: $ => seq( '(', optional($._parameters), - ')' + ')', ), lambda_parameters: $ => $._parameters, @@ -424,17 +438,17 @@ module.exports = grammar({ dictionary_splat: $ => seq( '**', - $.expression + $.expression, ), global_statement: $ => seq( 'global', - commaSep1($.identifier) + commaSep1($.identifier), ), nonlocal_statement: $ => seq( 'nonlocal', - commaSep1($.identifier) + commaSep1($.identifier), ), exec_statement: $ => seq( @@ -443,9 +457,9 @@ module.exports = grammar({ optional( seq( 'in', - commaSep1($.expression) - ) - ) + commaSep1($.expression), + ), + ), ), class_definition: $ => seq( @@ -453,7 +467,7 @@ module.exports = grammar({ field('name', $.identifier), field('superclasses', optional($.argument_list)), ':', - field('body', $._suite) + field('body', $._suite), ), parenthesized_list_splat: $ => prec(PREC.parenthesized_list_splat, seq( @@ -473,36 +487,36 @@ module.exports = grammar({ $.list_splat, $.dictionary_splat, alias($.parenthesized_list_splat, $.parenthesized_expression), - $.keyword_argument - ) + $.keyword_argument, + ), )), optional(','), - ')' + ')', ), decorated_definition: $ => seq( repeat1($.decorator), field('definition', choice( $.class_definition, - $.function_definition - )) + $.function_definition, + )), ), decorator: $ => seq( '@', $.expression, - $._newline + $._newline, ), _suite: $ => choice( alias($._simple_statements, $.block), seq($._indent, $.block), - alias($._newline, $.block) + alias($._newline, $.block), ), block: $ => seq( repeat($._statement), - $._dedent + $._dedent, ), expression_list: $ => prec.right(seq( @@ -512,11 +526,11 @@ module.exports = grammar({ seq( repeat1(seq( ',', - $.expression + $.expression, )), - optional(',') + optional(','), ), - ) + ), )), dotted_name: $ => sep1($.identifier, '.'), @@ -525,12 +539,12 @@ module.exports = grammar({ _parameters: $ => seq( commaSep1($.parameter), - optional(',') + optional(','), ), _patterns: $ => seq( commaSep1($.pattern), - optional(',') + optional(','), ), parameter: $ => choice( @@ -542,7 +556,7 @@ module.exports = grammar({ $.tuple_pattern, $.keyword_separator, $.positional_separator, - $.dictionary_splat_pattern + $.dictionary_splat_pattern, ), pattern: $ => choice( @@ -552,25 +566,25 @@ module.exports = grammar({ $.attribute, $.list_splat_pattern, $.tuple_pattern, - $.list_pattern + $.list_pattern, ), tuple_pattern: $ => seq( '(', optional($._patterns), - ')' + ')', ), list_pattern: $ => seq( '[', optional($._patterns), - ']' + ']', ), default_parameter: $ => seq( field('name', choice($.identifier, $.tuple_pattern)), '=', - field('value', $.expression) + field('value', $.expression), ), typed_default_parameter: $ => prec(PREC.typed_parameter, seq( @@ -578,17 +592,17 @@ module.exports = grammar({ ':', field('type', $.type), '=', - field('value', $.expression) + field('value', $.expression), )), list_splat_pattern: $ => seq( '*', - choice($.identifier, $.keyword_identifier, $.subscript, $.attribute) + choice($.identifier, $.keyword_identifier, $.subscript, $.attribute), ), dictionary_splat_pattern: $ => seq( '**', - choice($.identifier, $.keyword_identifier, $.subscript, $.attribute) + choice($.identifier, $.keyword_identifier, $.subscript, $.attribute), ), // Extended patterns (patterns allowed in match statement are far more flexible than simple patterns though still a subset of "expression") @@ -596,14 +610,14 @@ module.exports = grammar({ as_pattern: $ => prec.left(seq( $.expression, 'as', - field('alias', alias($.expression, $.as_pattern_target)) + field('alias', alias($.expression, $.as_pattern_target)), )), // Expressions _expression_within_for_in_clause: $ => choice( $.expression, - alias($.lambda_within_for_in_clause, $.lambda) + alias($.lambda_within_for_in_clause, $.lambda), ), expression: $ => choice( @@ -648,20 +662,20 @@ module.exports = grammar({ not_operator: $ => prec(PREC.not, seq( 'not', - field('argument', $.expression) + field('argument', $.expression), )), boolean_operator: $ => choice( prec.left(PREC.and, seq( field('left', $.expression), field('operator', 'and'), - field('right', $.expression) + field('right', $.expression), )), prec.left(PREC.or, seq( field('left', $.expression), field('operator', 'or'), - field('right', $.expression) - )) + field('right', $.expression), + )), ), binary_operator: $ => { @@ -681,16 +695,18 @@ module.exports = grammar({ [prec.left, '>>', PREC.shift], ]; + // @ts-ignore return choice(...table.map(([fn, operator, precedence]) => fn(precedence, seq( field('left', $.primary_expression), + // @ts-ignore field('operator', operator), - field('right', $.primary_expression) + field('right', $.primary_expression), )))); }, unary_operator: $ => prec(PREC.unary, seq( field('operator', choice('+', '-', '~')), - field('argument', $.primary_expression) + field('argument', $.primary_expression), )), comparison_operator: $ => prec.left(PREC.compare, seq( @@ -708,24 +724,24 @@ module.exports = grammar({ 'in', alias(seq('not', 'in'), 'not in'), 'is', - alias(seq('is', 'not'), 'is not') + alias(seq('is', 'not'), 'is not'), )), - $.primary_expression - )) + $.primary_expression, + )), )), lambda: $ => prec(PREC.lambda, seq( 'lambda', field('parameters', optional($.lambda_parameters)), ':', - field('body', $.expression) + field('body', $.expression), )), lambda_within_for_in_clause: $ => seq( 'lambda', field('parameters', optional($.lambda_parameters)), ':', - field('body', $._expression_within_for_in_clause) + field('body', $._expression_within_for_in_clause), ), assignment: $ => seq( @@ -733,17 +749,17 @@ module.exports = grammar({ choice( seq('=', field('right', $._right_hand_side)), seq(':', field('type', $.type)), - seq(':', field('type', $.type), '=', field('right', $._right_hand_side)) - ) + seq(':', field('type', $.type), '=', field('right', $._right_hand_side)), + ), ), augmented_assignment: $ => seq( field('left', $._left_hand_side), field('operator', choice( '+=', '-=', '*=', '/=', '@=', '//=', '%=', '**=', - '>>=', '<<=', '&=', '^=', '|=' + '>>=', '<<=', '&=', '^=', '|=', )), - field('right', $._right_hand_side) + field('right', $._right_hand_side), ), _left_hand_side: $ => choice( @@ -758,11 +774,11 @@ module.exports = grammar({ seq( repeat1(seq( ',', - $.pattern + $.pattern, )), - optional(',') - ) - ) + optional(','), + ), + ), ), _right_hand_side: $ => choice( @@ -771,7 +787,7 @@ module.exports = grammar({ $.assignment, $.augmented_assignment, $.pattern_list, - $.yield + $.yield, ), yield: $ => prec.right(seq( @@ -779,16 +795,16 @@ module.exports = grammar({ choice( seq( 'from', - $.expression + $.expression, ), - optional($._expressions) - ) + optional($._expressions), + ), )), attribute: $ => prec(PREC.call, seq( field('object', $.primary_expression), '.', - field('attribute', $.identifier) + field('attribute', $.identifier), )), subscript: $ => prec(PREC.call, seq( @@ -796,34 +812,34 @@ module.exports = grammar({ '[', commaSep1(field('subscript', choice($.expression, $.slice))), optional(','), - ']' + ']', )), slice: $ => seq( optional($.expression), ':', optional($.expression), - optional(seq(':', optional($.expression))) + optional(seq(':', optional($.expression))), ), - ellipsis: $ => '...', + ellipsis: _ => '...', call: $ => prec(PREC.call, seq( field('function', $.primary_expression), field('arguments', choice( $.generator_expression, - $.argument_list - )) + $.argument_list, + )), )), typed_parameter: $ => prec(PREC.typed_parameter, seq( choice( $.identifier, $.list_splat_pattern, - $.dictionary_splat_pattern + $.dictionary_splat_pattern, ), ':', - field('type', $.type) + field('type', $.type), )), type: $ => $.expression, @@ -831,7 +847,7 @@ module.exports = grammar({ keyword_argument: $ => seq( field('name', choice($.identifier, $.keyword_identifier)), '=', - field('value', $.expression) + field('value', $.expression), ), // Literals @@ -839,81 +855,81 @@ module.exports = grammar({ list: $ => seq( '[', optional($._collection_elements), - ']' + ']', ), set: $ => seq( '{', $._collection_elements, - '}' + '}', ), tuple: $ => seq( '(', optional($._collection_elements), - ')' + ')', ), dictionary: $ => seq( '{', optional(commaSep1(choice($.pair, $.dictionary_splat))), optional(','), - '}' + '}', ), pair: $ => seq( field('key', $.expression), ':', - field('value', $.expression) + field('value', $.expression), ), list_comprehension: $ => seq( '[', field('body', $.expression), $._comprehension_clauses, - ']' + ']', ), dictionary_comprehension: $ => seq( '{', field('body', $.pair), $._comprehension_clauses, - '}' + '}', ), set_comprehension: $ => seq( '{', field('body', $.expression), $._comprehension_clauses, - '}' + '}', ), generator_expression: $ => seq( '(', field('body', $.expression), $._comprehension_clauses, - ')' + ')', ), _comprehension_clauses: $ => seq( $.for_in_clause, repeat(choice( $.for_in_clause, - $.if_clause - )) + $.if_clause, + )), ), parenthesized_expression: $ => prec(PREC.parenthesized_expression, seq( '(', choice($.expression, $.yield), - ')' + ')', )), _collection_elements: $ => seq( commaSep1(choice( - $.expression, $.yield, $.list_splat, $.parenthesized_list_splat + $.expression, $.yield, $.list_splat, $.parenthesized_list_splat, )), - optional(',') + optional(','), ), for_in_clause: $ => prec.left(seq( @@ -922,12 +938,12 @@ module.exports = grammar({ field('left', $._left_hand_side), 'in', field('right', commaSep1($._expression_within_for_in_clause)), - optional(',') + optional(','), )), if_clause: $ => seq( 'if', - $.expression + $.expression, ), conditional_expression: $ => prec.right(PREC.conditional, seq( @@ -935,12 +951,12 @@ module.exports = grammar({ 'if', $.expression, 'else', - $.expression + $.expression, )), concatenated_string: $ => seq( $.string, - repeat1($.string) + repeat1($.string), ), string: $ => seq( @@ -954,7 +970,7 @@ module.exports = grammar({ $._escape_interpolation, $.escape_sequence, $._not_escape_sequence, - $._string_content + $._string_content, ))), interpolation: $ => seq( @@ -963,7 +979,7 @@ module.exports = grammar({ optional('='), optional(field('type_conversion', $.type_conversion)), optional(field('format_specifier', $.format_specifier)), - '}' + '}', ), _f_expression: $ => choice( @@ -973,9 +989,9 @@ module.exports = grammar({ $.yield, ), - _escape_interpolation: $ => token.immediate(choice('{{', '}}')), + _escape_interpolation: _ => token.immediate(choice('{{', '}}')), - escape_sequence: $ => token.immediate(prec(1, seq( + escape_sequence: _ => token.immediate(prec(1, seq( '\\', choice( /u[a-fA-F\d]{4}/, @@ -985,61 +1001,61 @@ module.exports = grammar({ /\r?\n/, /['"abfrntv\\]/, /N\{[^}]+\}/, - ) + ), ))), - _not_escape_sequence: $ => token.immediate('\\'), + _not_escape_sequence: _ => token.immediate('\\'), format_specifier: $ => seq( ':', repeat(choice( token(prec(1, /[^{}\n]+/)), - alias($.interpolation, $.format_expression) - )) + alias($.interpolation, $.format_expression), + )), ), - type_conversion: $ => /![a-z]/, + type_conversion: _ => /![a-z]/, - integer: $ => token(choice( + integer: _ => token(choice( seq( choice('0x', '0X'), repeat1(/_?[A-Fa-f0-9]+/), - optional(/[Ll]/) + optional(/[Ll]/), ), seq( choice('0o', '0O'), repeat1(/_?[0-7]+/), - optional(/[Ll]/) + optional(/[Ll]/), ), seq( choice('0b', '0B'), repeat1(/_?[0-1]+/), - optional(/[Ll]/) + optional(/[Ll]/), ), seq( repeat1(/[0-9]+_?/), choice( optional(/[Ll]/), // long numbers - optional(/[jJ]/) // complex numbers - ) - ) + optional(/[jJ]/), // complex numbers + ), + ), )), - float: $ => { + float: _ => { const digits = repeat1(/[0-9]+_?/); - const exponent = seq(/[eE][\+-]?/, digits) + const exponent = seq(/[eE][\+-]?/, digits); return token(seq( choice( seq(digits, '.', optional(digits), optional(exponent)), seq(optional(digits), '.', digits, optional(exponent)), - seq(digits, exponent) + seq(digits, exponent), ), - optional(choice(/[Ll]/, /[jJ]/)) - )) + optional(choice(/[Ll]/, /[jJ]/)), + )); }, - identifier: $ => /[_\p{XID_Start}][_\p{XID_Continue}]*/, + identifier: _ => /[_\p{XID_Start}][_\p{XID_Continue}]*/, keyword_identifier: $ => prec(-3, alias( choice( @@ -1047,33 +1063,53 @@ module.exports = grammar({ 'exec', 'async', 'await', - 'match' + 'match', ), - $.identifier + $.identifier, )), - true: $ => 'True', - false: $ => 'False', - none: $ => 'None', + true: _ => 'True', + false: _ => 'False', + none: _ => 'None', await: $ => prec(PREC.unary, seq( 'await', $.primary_expression, )), - comment: $ => token(seq('#', /.*/)), + comment: _ => token(seq('#', /.*/)), + + line_continuation: _ => token(seq('\\', choice(seq(optional('\r'), '\n'), '\0'))), - line_continuation: $ => token(seq('\\', choice(seq(optional('\r'), '\n'), '\0'))), + positional_separator: _ => '/', + keyword_separator: _ => '*', + }, +}); - positional_separator: $ => '/', - keyword_separator: $ => '*', - } -}) +module.exports.PREC = PREC; +/** + * Creates a rule to match one or more of the rules separated by a comma + * + * @param {RegExp|Rule|String} rule + * + * @return {SeqRule} + * + */ function commaSep1(rule) { - return sep1(rule, ',') + return sep1(rule, ','); } +/** + * Creates a rule to match one or more occurrences of `rule` separated by `sep` + * + * @param {RegExp|Rule|String} rule + * + * @param {RegExp|Rule|String} separator + * + * @return {SeqRule} + * + */ function sep1(rule, separator) { - return seq(rule, repeat(seq(separator, rule))) + return seq(rule, repeat(seq(separator, rule))); } diff --git a/package.json b/package.json index 988f983f..330daebc 100644 --- a/package.json +++ b/package.json @@ -13,12 +13,15 @@ "nan": "^2.15.0" }, "devDependencies": { + "eslint": "^8.45.0", + "eslint-config-google": "^0.14.0", "tree-sitter-cli": "^0.20.1" }, "scripts": { "build": "tree-sitter generate && node-gyp build", - "test": "tree-sitter test && script/parse-examples", + "lint": "eslint grammar.js", "parse": "tree-sitter parse", + "test": "tree-sitter test && script/parse-examples", "test-windows": "tree-sitter test" }, "repository": "https://github.com/tree-sitter/tree-sitter-python", diff --git a/queries/highlights.scm b/queries/highlights.scm index 90fcf1c4..8ccf82b8 100644 --- a/queries/highlights.scm +++ b/queries/highlights.scm @@ -66,14 +66,17 @@ "//=" "/=" "&" + "&=" "%" "%=" "^" + "^=" "+" "->" "+=" "<" "<<" + "<<=" "<=" "<>" "=" @@ -82,8 +85,11 @@ ">" ">=" ">>" + ">>=" "|" + "|=" "~" + "@=" "and" "in" "is" diff --git a/src/grammar.json b/src/grammar.json index 5247dd76..77fbc5c8 100644 --- a/src/grammar.json +++ b/src/grammar.json @@ -5471,6 +5471,26 @@ "primary_expression", "pattern", "parameter" - ] + ], + "PREC": { + "lambda": -2, + "typed_parameter": -1, + "conditional": -1, + "parenthesized_expression": 1, + "parenthesized_list_splat": 1, + "or": 10, + "and": 11, + "not": 12, + "compare": 13, + "bitwise_or": 14, + "bitwise_and": 15, + "xor": 16, + "shift": 17, + "plus": 18, + "times": 19, + "unary": 20, + "power": 21, + "call": 22 + } }