From 5c6a0dd03f7f6d8694d1b9a62f9770be44f0b809 Mon Sep 17 00:00:00 2001 From: Artem Malyshev Date: Mon, 26 Mar 2018 23:41:31 +0300 Subject: [PATCH 01/16] Add Emacs text editor integration to the README. (#79) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8aa0d06..d74d436 100644 --- a/README.md +++ b/README.md @@ -231,6 +231,7 @@ body. ### Editor integration * Visual Studio Code: [joslarson.black-vscode](https://marketplace.visualstudio.com/items?itemName=joslarson.black-vscode) +* Emacs: [proofit404/blacken](https://github.com/proofit404/blacken) Any tool that can pipe code through *Black* using its stdio mode (just [use `-` as the file name](http://www.tldp.org/LDP/abs/html/special-chars.html#DASHREF2)). @@ -238,9 +239,8 @@ The formatted code will be returned on stdout (unless `--check` was passed). *Black* will still emit messages on stderr but that shouldn't affect your use case. -There is currently no integration with any other text editors. Vim and -Atom/Nuclide integration is planned by the author, others will require -external contributions. +Vim and Atom/Nuclide integration is planned by the author, others will +require external contributions. Patches welcome! ✨ 🍰 ✨ -- 2.39.5 From 611737f9cc186d3e6463ef774fdbda4f77055d4c Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Mon, 26 Mar 2018 18:37:36 -0700 Subject: [PATCH 02/16] Big documentation deduplication Most is not generated from README.md so we no longer have to remember to update two Change Logs, and so on! If we decide to diverge from the README in Sphinx, that's fine, too. We will just create dedicated documents. --- README.md | 107 +++++++++++++-------- docs/authors.md | 1 + docs/change_log.md | 1 + docs/changelog.md | 93 ------------------ docs/conf.py | 44 +++++++++ docs/contributing_to_black.md | 1 + docs/editor_integration.md | 1 + docs/index.rst | 10 +- docs/installation_and_usage.md | 1 + docs/license.md | 1 + docs/requirements-docs.txt | 2 - docs/show_your_style.md | 1 + docs/technical_philosophy.md | 167 --------------------------------- docs/testimonials.md | 1 + docs/the_black_code_style.md | 1 + docs/usage.md | 64 ------------- 16 files changed, 129 insertions(+), 367 deletions(-) create mode 120000 docs/authors.md create mode 120000 docs/change_log.md delete mode 100644 docs/changelog.md create mode 120000 docs/contributing_to_black.md create mode 120000 docs/editor_integration.md create mode 120000 docs/installation_and_usage.md create mode 120000 docs/license.md delete mode 100644 docs/requirements-docs.txt create mode 120000 docs/show_your_style.md delete mode 100644 docs/technical_philosophy.md create mode 120000 docs/testimonials.md create mode 120000 docs/the_black_code_style.md delete mode 100644 docs/usage.md diff --git a/README.md b/README.md index d74d436..1cd1ef0 100644 --- a/README.md +++ b/README.md @@ -27,33 +27,28 @@ content instead. possible. -## NOTE: This is an early pre-release +## Installation and Usage -*Black* can already successfully format itself and the standard library. -It also sports a decent test suite. However, it is still very new. -Things will probably be wonky for a while. This is made explicit by the -"Alpha" trove classifier, as well as by the "a" in the version number. -What this means for you is that **until the formatter becomes stable, -you should expect some formatting to change in the future**. - -Also, as a temporary safety measure, *Black* will check that the -reformatted code still produces a valid AST that is equivalent to the -original. This slows it down. If you're feeling confident, use -``--fast``. - - -## Installation +### Installation *Black* can be installed by running `pip install black`. It requires Python 3.6.0+ to run but you can reformat Python 2 code with it, too. -*Black* is able to parse all of the new syntax supported on Python 3.6 -but also *effectively all* the Python 2 syntax at the same time. -## Usage +### Usage +To get started right away with sensible defaults: ``` +black {source_file_or_directory} +``` + +### Command line options + +Black doesn't provide many options. You can list them by running +`black --help`: + +```text black [OPTIONS] [SRC]... Options: @@ -78,7 +73,22 @@ Options: used). -## The philosophy behind *Black* +### NOTE: This is an early pre-release + +*Black* can already successfully format itself and the standard library. +It also sports a decent test suite. However, it is still very new. +Things will probably be wonky for a while. This is made explicit by the +"Alpha" trove classifier, as well as by the "a" in the version number. +What this means for you is that **until the formatter becomes stable, +you should expect some formatting to change in the future**. + +Also, as a temporary safety measure, *Black* will check that the +reformatted code still produces a valid AST that is equivalent to the +original. This slows it down. If you're feeling confident, use +``--fast``. + + +## The *Black* code style *Black* reformats entire files in place. It is not configurable. It doesn't take previous formatting into account. It doesn't reformat @@ -87,12 +97,13 @@ recognizes [YAPF](https://github.com/google/yapf)'s block comments to the same effect, as a courtesy for straddling code. -### How *Black* formats files +### How *Black* wraps lines *Black* ignores previous formatting and applies uniform horizontal and vertical whitespace to your code. The rules for horizontal whitespace are pretty obvious and can be summarized as: do whatever -makes `pycodestyle` happy. +makes `pycodestyle` happy. The coding style used by *Black* can be +viewed as a strict subset of PEP 8. As for vertical whitespace, *Black* tries to render one full expression or simple statement per line. If this fits the allotted line length, @@ -160,20 +171,6 @@ between two distinct sections of the code that otherwise share the same indentation level (like the arguments list and the docstring in the example above). -Unnecessary trailing commas are removed if an expression fits in one -line. This makes it 1% more likely that your line won't exceed the -allotted line length limit. - -*Black* avoids spurious vertical whitespace. This is in the spirit of -PEP 8 which says that in-function vertical whitespace should only be -used sparingly. One exception is control flow statements: *Black* will -always emit an extra empty line after ``return``, ``raise``, ``break``, -``continue``, and ``yield``. This is to make changes in control flow -more prominent to readers of your code. - -That's it. The rest of the whitespace formatting rules follow PEP 8 and -are designed to keep `pycodestyle` quiet. - ### Line length @@ -214,6 +211,13 @@ bother you if you overdo it by a few km/h". ### Empty lines +*Black* avoids spurious vertical whitespace. This is in the spirit of +PEP 8 which says that in-function vertical whitespace should only be +used sparingly. One exception is control flow statements: *Black* will +always emit an extra empty line after ``return``, ``raise``, ``break``, +``continue``, and ``yield``. This is to make changes in control flow +more prominent to readers of your code. + *Black* will allow single empty lines left by the original editors, except when they're added within parenthesized expressions. Since such expressions are always reformatted to fit minimal space, this whitespace @@ -228,7 +232,36 @@ entire function, use a docstring or put a leading comment in the function body. -### Editor integration +### Trailing commas + +*Black* will add trailing commas to expressions that are split +by comma where each element is on its own line. This includes function +signatures. + +Unnecessary trailing commas are removed if an expression fits in one +line. This makes it 1% more likely that your line won't exceed the +allotted line length limit. Moreover, in this scenario, if you added +another argument to your call, you'd probably fit it in the same line +anyway. That doesn't make diffs any larger. + +One exception to removing trailing commas is tuple expressions with +just one element. In this case *Black* won't touch the single trailing +comma as this would unexpectedly change the underlying data type. Note +that this is also the case when commas are used while indexing. This is +a tuple in disguise: ```numpy_array[3, ]```. + +One exception to adding trailing commas is function signatures +containing `*`, `*args`, or `**kwargs`. In this case a trailing comma +is only safe to use on Python 3.6. *Black* will detect if your file is +already 3.6+ only and use trailing commas in this situation. If you +wonder how it knows, it looks for f-strings and existing use of trailing +commas in function signatures that have stars in them. In other words, +if you'd like a trailing comma in this situation and *Black* didn't +recognize it was safe to do so, put it there manually and *Black* will +keep it. + + +## Editor integration * Visual Studio Code: [joslarson.black-vscode](https://marketplace.visualstudio.com/items?itemName=joslarson.black-vscode) * Emacs: [proofit404/blacken](https://github.com/proofit404/blacken) @@ -282,7 +315,7 @@ Looks like this: [![Code style: black](https://img.shields.io/badge/code%20style MIT -## Contributing +## Contributing to Black In terms of inspiration, *Black* is about as configurable as *gofmt* and *rustfmt* are. This is deliberate. diff --git a/docs/authors.md b/docs/authors.md new file mode 120000 index 0000000..704a421 --- /dev/null +++ b/docs/authors.md @@ -0,0 +1 @@ +/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/authors.md \ No newline at end of file diff --git a/docs/change_log.md b/docs/change_log.md new file mode 120000 index 0000000..36bf3dc --- /dev/null +++ b/docs/change_log.md @@ -0,0 +1 @@ +/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/change_log.md \ No newline at end of file diff --git a/docs/changelog.md b/docs/changelog.md deleted file mode 100644 index af78900..0000000 --- a/docs/changelog.md +++ /dev/null @@ -1,93 +0,0 @@ -## Change Log - -### 18.3a4 - -* `# fmt: off` and `# fmt: on` are implemented (#5) - -* automatic detection of deprecated Python 2 forms of print statements - and exec statements in the formatted file (#49) - -* use proper spaces for complex expressions in default values of typed - function arguments (#60) - -* only return exit code 1 when --check is used (#50) - -* don't remove single trailing commas from square bracket indexing - (#59) - -* don't omit whitespace if the previous factor leaf wasn't a math - operator (#55) - -* omit extra space in kwarg unpacking if it's the first argument (#46) - -* omit extra space in [Sphinx auto-attribute comments](http://www.sphinx-doc.org/en/stable/ext/autodoc.html#directive-autoattribute) - (#68) - - -### 18.3a3 - -* don't remove single empty lines outside of bracketed expressions - (#19) - -* added ability to pipe formatting from stdin to stdin (#25) - -* restored ability to format code with legacy usage of `async` as - a name (#20, #42) - -* even better handling of numpy-style array indexing (#33, again) - - -### 18.3a2 - -* changed positioning of binary operators to occur at beginning of lines - instead of at the end, following [a recent change to PEP8](https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b) - (#21) - -* ignore empty bracket pairs while splitting. This avoids very weirdly - looking formattings (#34, #35) - -* remove a trailing comma if there is a single argument to a call - -* if top level functions were separated by a comment, don't put four - empty lines after the upper function - -* fixed unstable formatting of newlines with imports - -* fixed unintentional folding of post scriptum standalone comments - into last statement if it was a simple statement (#18, #28) - -* fixed missing space in numpy-style array indexing (#33) - -* fixed spurious space after star-based unary expressions (#31) - - -### 18.3a1 - -* added `--check` - -* only put trailing commas in function signatures and calls if it's - safe to do so. If the file is Python 3.6+ it's always safe, otherwise - only safe if there are no `*args` or `**kwargs` used in the signature - or call. (#8) - -* fixed invalid spacing of dots in relative imports (#6, #13) - -* fixed invalid splitting after comma on unpacked variables in for-loops - (#23) - -* fixed spurious space in parenthesized set expressions (#7) - -* fixed spurious space after opening parentheses and in default - arguments (#14, #17) - -* fixed spurious space after unary operators when the operand was - a complex expression (#15) - - -### 18.3a0 - -* first published version, Happy 🍰 Day 2018! - -* alpha quality - -* date-versioned (see: https://calver.org/) diff --git a/docs/conf.py b/docs/conf.py index 5d0e9e3..6edddba 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,6 +15,7 @@ import ast from pathlib import Path import re +import shutil import string from recommonmark.parser import CommonMarkParser @@ -40,6 +41,48 @@ def make_pypi_svg(version): f.write(svg) +def make_filename(line): + non_letters = re.compile(r'[^a-z]+') + filename = line[3:].rstrip().lower() + filename = non_letters.sub('_', filename) + if filename.startswith('_'): + filename = filename[1:] + if filename.endswith('_'): + filename = filename[:-1] + return filename + '.md' + + +def generate_sections_from_readme(): + target_dir = CURRENT_DIR / '_build' / 'generated' + readme = CURRENT_DIR / '..' / 'README.md' + shutil.rmtree(str(target_dir), ignore_errors=True) + target_dir.mkdir(parents=True) + + output = None + with open(str(readme), 'r', encoding='utf8') as f: + for line in f: + if line.startswith('## '): + if output is not None: + output.close() + filename = make_filename(line) + output_path = CURRENT_DIR / filename + if output_path.is_symlink() or output_path.is_file(): + output_path.unlink() + output_path.symlink_to(target_dir / filename) + output = open(str(output_path), 'w', encoding='utf8') + output.write( + '[//]: # (NOTE: THIS FILE IS AUTOGENERATED FROM README.md)\n\n' + ) + + if output is None: + continue + + if line.startswith('##'): + line = line[1:] + + output.write(line) + + # -- Project information ----------------------------------------------------- project = 'Black' @@ -54,6 +97,7 @@ version = release for sp in 'abcfr': version = version.split(sp)[0] make_pypi_svg(release) +generate_sections_from_readme() # -- General configuration --------------------------------------------------- diff --git a/docs/contributing_to_black.md b/docs/contributing_to_black.md new file mode 120000 index 0000000..079bd4a --- /dev/null +++ b/docs/contributing_to_black.md @@ -0,0 +1 @@ +/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/contributing_to_black.md \ No newline at end of file diff --git a/docs/editor_integration.md b/docs/editor_integration.md new file mode 120000 index 0000000..e234140 --- /dev/null +++ b/docs/editor_integration.md @@ -0,0 +1 @@ +/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/editor_integration.md \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 7a0995b..9422402 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,7 +16,7 @@ can focus on the content instead. .. note:: - `Black is an early pre-release `_. + `Black is an early pre-release `_. Testimonials @@ -46,10 +46,12 @@ Contents .. toctree:: :maxdepth: 2 - usage - technical_philosophy + installation_and_usage + the_black_code_style + editor_integration contributing - changelog + change_log + authors Indices and tables diff --git a/docs/installation_and_usage.md b/docs/installation_and_usage.md new file mode 120000 index 0000000..64caa30 --- /dev/null +++ b/docs/installation_and_usage.md @@ -0,0 +1 @@ +/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/installation_and_usage.md \ No newline at end of file diff --git a/docs/license.md b/docs/license.md new file mode 120000 index 0000000..cf360a1 --- /dev/null +++ b/docs/license.md @@ -0,0 +1 @@ +/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/license.md \ No newline at end of file diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt deleted file mode 100644 index 5e5376b..0000000 --- a/docs/requirements-docs.txt +++ /dev/null @@ -1,2 +0,0 @@ -sphinx>=1.7 -recommonmark==0.4.0 \ No newline at end of file diff --git a/docs/show_your_style.md b/docs/show_your_style.md new file mode 120000 index 0000000..15ad1c1 --- /dev/null +++ b/docs/show_your_style.md @@ -0,0 +1 @@ +/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/show_your_style.md \ No newline at end of file diff --git a/docs/technical_philosophy.md b/docs/technical_philosophy.md deleted file mode 100644 index 1052f0c..0000000 --- a/docs/technical_philosophy.md +++ /dev/null @@ -1,167 +0,0 @@ -# Technical Overview - -## The philosophy behind *Black* - -*Black* reformats entire files in place. It is not configurable. It -doesn't take previous formatting into account. It doesn't reformat -blocks that start with `# fmt: off` and end with `# fmt: on`. It also -recognizes [YAPF](https://github.com/google/yapf)'s block comments to -the same effect, as a courtesy for straddling code. - -## How *Black* formats files - -*Black* ignores previous formatting and applies uniform horizontal -and vertical whitespace to your code. The rules for horizontal -whitespace are pretty obvious and can be summarized as: do whatever -makes `pycodestyle` happy. - -As for vertical whitespace, *Black* tries to render one full expression -or simple statement per line. If this fits the allotted line length, -great. - -```py3 -# in: - -l = [1, - 2, - 3, -] - -# out: - -l = [1, 2, 3] -``` - -If not, *Black* will look at the contents of the first outer matching -brackets and put that in a separate indented line. - -```py3 -# in: - -l = [[n for n in list_bosses()], [n for n in list_employees()]] - -# out: - -l = [ - [n for n in list_bosses()], [n for n in list_employees()] -] -``` - -If that still doesn't fit the bill, it will decompose the internal -expression further using the same rule, indenting matching brackets -every time. If the contents of the matching brackets pair are -comma-separated (like an argument list, or a dict literal, and so on) -then *Black* will first try to keep them on the same line with the -matching brackets. If that doesn't work, it will put all of them in -separate lines. - -```py3 -# in: - -def very_important_function(template: str, *variables, file: os.PathLike, debug: bool = False): - """Applies `variables` to the `template` and writes to `file`.""" - with open(file, 'w') as f: - ... - -# out: - -def very_important_function( - template: str, - *variables, - file: os.PathLike, - debug: bool = False, -): - """Applies `variables` to the `template` and writes to `file`.""" - with open(file, 'w') as f: - ... -``` - -You might have noticed that closing brackets are always dedented and -that a trailing comma is always added. Such formatting produces smaller -diffs; when you add or remove an element, it's always just one line. -Also, having the closing bracket dedented provides a clear delimiter -between two distinct sections of the code that otherwise share the same -indentation level (like the arguments list and the docstring in the -example above). - -Unnecessary trailing commas are removed if an expression fits in one -line. This makes it 1% more likely that your line won't exceed the -allotted line length limit. - -*Black* avoids spurious vertical whitespace. This is in the spirit of -PEP 8 which says that in-function vertical whitespace should only be -used sparingly. One exception is control flow statements: *Black* will -always emit an extra empty line after ``return``, ``raise``, ``break``, -``continue``, and ``yield``. This is to make changes in control flow -more prominent to readers of your code. - -That's it. The rest of the whitespace formatting rules follow PEP 8 and -are designed to keep `pycodestyle` quiet. - -## Line length - -You probably noticed the peculiar default line length. *Black* defaults -to 88 characters per line, which happens to be 10% over 80. This number -was found to produce significantly shorter files than sticking with 80 -(the most popular), or even 79 (used by the standard library). In -general, [90-ish seems like the wise choice](https://youtu.be/wf-BqAjZb8M?t=260). - -If you're paid by the line of code you write, you can pass -`--line-length` with a lower number. *Black* will try to respect that. -However, sometimes it won't be able to without breaking other rules. In -those rare cases, auto-formatted code will exceed your allotted limit. - -You can also increase it, but remember that people with sight disabilities -find it harder to work with line lengths exceeding 100 characters. -It also adversely affects side-by-side diff review on typical screen -resolutions. Long lines also make it harder to present code neatly -in documentation or talk slides. - -If you're using Flake8, you can bump `max-line-length` to 88 and forget -about it. Alternatively, use [Bugbear](https://github.com/PyCQA/flake8-bugbear)'s -B950 warning instead of E501 and keep the max line length at 80 which -you are probably already using. You'd do it like this: - -```ini -[flake8] -max-line-length = 80 -... -select = C,E,F,W,B,B950 -ignore = E501 -``` - -You'll find *Black*'s own .flake8 config file is configured like this. -If you're curious about the reasoning behind B950, Bugbear's documentation -explains it. The tl;dr is "it's like highway speed limits, we won't -bother you if you overdo it by a few km/h". - -## Empty lines - -*Black* will allow single empty lines left by the original editors, -except when they're added within parenthesized expressions. Since such -expressions are always reformatted to fit minimal space, this whitespace -is lost. - -It will also insert proper spacing before and after function definitions. -It's one line before and after inner functions and two lines before and -after module-level functions. *Black* will put those empty lines also -between the function definition and any standalone comments that -immediately precede the given function. If you want to comment on the -entire function, use a docstring or put a leading comment in the function -body. - -## Editor integration - -* Visual Studio Code: [joslarson.black-vscode](https://marketplace.visualstudio.com/items?itemName=joslarson.black-vscode) - -Any tool that can pipe code through *Black* using its stdio mode (just -[use `-` as the file name](http://www.tldp.org/LDP/abs/html/special-chars.html#DASHREF2)). -The formatted code will be returned on stdout (unless `--check` was -passed). *Black* will still emit messages on stderr but that shouldn't -affect your use case. - -There is currently no integration with any other text editors. Vim and -Atom/Nuclide integration is planned by the author, others will require -external contributions. - -Patches welcome! ✨ 🍰 ✨ diff --git a/docs/testimonials.md b/docs/testimonials.md new file mode 120000 index 0000000..03fc5ae --- /dev/null +++ b/docs/testimonials.md @@ -0,0 +1 @@ +/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/testimonials.md \ No newline at end of file diff --git a/docs/the_black_code_style.md b/docs/the_black_code_style.md new file mode 120000 index 0000000..29b288b --- /dev/null +++ b/docs/the_black_code_style.md @@ -0,0 +1 @@ +/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/the_black_code_style.md \ No newline at end of file diff --git a/docs/usage.md b/docs/usage.md deleted file mode 100644 index 69ce846..0000000 --- a/docs/usage.md +++ /dev/null @@ -1,64 +0,0 @@ -# Installation and Usage - -## Installation - -*Black* can be installed by running `pip install black`. It requires -Python 3.6.0+ to run but you can reformat Python 2 code with it, too. - - -## Usage - -To get started right away with sensible defaults: - -``` -black {source_file_or_directory} -``` - - -### Command line options - -Black doesn't provide many options. You can list them by running -`black --help`: - -```text -Usage: black [OPTIONS] [SRC]... - - The uncompromising code formatter. - -Options: - -l, --line-length INTEGER How many character per line to allow. - [default: 88] - --check Don't write back the files, just return the - status. Return code 0 means nothing would - change. Return code 1 means some files would be - reformatted. Return code 123 means there was an - internal error. - --fast / --safe If --fast given, skip temporary sanity checks. - [default: --safe] - --version Show the version and exit. - --help Show this message and exit. -``` - -`Black` is a well-behaved Unix-style command-line tool: - -* it does nothing if no sources are passed to it; -* it will read from standard input and write to standard output if `-` - is used as the filename; -* it only outputs messages to users on standard error; -* exits with code 0 unless an internal error occured (or `--check` was - used). - - -## NOTE: This tool is alpha quality at the moment - -*Black* can already successfully format itself and the standard library. -It also sports a decent test suite. However, it is still very new. -Things will probably be wonky for a while. This is made explicit by the -"Alpha" trove classifier, as well as by the "a" in the version number. -What this means for you is that **until the formatter becomes stable, -you should expect some formatting to change in the future**. - -Also, as a temporary safety measure, *Black* will check that the -reformatted code still produces a valid AST that is equivalent to the -original. This slows it down. If you're feeling confident, use -``--fast``. -- 2.39.5 From fc869039ebcc0c0ff922ea9b2713480c119e5341 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Mon, 26 Mar 2018 18:41:25 -0700 Subject: [PATCH 03/16] Don't crash and burn on empty lines with trailing whitespace Fixes #80 --- README.md | 6 ++++++ blib2to3/pgen2/tokenize.py | 24 ++++++++++++------------ tests/function.py | 1 + tests/test_black.py | 2 ++ 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 1cd1ef0..1e28d52 100644 --- a/README.md +++ b/README.md @@ -333,6 +333,12 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md). ## Change Log +### 18.3a5 (unreleased) + +* fixed 18.3a4 regression: don't crash and burn on empty lines with + trailing whitespace (#80) + + ### 18.3a4 * `# fmt: off` and `# fmt: on` are implemented (#5) diff --git a/blib2to3/pgen2/tokenize.py b/blib2to3/pgen2/tokenize.py index 669c3f1..b6bbf4e 100644 --- a/blib2to3/pgen2/tokenize.py +++ b/blib2to3/pgen2/tokenize.py @@ -430,24 +430,24 @@ def generate_tokens(readline): yield stashed stashed = None + if line[pos] in '\r\n': # skip blank lines + yield (NL, line[pos:], (lnum, pos), (lnum, len(line)), line) + continue + if column > indents[-1]: # count indents indents.append(column) yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) - if line[pos] in '#\r\n': # skip comments or blank lines - if line[pos] == '#': - comment_token = line[pos:].rstrip('\r\n') - nl_pos = pos + len(comment_token) - yield (COMMENT, comment_token, - (lnum, pos), (lnum, pos + len(comment_token)), line) - yield (NL, line[nl_pos:], - (lnum, nl_pos), (lnum, len(line)), line) - else: - yield ((NL, COMMENT)[line[pos] == '#'], line[pos:], - (lnum, pos), (lnum, len(line)), line) + if line[pos] == '#': # skip comments + comment_token = line[pos:].rstrip('\r\n') + nl_pos = pos + len(comment_token) + yield (COMMENT, comment_token, + (lnum, pos), (lnum, pos + len(comment_token)), line) + yield (NL, line[nl_pos:], + (lnum, nl_pos), (lnum, len(line)), line) continue - while column < indents[-1]: # count dedents + while column < indents[-1]: # count dedents if column not in indents: raise IndentationError( "unindent does not match any outer indentation level", diff --git a/tests/function.py b/tests/function.py index 08ab2b8..387e441 100644 --- a/tests/function.py +++ b/tests/function.py @@ -34,6 +34,7 @@ def spaces(a=1, b=(), c=[], d={}, e=True, f=-1, g=1 if False else 2, h="", i=r'' def spaces_types(a: int = 1, b: tuple = (), c: list = [], d: dict = {}, e: bool = True, f: int = -1, g: int = 1 if False else 2, h: str = "", i: str = r''): ... def spaces2(result= _core.Value(None)): ... + # EMPTY LINE WITH WHITESPACE (this comment will be removed) def example(session): result = session.query(models.Customer.id).filter( models.Customer.account_id == account_id, diff --git a/tests/test_black.py b/tests/test_black.py index 62b4b1a..759bda5 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -17,6 +17,7 @@ ff = partial(black.format_file_in_place, line_length=ll, fast=True) fs = partial(black.format_str, line_length=ll) THIS_FILE = Path(__file__) THIS_DIR = THIS_FILE.parent +EMPTY_LINE = '# EMPTY LINE WITH WHITESPACE' + ' (this comment will be removed)' def dump_to_stderr(*output: str) -> str: @@ -33,6 +34,7 @@ def read_data(name: str) -> Tuple[str, str]: lines = test.readlines() result = _input for line in lines: + line = line.replace(EMPTY_LINE, '') if line.rstrip() == '# output': result = _output continue -- 2.39.5 From 1f445a01c8c073058ccd6ca6ceba8f527e6cbf15 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Mon, 26 Mar 2018 21:29:49 -0700 Subject: [PATCH 04/16] =?utf8?q?It's=20obviously=20not=20just=20me,=20yo.?= =?utf8?q?=20Thanks=20y'all=20=F0=9F=96=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 1e28d52..69df9bc 100644 --- a/README.md +++ b/README.md @@ -435,3 +435,14 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md). ## Authors Glued together by [Łukasz Langa](mailto:lukasz@langa.pl). + +Maintained with [Carol Willing](mailto:carolcode@willingconsulting.com) +and [Carl Meyer](mailto:carl@oddbird.net). + +Multiple contributions by: +* [Artem Malyshev](mailto:proofit404@gmail.com) +* [Daniel M. Capella](mailto:polycitizen@gmail.com) +* [Eli Treuherz](mailto:eli.treuherz@cgi.com) +* Hugo van Kemenade +* [Mika⠙](mailto:mail@autophagy.io) +* [Osaetin Daniel](mailto:osaetindaniel@gmail.com) -- 2.39.5 From e5f8251704c22b143b79474905c6c4b7e10ddb47 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Mon, 26 Mar 2018 22:54:08 -0700 Subject: [PATCH 05/16] Allow up to two empty lines on module level and single empty lines otherwise Fixes #74 --- README.md | 11 +++++++---- black.py | 4 ++-- tests/empty_lines.py | 4 ++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 69df9bc..182e264 100644 --- a/README.md +++ b/README.md @@ -218,10 +218,10 @@ always emit an extra empty line after ``return``, ``raise``, ``break``, ``continue``, and ``yield``. This is to make changes in control flow more prominent to readers of your code. -*Black* will allow single empty lines left by the original editors, -except when they're added within parenthesized expressions. Since such -expressions are always reformatted to fit minimal space, this whitespace -is lost. +*Black* will allow single empty lines inside functions, and single and +double empty lines on module level left by the original editors, except +when they're within parenthesized expressions. Since such expressions +are always reformatted to fit minimal space, this whitespace is lost. It will also insert proper spacing before and after function definitions. It's one line before and after inner functions and two lines before and @@ -338,6 +338,9 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md). * fixed 18.3a4 regression: don't crash and burn on empty lines with trailing whitespace (#80) +* only allow up to two empty lines on module level and only single empty + lines within functions (#74) + ### 18.3a4 diff --git a/black.py b/black.py index 9ea4694..da7af03 100644 --- a/black.py +++ b/black.py @@ -754,13 +754,13 @@ class EmptyLineTracker: def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: max_allowed = 1 - if current_line.is_comment and current_line.depth == 0: + if current_line.depth == 0: max_allowed = 2 if current_line.leaves: # Consume the first leaf's extra newlines. first_leaf = current_line.leaves[0] before = first_leaf.prefix.count('\n') - before = min(before, max(before, max_allowed)) + before = min(before, max_allowed) first_leaf.prefix = '' else: before = 0 diff --git a/tests/empty_lines.py b/tests/empty_lines.py index 8f00ddc..ec04337 100644 --- a/tests/empty_lines.py +++ b/tests/empty_lines.py @@ -12,15 +12,19 @@ def f(): if t == token.COMMENT: # another trailing comment return DOUBLESPACE + assert p is not None, f"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}" + prev = leaf.prev_sibling if not prev: prevp = preceding_leaf(p) if not prevp or prevp.type in OPENING_BRACKETS: + return NO + if prevp.type == token.EQUAL: if prevp.parent and prevp.parent.type in { syms.typedargslist, -- 2.39.5 From 10bb45c35e8e08215ad9a060aca33be91a98b864 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Tue, 27 Mar 2018 02:31:51 -0700 Subject: [PATCH 06/16] First stab at the Vim plugin! --- README.md | 62 ++++++++++++++++++++++--- vim/plugin/black.vim | 108 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 6 deletions(-) create mode 100644 vim/plugin/black.vim diff --git a/README.md b/README.md index 182e264..ac4caa1 100644 --- a/README.md +++ b/README.md @@ -263,8 +263,61 @@ keep it. ## Editor integration -* Visual Studio Code: [joslarson.black-vscode](https://marketplace.visualstudio.com/items?itemName=joslarson.black-vscode) -* Emacs: [proofit404/blacken](https://github.com/proofit404/blacken) +### Emacs + +Use [proofit404/blacken](https://github.com/proofit404/blacken). + + +### Vim + +Commands and shortcuts: + +* `,=` or `:Black` to format the entire file (ranges not supported); +* `:BlackUpgrade` to upgrade *Black* inside the virtualenv; +* `:BlackVersion` to get the current version of *Black* inside the + virtualenv. + +Configuration: +* `g:black_fast` (defaults to `0`) +* `g:black_linelength` (defaults to `88`) +* `g:black_virtualenv` (defaults to `~/.vim/black`) + +To install, copy the plugin from [vim/plugin/black.vim](https://github.com/ambv/black/tree/master/vim/plugin/black.vim). +Let me know if this requires any changes to work with Vim 8's builtin +`packadd`, or Pathogen, or Vundle, and so on. + +This plugin **requires Vim 7.0+ built with Python 3.6+ support**. It +needs Python 3.6 to be able to run *Black* inside the Vim process which +is much faster than calling an external command. + +On first run, the plugin creates its own virtualenv using the right +Python version and automatically installs *Black*. You can upgrade it later +by calling `:BlackUpgrade` and restarting Vim. + +If you need to do anything special to make your virtualenv work and +install *Black* (for example you want to run a version from master), just +create a virtualenv manually and point `g:black_virtualenv` to it. +The plugin will use it. + +**How to get Vim with Python 3.6?** +On Ubuntu 17.10 Vim comes with Python 3.6 by default. +On macOS with HomeBrew run: `brew install vim --with-python3`. +When building Vim from source, use: +`./configure --enable-python3interp=yes`. There's many guides online how +to do this. + + +### Visual Studio Code + +Use [joslarson.black-vscode](https://marketplace.visualstudio.com/items?itemName=joslarson.black-vscode). + + +### Other editors + +Atom/Nuclide integration is planned by the author, others will +require external contributions. + +Patches welcome! ✨ 🍰 ✨ Any tool that can pipe code through *Black* using its stdio mode (just [use `-` as the file name](http://www.tldp.org/LDP/abs/html/special-chars.html#DASHREF2)). @@ -272,10 +325,7 @@ The formatted code will be returned on stdout (unless `--check` was passed). *Black* will still emit messages on stderr but that shouldn't affect your use case. -Vim and Atom/Nuclide integration is planned by the author, others will -require external contributions. - -Patches welcome! ✨ 🍰 ✨ +This can be used for example with PyCharm's [File Watchers](https://www.jetbrains.com/help/pycharm/file-watchers.html). ## Testimonials diff --git a/vim/plugin/black.vim b/vim/plugin/black.vim new file mode 100644 index 0000000..d0afdef --- /dev/null +++ b/vim/plugin/black.vim @@ -0,0 +1,108 @@ +" black.vim +" Author: Łukasz Langa +" Created: Mon Mar 26 23:27:53 2018 -0700 +" Requires: Vim Ver7.0+ +" Version: 1.0 +" +" Documentation: +" This plugin formats Python files. +" +" History: +" 1.0: +" - initial version + +if v:version < 700 || !has('python3') + echo "This script requires vim7.0+ with Python 3.6 support." + finish +endif + +if exists("g:load_black") + finish +endif + +let g:load_black = "py1.0" +if !exists("g:black_virtualenv") + let g:black_virtualenv = "~/.vim/black" +endif +if !exists("g:black_fast") + let g:black_fast = 0 +endif +if !exists("g:black_linelength") + let g:black_linelength = 88 +endif + +python3 << endpython3 +import sys +import vim + +def _initialize_black_env(upgrade=False): + pyver = sys.version_info[:2] + if pyver < (3, 6): + print("Sorry, Black requires Python 3.6+ to run.") + return False + + from pathlib import Path + import subprocess + import venv + virtualenv_path = Path(vim.eval("g:black_virtualenv")).expanduser() + virtualenv_site_packages = str( + virtualenv_path / 'lib' / f'python{pyver[0]}.{pyver[1]}' / 'site-packages' + ) + first_install = False + if not virtualenv_path.is_dir(): + print('Please wait, one time setup for Black.') + _executable = sys.executable + try: + sys.executable = str(Path(sys.exec_prefix) / 'bin' / 'python3') + print(f'Creating a virtualenv in {virtualenv_path}...') + print('(this path can be customized in .vimrc by setting g:black_virtualenv)') + venv.create(virtualenv_path, with_pip=True) + finally: + sys.executable = _executable + first_install = True + if first_install: + print('Installing Black with pip...') + if upgrade: + print('Upgrading Black with pip...') + if first_install or upgrade: + subprocess.run([str(virtualenv_path / 'bin' / 'pip'), 'install', '-U', 'black']) + print('DONE! You are all set, thanks for waiting ✨ 🍰 ✨') + if first_install: + print('Pro-tip: to upgrade Black in the future, use the :BlackUpgrade command and restart Vim.\n') + if sys.path[0] != virtualenv_site_packages: + sys.path.insert(0, virtualenv_site_packages) + return True + +if _initialize_black_env(): + import black + import time + +def Black(): + start = time.time() + fast = bool(int(vim.eval("g:black_fast"))) + line_length = int(vim.eval("g:black_linelength")) + buffer_str = '\n'.join(vim.current.buffer) + '\n' + try: + new_buffer_str = black.format_file_contents(buffer_str, line_length=line_length, fast=fast) + except black.NothingChanged: + print(f'Already well formatted, good job. (took {time.time() - start:.4f}s)') + except Exception as exc: + print(exc) + else: + vim.current.buffer[:] = new_buffer_str.split('\n')[:-1] + print(f'Reformatted in {time.time() - start:.4f}s.') + +def BlackUpgrade(): + _initialize_black_env(upgrade=True) + +def BlackVersion(): + print(f'Black, version {black.__version__} on Python {sys.version}.') + +endpython3 + +command! Black :py3 Black() +command! BlackUpgrade :py3 BlackUpgrade() +command! BlackVersion :py3 BlackVersion() + +nmap ,= :Black +vmap ,= :Black -- 2.39.5 From a9f50cd0b58259a11a1c851bde7b4f11321e5b3b Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Wed, 28 Mar 2018 10:12:27 -0700 Subject: [PATCH 07/16] document classes, functions, exceptions (#82) --- docs/conf.py | 1 + docs/index.rst | 2 +- docs/reference/reference_classes.rst | 44 +++++++++++++++ docs/reference/reference_exceptions.rst | 12 ++++ docs/reference/reference_functions.rst | 74 +++++++++++++++++++++++++ docs/reference/reference_summary.rst | 11 ++++ 6 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 docs/reference/reference_classes.rst create mode 100644 docs/reference/reference_exceptions.rst create mode 100644 docs/reference/reference_functions.rst create mode 100644 docs/reference/reference_summary.rst diff --git a/docs/conf.py b/docs/conf.py index 6edddba..908a1a7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -112,6 +112,7 @@ generate_sections_from_readme() extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', + 'sphinx.ext.napoleon', ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/index.rst b/docs/index.rst index 9422402..a293019 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -51,9 +51,9 @@ Contents editor_integration contributing change_log + reference/reference_summary authors - Indices and tables ================== diff --git a/docs/reference/reference_classes.rst b/docs/reference/reference_classes.rst new file mode 100644 index 0000000..ee490d3 --- /dev/null +++ b/docs/reference/reference_classes.rst @@ -0,0 +1,44 @@ +*Black* classes +=============== + +*Contents are subject to change.* + +.. currentmodule:: black + +:class:`EmptyLineTracker` +------------------------- + +.. autoclass:: black.EmptyLineTracker + :members: + +:class:`FormatOn` +----------------- + +.. autoclass:: black.FormatOn + :members: + +:class:`FormatOff` +------------------ + +.. autoclass:: black.FormatOff + :members: + +:class:`LineGenerator` +---------------------- + +.. autoclass:: black.LineGenerator + :members: + +:class:`Report` +--------------- + +.. autoclass:: black.Report + :members: + +:class:`Visitor` +---------------------- + +.. autoclass:: black.Visitor + :members: + + diff --git a/docs/reference/reference_exceptions.rst b/docs/reference/reference_exceptions.rst new file mode 100644 index 0000000..bd65412 --- /dev/null +++ b/docs/reference/reference_exceptions.rst @@ -0,0 +1,12 @@ +*Black* exceptions +================== + +*Contents are subject to change.* + +.. currentmodule:: black + +.. autoexception:: black.CannotSplit + +.. autoexception:: black.FormatError + +.. autoexception:: black.NothingChanged diff --git a/docs/reference/reference_functions.rst b/docs/reference/reference_functions.rst new file mode 100644 index 0000000..f9cd505 --- /dev/null +++ b/docs/reference/reference_functions.rst @@ -0,0 +1,74 @@ +*Black* functions +================= + +*Contents are subject to change.* + +.. currentmodule:: black + +Assertions and checks +--------------------- + +.. autofunction:: black.assert_equivalent + +.. autofunction:: black.assert_stable + +.. autofunction:: black.is_delimiter + +.. autofunction:: black.is_import + +.. autofunction:: black.is_python36 + +Formatting +---------- + +.. autofunction:: black.format_file_contents + +.. autofunction:: black.format_file_in_place + +.. autofunction:: black.format_stdin_to_stdout + +.. autofunction:: black.format_str + +.. autofunction:: black.schedule_formatting + +File operations +--------------- + +.. autofunction:: black.dump_to_file + +.. autofunction:: black.gen_python_files_in_dir + +Parsing +------- + +.. autofunction:: black.lib2to3_parse + +.. autofunction:: black.lib2to3_unparse + +Split functions +--------------- + +.. autofunction:: black.delimiter_split + +.. autofunction:: black.left_hand_split + +.. autofunction:: black.right_hand_split + +.. autofunction:: black.split_line + +.. autofunction:: black.split_succeeded_or_raise + +Utilities +--------- + +.. autofunction:: black.diff + +.. autofunction:: black.generate_comments + +.. autofunction:: black.make_comment + +.. autofunction:: black.normalize_prefix + +.. autofunction:: black.preceding_leaf + +.. autofunction:: black.whitespace \ No newline at end of file diff --git a/docs/reference/reference_summary.rst b/docs/reference/reference_summary.rst new file mode 100644 index 0000000..780a4b4 --- /dev/null +++ b/docs/reference/reference_summary.rst @@ -0,0 +1,11 @@ +Developer reference +=================== + +*Contents are subject to change.* + +.. toctree:: + :maxdepth: 2 + + reference_classes + reference_functions + reference_exceptions -- 2.39.5 From d01460d9393d611c0673723320f7a1e50c424e21 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Wed, 28 Mar 2018 17:43:18 -0700 Subject: [PATCH 08/16] Auto-generated documentation-related fixes --- black.py | 161 ++++++++++++++++++++---- docs/authors.md | 2 +- docs/change_log.md | 2 +- docs/conf.py | 1 + docs/contributing_to_black.md | 2 +- docs/editor_integration.md | 2 +- docs/installation_and_usage.md | 2 +- docs/license.md | 2 +- docs/reference/reference_classes.rst | 30 +++-- docs/reference/reference_exceptions.rst | 4 + docs/reference/reference_functions.rst | 4 +- docs/show_your_style.md | 2 +- docs/testimonials.md | 2 +- docs/the_black_code_style.md | 2 +- 14 files changed, 172 insertions(+), 46 deletions(-) diff --git a/black.py b/black.py index da7af03..a61a40f 100644 --- a/black.py +++ b/black.py @@ -49,7 +49,7 @@ class CannotSplit(Exception): class FormatError(Exception): - """Base fmt: on/off error. + """Base exception for `# fmt: on` and `# fmt: off` handling. It holds the number of bytes of the prefix consumed before the format control comment appeared. @@ -309,9 +309,18 @@ T = TypeVar('T') class Visitor(Generic[T]): - """Basic lib2to3 visitor that yields things on visiting.""" + """Basic lib2to3 visitor that yields things of type `T` on `visit()`.""" def visit(self, node: LN) -> Iterator[T]: + """Main method to start the visit process. Yields objects of type `T`. + + It tries to find a `visit_*()` method for the given `node.type`, like + `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects. + If no dedicated `visit_*()` method is found, chooses `visit_default()` + instead. + + Then yields objects of type `T` from the selected visitor. + """ if node.type < 256: name = token.tok_name[node.type] else: @@ -319,6 +328,7 @@ class Visitor(Generic[T]): yield from getattr(self, f'visit_{name}', self.visit_default)(node) def visit_default(self, node: LN) -> Iterator[T]: + """Default `visit_*()` implementation. Recurses to children of `node`.""" if isinstance(node, Node): for child in node.children: yield from self.visit(child) @@ -406,12 +416,32 @@ MATH_PRIORITY = 1 @dataclass class BracketTracker: + """Keeps track of brackets on a line.""" + + #: Current bracket depth. depth: int = 0 + #: All currently unclosed brackets. bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict) + #: All current delimiters with their assigned priority. delimiters: Dict[LeafID, Priority] = Factory(dict) + #: Last processed leaf, if any. previous: Optional[Leaf] = None def mark(self, leaf: Leaf) -> None: + """Marks `leaf` with bracket-related metadata. Keeps track of delimiters. + + All leaves receive an int `bracket_depth` field that stores how deep + within brackets a given leaf is. 0 means there are no enclosing brackets + that started on this line. + + If a leaf is itself a closing bracket, it receives an `opening_bracket` + field that it forms a pair with. This is a one-directional link to + avoid reference cycles. + + If a leaf is a delimiter (a token on which Black can split the line if + needed) and it's on depth 0, its `id()` is stored in the tracker's + `delimiters` field. + """ if leaf.type == token.COMMENT: return @@ -456,7 +486,7 @@ class BracketTracker: """Returns True if there is an yet unmatched open bracket on the line.""" return bool(self.bracket_match) - def max_priority(self, exclude: Iterable[LeafID] = ()) -> int: + def max_delimiter_priority(self, exclude: Iterable[LeafID] = ()) -> int: """Returns the highest priority of a delimiter found on the line. Values are consistent with what `is_delimiter()` returns. @@ -466,8 +496,13 @@ class BracketTracker: @dataclass class Line: + """Holds leaves and comments. Can be printed with `str(line)`.""" + + #: indentation level depth: int = 0 + #: list of leaves leaves: List[Leaf] = Factory(list) + #: inline comments that belong on this line comments: Dict[LeafID, Leaf] = Factory(dict) bracket_tracker: BracketTracker = Factory(BracketTracker) inside_brackets: bool = False @@ -475,6 +510,15 @@ class Line: _for_loop_variable: bool = False def append(self, leaf: Leaf, preformatted: bool = False) -> None: + """Add a new `leaf` to the end of the line. + + Unless `preformatted` is True, the `leaf` will receive a new consistent + whitespace prefix and metadata applied by :class:`BracketTracker`. + Trailing commas are maybe removed, unpacked for loop variables are + demoted from being delimiters. + + Inline comments are put aside. + """ has_value = leaf.value.strip() if not has_value: return @@ -496,18 +540,22 @@ class Line: @property def is_comment(self) -> bool: + """Is this line a standalone comment?""" return bool(self) and self.leaves[0].type == STANDALONE_COMMENT @property def is_decorator(self) -> bool: + """Is this line a decorator?""" return bool(self) and self.leaves[0].type == token.AT @property def is_import(self) -> bool: + """Is this an import line?""" return bool(self) and is_import(self.leaves[0]) @property def is_class(self) -> bool: + """Is this a class definition?""" return ( bool(self) and self.leaves[0].type == token.NAME @@ -516,7 +564,7 @@ class Line: @property def is_def(self) -> bool: - """Also returns True for async defs.""" + """Is this a function definition? (Also returns True for async defs.)""" try: first_leaf = self.leaves[0] except IndexError: @@ -538,6 +586,10 @@ class Line: @property def is_flow_control(self) -> bool: + """Is this a flow control statement? + + Those are `return`, `raise`, `break`, and `continue`. + """ return ( bool(self) and self.leaves[0].type == token.NAME @@ -546,6 +598,7 @@ class Line: @property def is_yield(self) -> bool: + """Is this a yield statement?""" return ( bool(self) and self.leaves[0].type == token.NAME @@ -553,6 +606,7 @@ class Line: ) def maybe_remove_trailing_comma(self, closing: Leaf) -> bool: + """Remove trailing comma if there is one and it's safe.""" if not ( self.leaves and self.leaves[-1].type == token.COMMA @@ -615,7 +669,7 @@ class Line: return False def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool: - # See `maybe_increment_for_loop_variable` above for explanation. + """See `maybe_increment_for_loop_variable` above for explanation.""" if self._for_loop_variable and leaf.type == token.NAME and leaf.value == 'in': self.bracket_tracker.depth -= 1 self._for_loop_variable = False @@ -643,6 +697,7 @@ class Line: return self.append_comment(comment) def append_comment(self, comment: Leaf) -> bool: + """Add an inline comment to the line.""" if comment.type != token.COMMENT: return False @@ -661,6 +716,7 @@ class Line: return True def last_non_delimiter(self) -> Leaf: + """Returns the last non-delimiter on the line. Raises LookupError otherwise.""" for i in range(len(self.leaves)): last = self.leaves[-i - 1] if not is_delimiter(last): @@ -669,6 +725,7 @@ class Line: raise LookupError("No non-delimiters found") def __str__(self) -> str: + """Render the line.""" if not self: return '\n' @@ -683,12 +740,21 @@ class Line: return res + '\n' def __bool__(self) -> bool: + """Returns True if the line has leaves or comments.""" return bool(self.leaves or self.comments) class UnformattedLines(Line): + """Just like :class:`Line` but stores lines which aren't reformatted.""" - def append(self, leaf: Leaf, preformatted: bool = False) -> None: + def append(self, leaf: Leaf, preformatted: bool = True) -> None: + """Just add a new `leaf` to the end of the lines. + + The `preformatted` argument is ignored. + + Keeps track of indentation `depth`, which is useful when the user + says `# fmt: on`. Otherwise, doesn't do anything with the `leaf`. + """ try: list(generate_comments(leaf)) except FormatOn as f_on: @@ -702,18 +768,26 @@ class UnformattedLines(Line): self.depth -= 1 def append_comment(self, comment: Leaf) -> bool: + """Not implemented in this class.""" raise NotImplementedError("Unformatted lines don't store comments separately.") def maybe_remove_trailing_comma(self, closing: Leaf) -> bool: + """Does nothing and returns False.""" return False def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool: + """Does nothing and returns False.""" return False def maybe_adapt_standalone_comment(self, comment: Leaf) -> bool: + """Does nothing and returns False.""" return False def __str__(self) -> str: + """Renders unformatted lines from leaves which were added with `append()`. + + `depth` is not used for indentation in this case. + """ if not self: return '\n' @@ -835,7 +909,7 @@ class LineGenerator(Visitor[Line]): yield complete_line def visit(self, node: LN) -> Iterator[Line]: - """High-level entry point to the visitor.""" + """Main method to start the visit process. Yields :class:`Line` objects.""" if isinstance(self.current_line, UnformattedLines): # File contained `# fmt: off` yield from self.visit_unformatted(node) @@ -844,6 +918,7 @@ class LineGenerator(Visitor[Line]): yield from super().visit(node) def visit_default(self, node: LN) -> Iterator[Line]: + """Default `visit_*()` implementation. Recurses to children of `node`.""" if isinstance(node, Leaf): any_open_brackets = self.current_line.bracket_tracker.any_open_brackets() try: @@ -881,18 +956,24 @@ class LineGenerator(Visitor[Line]): yield from super().visit_default(node) def visit_INDENT(self, node: Node) -> Iterator[Line]: + """Increases indentation level, maybe yields a line.""" + # In blib2to3 INDENT never holds comments. yield from self.line(+1) yield from self.visit_default(node) def visit_DEDENT(self, node: Node) -> Iterator[Line]: + """Decreases indentation level, maybe yields a line.""" # DEDENT has no value. Additionally, in blib2to3 it never holds comments. yield from self.line(-1) def visit_stmt(self, node: Node, keywords: Set[str]) -> Iterator[Line]: - """Visit a statement. + """Visits a statement. - The relevant Python language keywords for this statement are NAME leaves - within it. + This implementation is shared for `if`, `while`, `for`, `try`, `except`, + `def`, `with`, and `class`. + + The relevant Python language `keywords` for a given statement will be NAME + leaves within it. This methods puts those on a separate line. """ for child in node.children: if child.type == token.NAME and child.value in keywords: # type: ignore @@ -901,7 +982,7 @@ class LineGenerator(Visitor[Line]): yield from self.visit(child) def visit_simple_stmt(self, node: Node) -> Iterator[Line]: - """A statement without nested statements.""" + """Visits a statement without nested statements.""" is_suite_like = node.parent and node.parent.type in STATEMENT if is_suite_like: yield from self.line(+1) @@ -913,6 +994,7 @@ class LineGenerator(Visitor[Line]): yield from self.visit_default(node) def visit_async_stmt(self, node: Node) -> Iterator[Line]: + """Visits `async def`, `async for`, `async with`.""" yield from self.line() children = iter(node.children) @@ -927,18 +1009,28 @@ class LineGenerator(Visitor[Line]): yield from self.visit(child) def visit_decorators(self, node: Node) -> Iterator[Line]: + """Visits decorators.""" for child in node.children: yield from self.line() yield from self.visit(child) def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]: + """Semicolons are always removed. + + Statements between them are put on separate lines. + """ yield from self.line() def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]: + """End of file. + + Process outstanding comments and end with a newline. + """ yield from self.visit_default(leaf) yield from self.line() def visit_unformatted(self, node: LN) -> Iterator[Line]: + """Used when file contained a `# fmt: off`.""" if isinstance(node, Node): for child in node.children: yield from self.visit(child) @@ -1302,6 +1394,13 @@ def generate_comments(leaf: Leaf) -> Iterator[Leaf]: def make_comment(content: str) -> str: + """Returns a consistently formatted comment from the given `content` string. + + All comments (except for "##", "#!", "#:") should have a single space between + the hash sign and the content. + + If `content` didn't start with a hash sign, one is provided. + """ content = content.rstrip() if not content: return '#' @@ -1370,7 +1469,7 @@ def split_line( def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: - """Split line into many lines, starting with the first matching bracket pair. + """Splits line into many lines, starting with the first matching bracket pair. Note: this usually looks weird, only use this for function definitions. Prefer RHS otherwise. @@ -1407,14 +1506,14 @@ def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: comment_after = line.comments.get(id(leaf)) if comment_after: result.append(comment_after, preformatted=True) - split_succeeded_or_raise(head, body, tail) + bracket_split_succeeded_or_raise(head, body, tail) for result in (head, body, tail): if result: yield result def right_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: - """Split line into many lines, starting with the last matching bracket pair.""" + """Splits line into many lines, starting with the last matching bracket pair.""" head = Line(depth=line.depth) body = Line(depth=line.depth + 1, inside_brackets=True) tail = Line(depth=line.depth) @@ -1447,13 +1546,26 @@ def right_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: comment_after = line.comments.get(id(leaf)) if comment_after: result.append(comment_after, preformatted=True) - split_succeeded_or_raise(head, body, tail) + bracket_split_succeeded_or_raise(head, body, tail) for result in (head, body, tail): if result: yield result -def split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None: +def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None: + """Raise :exc:`CannotSplit` if the last left- or right-hand split failed. + + Do nothing otherwise. + + A left- or right-hand split is based on a pair of brackets. Content before + (and including) the opening bracket is left on one line, content inside the + brackets is put on a separate line, and finally content starting with and + following the closing bracket is put on a separate line. + + Those are called `head`, `body`, and `tail`, respectively. If the split + produced the same line (all content in `head`) or ended up with an empty `body` + and the `tail` is just the closing bracket, then it's considered failed. + """ tail_len = len(str(tail).strip()) if not body: if tail_len == 0: @@ -1467,11 +1579,11 @@ def split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None: def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]: - """Split according to delimiters of the highest priority. + """Splits according to delimiters of the highest priority. This kind of split doesn't increase indentation. If `py36` is True, the split will add trailing commas also in function - signatures that contain * and **. + signatures that contain `*` and `**`. """ try: last_leaf = line.leaves[-1] @@ -1480,7 +1592,9 @@ def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]: delimiters = line.bracket_tracker.delimiters try: - delimiter_priority = line.bracket_tracker.max_priority(exclude={id(last_leaf)}) + delimiter_priority = line.bracket_tracker.max_delimiter_priority( + exclude={id(last_leaf)} + ) except ValueError: raise CannotSplit("No delimiters found") @@ -1531,9 +1645,9 @@ def is_import(leaf: Leaf) -> bool: def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: - """Leave existing extra newlines if not `inside_brackets`. + """Leaves existing extra newlines if not `inside_brackets`. - Remove everything else. Note: don't use backslashes for formatting or + Removes everything else. Note: don't use backslashes for formatting or you'll lose your voting rights. """ if not inside_brackets: @@ -1580,6 +1694,9 @@ BLACKLISTED_DIRECTORIES = { def gen_python_files_in_dir(path: Path) -> Iterator[Path]: + """Generates all files under `path` which aren't under BLACKLISTED_DIRECTORIES + and have one of the PYTHON_EXTENSIONS. + """ for child in path.iterdir(): if child.is_dir(): if child.name in BLACKLISTED_DIRECTORIES: @@ -1593,7 +1710,7 @@ def gen_python_files_in_dir(path: Path) -> Iterator[Path]: @dataclass class Report: - """Provides a reformatting counter.""" + """Provides a reformatting counter. Can be rendered with `str(report)`.""" check: bool = False change_count: int = 0 same_count: int = 0 diff --git a/docs/authors.md b/docs/authors.md index 704a421..e61774c 120000 --- a/docs/authors.md +++ b/docs/authors.md @@ -1 +1 @@ -/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/authors.md \ No newline at end of file +_build/generated/authors.md \ No newline at end of file diff --git a/docs/change_log.md b/docs/change_log.md index 36bf3dc..c4b5e46 120000 --- a/docs/change_log.md +++ b/docs/change_log.md @@ -1 +1 @@ -/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/change_log.md \ No newline at end of file +_build/generated/change_log.md \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 908a1a7..36b9a98 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,6 +59,7 @@ def generate_sections_from_readme(): target_dir.mkdir(parents=True) output = None + target_dir = target_dir.relative_to(CURRENT_DIR) with open(str(readme), 'r', encoding='utf8') as f: for line in f: if line.startswith('## '): diff --git a/docs/contributing_to_black.md b/docs/contributing_to_black.md index 079bd4a..7e940c5 120000 --- a/docs/contributing_to_black.md +++ b/docs/contributing_to_black.md @@ -1 +1 @@ -/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/contributing_to_black.md \ No newline at end of file +_build/generated/contributing_to_black.md \ No newline at end of file diff --git a/docs/editor_integration.md b/docs/editor_integration.md index e234140..2310b98 120000 --- a/docs/editor_integration.md +++ b/docs/editor_integration.md @@ -1 +1 @@ -/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/editor_integration.md \ No newline at end of file +_build/generated/editor_integration.md \ No newline at end of file diff --git a/docs/installation_and_usage.md b/docs/installation_and_usage.md index 64caa30..657c53a 120000 --- a/docs/installation_and_usage.md +++ b/docs/installation_and_usage.md @@ -1 +1 @@ -/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/installation_and_usage.md \ No newline at end of file +_build/generated/installation_and_usage.md \ No newline at end of file diff --git a/docs/license.md b/docs/license.md index cf360a1..3981c33 120000 --- a/docs/license.md +++ b/docs/license.md @@ -1 +1 @@ -/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/license.md \ No newline at end of file +_build/generated/license.md \ No newline at end of file diff --git a/docs/reference/reference_classes.rst b/docs/reference/reference_classes.rst index ee490d3..380d391 100644 --- a/docs/reference/reference_classes.rst +++ b/docs/reference/reference_classes.rst @@ -5,26 +5,26 @@ .. currentmodule:: black -:class:`EmptyLineTracker` +:class:`BracketTracker` ------------------------- -.. autoclass:: black.EmptyLineTracker +.. autoclass:: black.BracketTracker :members: -:class:`FormatOn` ------------------ +:class:`EmptyLineTracker` +------------------------- -.. autoclass:: black.FormatOn +.. autoclass:: black.EmptyLineTracker :members: -:class:`FormatOff` ------------------- +:class:`Line` +------------- -.. autoclass:: black.FormatOff +.. autoclass:: black.Line :members: -:class:`LineGenerator` ----------------------- +:class:`LineGenerator` (:class:`Visitor` [:class:`Line`]) +------------------------------------------------------- .. autoclass:: black.LineGenerator :members: @@ -35,10 +35,14 @@ .. autoclass:: black.Report :members: -:class:`Visitor` ----------------------- +:class:`UnformattedLines` (:class:`Line`) +---------------------------------------- -.. autoclass:: black.Visitor +.. autoclass:: black.UnformattedLines :members: +:class:`Visitor` (Generic[T]) +---------------------------- +.. autoclass:: black.Visitor + :members: diff --git a/docs/reference/reference_exceptions.rst b/docs/reference/reference_exceptions.rst index bd65412..46b042e 100644 --- a/docs/reference/reference_exceptions.rst +++ b/docs/reference/reference_exceptions.rst @@ -9,4 +9,8 @@ .. autoexception:: black.FormatError +.. autoexception:: black.FormatOn + +.. autoexception:: black.FormatOff + .. autoexception:: black.NothingChanged diff --git a/docs/reference/reference_functions.rst b/docs/reference/reference_functions.rst index f9cd505..098f155 100644 --- a/docs/reference/reference_functions.rst +++ b/docs/reference/reference_functions.rst @@ -56,7 +56,7 @@ Split functions .. autofunction:: black.split_line -.. autofunction:: black.split_succeeded_or_raise +.. autofunction:: black.bracket_split_succeeded_or_raise Utilities --------- @@ -71,4 +71,4 @@ Utilities .. autofunction:: black.preceding_leaf -.. autofunction:: black.whitespace \ No newline at end of file +.. autofunction:: black.whitespace diff --git a/docs/show_your_style.md b/docs/show_your_style.md index 15ad1c1..6b8bfcc 120000 --- a/docs/show_your_style.md +++ b/docs/show_your_style.md @@ -1 +1 @@ -/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/show_your_style.md \ No newline at end of file +_build/generated/show_your_style.md \ No newline at end of file diff --git a/docs/testimonials.md b/docs/testimonials.md index 03fc5ae..f564808 120000 --- a/docs/testimonials.md +++ b/docs/testimonials.md @@ -1 +1 @@ -/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/testimonials.md \ No newline at end of file +_build/generated/testimonials.md \ No newline at end of file diff --git a/docs/the_black_code_style.md b/docs/the_black_code_style.md index 29b288b..734a71a 120000 --- a/docs/the_black_code_style.md +++ b/docs/the_black_code_style.md @@ -1 +1 @@ -/Users/ambv/Dropbox (Personal)/Python/black/docs/_build/generated/the_black_code_style.md \ No newline at end of file +_build/generated/the_black_code_style.md \ No newline at end of file -- 2.39.5 From 7f7b31058af65b245bfc1c35fd37f2ff6e78e43d Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Wed, 28 Mar 2018 19:03:16 -0700 Subject: [PATCH 09/16] More minor documentation-related changes --- black.py | 42 +++++++++++++++++++--------- docs/conf.py | 2 ++ docs/reference/reference_classes.rst | 21 ++++++++------ 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/black.py b/black.py index a61a40f..de86156 100644 --- a/black.py +++ b/black.py @@ -38,7 +38,7 @@ err = partial(click.secho, fg='red', err=True) class NothingChanged(UserWarning): - """Raised by `format_file` when the reformatted code is the same as source.""" + """Raised by `format_file()` when the reformatted code is the same as source.""" class CannotSplit(Exception): @@ -165,6 +165,13 @@ async def schedule_formatting( loop: BaseEventLoop, executor: Executor, ) -> int: + """Run formatting of `sources` in parallel using the provided `executor`. + + (Use ProcessPoolExecutors for actual parallelism.) + + `line_length`, `write_back`, and `fast` options are passed to + :func:`format_file_in_place`. + """ tasks = { src: loop.run_in_executor( executor, format_file_in_place, src, line_length, fast, write_back @@ -193,7 +200,11 @@ async def schedule_formatting( def format_file_in_place( src: Path, line_length: int, fast: bool, write_back: bool = False ) -> bool: - """Format the file and rewrite if changed. Return True if changed.""" + """Format file under `src` path. Return True if changed. + + If `write_back` is True, write reformatted code back to stdout. + `line_length` and `fast` options are passed to :func:`format_file_contents`. + """ with tokenize.open(src) as src_buffer: src_contents = src_buffer.read() try: @@ -212,7 +223,11 @@ def format_file_in_place( def format_stdin_to_stdout( line_length: int, fast: bool, write_back: bool = False ) -> bool: - """Format file on stdin and pipe output to stdout. Return True if changed.""" + """Format file on stdin. Return True if changed. + + If `write_back` is True, write reformatted code back to stdout. + `line_length` and `fast` arguments are passed to :func:`format_file_contents`. + """ contents = sys.stdin.read() try: contents = format_file_contents(contents, line_length=line_length, fast=fast) @@ -229,7 +244,12 @@ def format_stdin_to_stdout( def format_file_contents( src_contents: str, line_length: int, fast: bool ) -> FileContent: - """Reformats a file and returns its contents and encoding.""" + """Reformats a file and returns its contents and encoding. + + If `fast` is False, additionally confirm that the reformatted code is + valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it. + `line_length` is passed to :func:`format_str`. + """ if src_contents.strip() == '': raise NothingChanged @@ -244,7 +264,10 @@ def format_file_contents( def format_str(src_contents: str, line_length: int) -> FileContent: - """Reformats a string and returns new contents.""" + """Reformats a string and returns new contents. + + `line_length` determines how many characters per line are allowed. + """ src_node = lib2to3_parse(src_contents) dst_contents = "" lines = LineGenerator() @@ -312,7 +335,7 @@ class Visitor(Generic[T]): """Basic lib2to3 visitor that yields things of type `T` on `visit()`.""" def visit(self, node: LN) -> Iterator[T]: - """Main method to start the visit process. Yields objects of type `T`. + """Main method to visit `node` and its children. It tries to find a `visit_*()` method for the given `node.type`, like `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects. @@ -418,13 +441,9 @@ MATH_PRIORITY = 1 class BracketTracker: """Keeps track of brackets on a line.""" - #: Current bracket depth. depth: int = 0 - #: All currently unclosed brackets. bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict) - #: All current delimiters with their assigned priority. delimiters: Dict[LeafID, Priority] = Factory(dict) - #: Last processed leaf, if any. previous: Optional[Leaf] = None def mark(self, leaf: Leaf) -> None: @@ -498,11 +517,8 @@ class BracketTracker: class Line: """Holds leaves and comments. Can be printed with `str(line)`.""" - #: indentation level depth: int = 0 - #: list of leaves leaves: List[Leaf] = Factory(list) - #: inline comments that belong on this line comments: Dict[LeafID, Leaf] = Factory(dict) bracket_tracker: BracketTracker = Factory(BracketTracker) inside_brackets: bool = False diff --git a/docs/conf.py b/docs/conf.py index 36b9a98..9599afd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -272,6 +272,8 @@ epub_exclude_files = ['search.html'] # -- Extension configuration ------------------------------------------------- +autodoc_member_order = 'bysource' + # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. diff --git a/docs/reference/reference_classes.rst b/docs/reference/reference_classes.rst index 380d391..dc0805d 100644 --- a/docs/reference/reference_classes.rst +++ b/docs/reference/reference_classes.rst @@ -9,24 +9,26 @@ ------------------------- .. autoclass:: black.BracketTracker - :members: + :members: :class:`EmptyLineTracker` ------------------------- .. autoclass:: black.EmptyLineTracker - :members: + :members: :class:`Line` ------------- .. autoclass:: black.Line :members: + :special-members: __str__, __bool__ -:class:`LineGenerator` (:class:`Visitor` [:class:`Line`]) -------------------------------------------------------- +:class:`LineGenerator` +---------------------- .. autoclass:: black.LineGenerator + :show-inheritance: :members: :class:`Report` @@ -34,15 +36,18 @@ .. autoclass:: black.Report :members: + :special-members: __str__ -:class:`UnformattedLines` (:class:`Line`) ----------------------------------------- +:class:`UnformattedLines` +------------------------- .. autoclass:: black.UnformattedLines + :show-inheritance: :members: -:class:`Visitor` (Generic[T]) ----------------------------- +:class:`Visitor` +---------------- .. autoclass:: black.Visitor + :show-inheritance: :members: -- 2.39.5 From 653aa40579a8741d1405679460b36751ff50b531 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Wed, 28 Mar 2018 19:28:31 -0700 Subject: [PATCH 10/16] Add DebugVisitor.show() to documentation under utility functions --- docs/reference/reference_functions.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/reference/reference_functions.rst b/docs/reference/reference_functions.rst index 098f155..c167c4b 100644 --- a/docs/reference/reference_functions.rst +++ b/docs/reference/reference_functions.rst @@ -61,6 +61,10 @@ Split functions Utilities --------- +.. py:function:: black.DebugVisitor.show(code: str) -> None + + Pretty-print the lib2to3 AST of a given string of `code`. + .. autofunction:: black.diff .. autofunction:: black.generate_comments -- 2.39.5 From b901d75deb497aa56a1d7651a3386af8a495afc9 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Wed, 28 Mar 2018 19:31:40 -0700 Subject: [PATCH 11/16] Show __str__ in UnformattedLines --- docs/reference/reference_classes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/reference_classes.rst b/docs/reference/reference_classes.rst index dc0805d..d99057e 100644 --- a/docs/reference/reference_classes.rst +++ b/docs/reference/reference_classes.rst @@ -44,6 +44,7 @@ .. autoclass:: black.UnformattedLines :show-inheritance: :members: + :special-members: __str__ :class:`Visitor` ---------------- -- 2.39.5 From 44a235173119eb63b27a648b931940f13c04f424 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Wed, 28 Mar 2018 19:31:53 -0700 Subject: [PATCH 12/16] Use imperative language in all docstrings --- black.py | 146 +++++++++++++++++++++++++++---------------------------- 1 file changed, 72 insertions(+), 74 deletions(-) diff --git a/black.py b/black.py index de86156..434bc33 100644 --- a/black.py +++ b/black.py @@ -38,13 +38,14 @@ err = partial(click.secho, fg='red', err=True) class NothingChanged(UserWarning): - """Raised by `format_file()` when the reformatted code is the same as source.""" + """Raised by :func:`format_file` when reformatted code is the same as source.""" class CannotSplit(Exception): """A readable split that fits the allotted line length is impossible. - Raised by `left_hand_split()`, `right_hand_split()`, and `delimiter_split()`. + Raised by :func:`left_hand_split`, :func:`right_hand_split`, and + :func:`delimiter_split`. """ @@ -244,7 +245,7 @@ def format_stdin_to_stdout( def format_file_contents( src_contents: str, line_length: int, fast: bool ) -> FileContent: - """Reformats a file and returns its contents and encoding. + """Reformat contents a file and return new contents. If `fast` is False, additionally confirm that the reformatted code is valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it. @@ -264,7 +265,7 @@ def format_file_contents( def format_str(src_contents: str, line_length: int) -> FileContent: - """Reformats a string and returns new contents. + """Reformat a string and return new contents. `line_length` determines how many characters per line are allowed. """ @@ -383,7 +384,7 @@ class DebugVisitor(Visitor[T]): @classmethod def show(cls, code: str) -> None: - """Pretty-prints a given string of `code`. + """Pretty-print the lib2to3 AST of a given string of `code`. Convenience method for debugging. """ @@ -447,7 +448,7 @@ class BracketTracker: previous: Optional[Leaf] = None def mark(self, leaf: Leaf) -> None: - """Marks `leaf` with bracket-related metadata. Keeps track of delimiters. + """Mark `leaf` with bracket-related metadata. Keep track of delimiters. All leaves receive an int `bracket_depth` field that stores how deep within brackets a given leaf is. 0 means there are no enclosing brackets @@ -502,11 +503,11 @@ class BracketTracker: self.previous = leaf def any_open_brackets(self) -> bool: - """Returns True if there is an yet unmatched open bracket on the line.""" + """Return True if there is an yet unmatched open bracket on the line.""" return bool(self.bracket_match) def max_delimiter_priority(self, exclude: Iterable[LeafID] = ()) -> int: - """Returns the highest priority of a delimiter found on the line. + """Return the highest priority of a delimiter found on the line. Values are consistent with what `is_delimiter()` returns. """ @@ -571,7 +572,7 @@ class Line: @property def is_class(self) -> bool: - """Is this a class definition?""" + """Is this line a class definition?""" return ( bool(self) and self.leaves[0].type == token.NAME @@ -602,7 +603,7 @@ class Line: @property def is_flow_control(self) -> bool: - """Is this a flow control statement? + """Is this line a flow control statement? Those are `return`, `raise`, `break`, and `continue`. """ @@ -614,7 +615,7 @@ class Line: @property def is_yield(self) -> bool: - """Is this a yield statement?""" + """Is this line a yield statement?""" return ( bool(self) and self.leaves[0].type == token.NAME @@ -673,8 +674,8 @@ class Line: def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool: """In a for loop, or comprehension, the variables are often unpacks. - To avoid splitting on the comma in this situation, we will increase - the depth of tokens between `for` and `in`. + To avoid splitting on the comma in this situation, increase the depth of + tokens between `for` and `in`. """ if leaf.type == token.NAME and leaf.value == 'for': self.has_for = True @@ -732,7 +733,7 @@ class Line: return True def last_non_delimiter(self) -> Leaf: - """Returns the last non-delimiter on the line. Raises LookupError otherwise.""" + """Return the last non-delimiter on the line. Raise LookupError otherwise.""" for i in range(len(self.leaves)): last = self.leaves[-i - 1] if not is_delimiter(last): @@ -756,7 +757,7 @@ class Line: return res + '\n' def __bool__(self) -> bool: - """Returns True if the line has leaves or comments.""" + """Return True if the line has leaves or comments.""" return bool(self.leaves or self.comments) @@ -783,8 +784,21 @@ class UnformattedLines(Line): elif leaf.type == token.DEDENT: self.depth -= 1 + def __str__(self) -> str: + """Render unformatted lines from leaves which were added with `append()`. + + `depth` is not used for indentation in this case. + """ + if not self: + return '\n' + + res = '' + for leaf in self.leaves: + res += str(leaf) + return res + def append_comment(self, comment: Leaf) -> bool: - """Not implemented in this class.""" + """Not implemented in this class. Raises `NotImplementedError`.""" raise NotImplementedError("Unformatted lines don't store comments separately.") def maybe_remove_trailing_comma(self, closing: Leaf) -> bool: @@ -799,19 +813,6 @@ class UnformattedLines(Line): """Does nothing and returns False.""" return False - def __str__(self) -> str: - """Renders unformatted lines from leaves which were added with `append()`. - - `depth` is not used for indentation in this case. - """ - if not self: - return '\n' - - res = '' - for leaf in self.leaves: - res += str(leaf) - return res - @dataclass class EmptyLineTracker: @@ -827,11 +828,11 @@ class EmptyLineTracker: previous_defs: List[int] = Factory(list) def maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: - """Returns the number of extra empty lines before and after the `current_line`. + """Return the number of extra empty lines before and after the `current_line`. - This is for separating `def`, `async def` and `class` with extra empty lines - (two on module-level), as well as providing an extra empty line after flow - control keywords to make them more prominent. + This is for separating `def`, `async def` and `class` with extra empty + lines (two on module-level), as well as providing an extra empty line + after flow control keywords to make them more prominent. """ if isinstance(current_line, UnformattedLines): return 0, 0 @@ -925,7 +926,10 @@ class LineGenerator(Visitor[Line]): yield complete_line def visit(self, node: LN) -> Iterator[Line]: - """Main method to start the visit process. Yields :class:`Line` objects.""" + """Main method to visit `node` and its children. + + Yields :class:`Line` objects. + """ if isinstance(self.current_line, UnformattedLines): # File contained `# fmt: off` yield from self.visit_unformatted(node) @@ -972,18 +976,18 @@ class LineGenerator(Visitor[Line]): yield from super().visit_default(node) def visit_INDENT(self, node: Node) -> Iterator[Line]: - """Increases indentation level, maybe yields a line.""" + """Increase indentation level, maybe yield a line.""" # In blib2to3 INDENT never holds comments. yield from self.line(+1) yield from self.visit_default(node) def visit_DEDENT(self, node: Node) -> Iterator[Line]: - """Decreases indentation level, maybe yields a line.""" + """Decrease indentation level, maybe yield a line.""" # DEDENT has no value. Additionally, in blib2to3 it never holds comments. yield from self.line(-1) def visit_stmt(self, node: Node, keywords: Set[str]) -> Iterator[Line]: - """Visits a statement. + """Visit a statement. This implementation is shared for `if`, `while`, `for`, `try`, `except`, `def`, `with`, and `class`. @@ -998,7 +1002,7 @@ class LineGenerator(Visitor[Line]): yield from self.visit(child) def visit_simple_stmt(self, node: Node) -> Iterator[Line]: - """Visits a statement without nested statements.""" + """Visit a statement without nested statements.""" is_suite_like = node.parent and node.parent.type in STATEMENT if is_suite_like: yield from self.line(+1) @@ -1010,7 +1014,7 @@ class LineGenerator(Visitor[Line]): yield from self.visit_default(node) def visit_async_stmt(self, node: Node) -> Iterator[Line]: - """Visits `async def`, `async for`, `async with`.""" + """Visit `async def`, `async for`, `async with`.""" yield from self.line() children = iter(node.children) @@ -1025,23 +1029,17 @@ class LineGenerator(Visitor[Line]): yield from self.visit(child) def visit_decorators(self, node: Node) -> Iterator[Line]: - """Visits decorators.""" + """Visit decorators.""" for child in node.children: yield from self.line() yield from self.visit(child) def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]: - """Semicolons are always removed. - - Statements between them are put on separate lines. - """ + """Remove a semicolon and put the other statement on a separate line.""" yield from self.line() def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]: - """End of file. - - Process outstanding comments and end with a newline. - """ + """End of file. Process outstanding comments and end with a newline.""" yield from self.visit_default(leaf) yield from self.line() @@ -1319,7 +1317,7 @@ def whitespace(leaf: Leaf) -> str: # noqa C901 def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]: - """Returns the first leaf that precedes `node`, if any.""" + """Return the first leaf that precedes `node`, if any.""" while node: res = node.prev_sibling if res: @@ -1337,7 +1335,7 @@ def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]: def is_delimiter(leaf: Leaf) -> int: - """Returns the priority of the `leaf` delimiter. Returns 0 if not delimiter. + """Return the priority of the `leaf` delimiter. Return 0 if not delimiter. Higher numbers are higher priority. """ @@ -1358,7 +1356,7 @@ def is_delimiter(leaf: Leaf) -> int: def generate_comments(leaf: Leaf) -> Iterator[Leaf]: - """Cleans the prefix of the `leaf` and generates comments from it, if any. + """Clean the prefix of the `leaf` and generate comments from it, if any. Comments in lib2to3 are shoved into the whitespace prefix. This happens in `pgen2/driver.py:Driver.parse_tokens()`. This was a brilliant implementation @@ -1410,7 +1408,7 @@ def generate_comments(leaf: Leaf) -> Iterator[Leaf]: def make_comment(content: str) -> str: - """Returns a consistently formatted comment from the given `content` string. + """Return a consistently formatted comment from the given `content` string. All comments (except for "##", "#!", "#:") should have a single space between the hash sign and the content. @@ -1431,7 +1429,7 @@ def make_comment(content: str) -> str: def split_line( line: Line, line_length: int, inner: bool = False, py36: bool = False ) -> Iterator[Line]: - """Splits a `line` into potentially many lines. + """Split a `line` into potentially many lines. They should fit in the allotted `line_length` but might not be able to. `inner` signifies that there were a pair of brackets somewhere around the @@ -1485,7 +1483,7 @@ def split_line( def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: - """Splits line into many lines, starting with the first matching bracket pair. + """Split line into many lines, starting with the first matching bracket pair. Note: this usually looks weird, only use this for function definitions. Prefer RHS otherwise. @@ -1529,7 +1527,7 @@ def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: def right_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: - """Splits line into many lines, starting with the last matching bracket pair.""" + """Split line into many lines, starting with the last matching bracket pair.""" head = Line(depth=line.depth) body = Line(depth=line.depth + 1, inside_brackets=True) tail = Line(depth=line.depth) @@ -1595,7 +1593,7 @@ def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]: - """Splits according to delimiters of the highest priority. + """Split according to delimiters of the highest priority. This kind of split doesn't increase indentation. If `py36` is True, the split will add trailing commas also in function @@ -1647,7 +1645,7 @@ def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]: def is_import(leaf: Leaf) -> bool: - """Returns True if the given leaf starts an import statement.""" + """Return True if the given leaf starts an import statement.""" p = leaf.parent t = leaf.type v = leaf.value @@ -1661,10 +1659,10 @@ def is_import(leaf: Leaf) -> bool: def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: - """Leaves existing extra newlines if not `inside_brackets`. + """Leave existing extra newlines if not `inside_brackets`. Remove everything + else. - Removes everything else. Note: don't use backslashes for formatting or - you'll lose your voting rights. + Note: don't use backslashes for formatting or you'll lose your voting rights. """ if not inside_brackets: spl = leaf.prefix.split('#') @@ -1679,7 +1677,7 @@ def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: def is_python36(node: Node) -> bool: - """Returns True if the current file is using Python 3.6+ features. + """Return True if the current file is using Python 3.6+ features. Currently looking for: - f-strings; and @@ -1710,7 +1708,7 @@ BLACKLISTED_DIRECTORIES = { def gen_python_files_in_dir(path: Path) -> Iterator[Path]: - """Generates all files under `path` which aren't under BLACKLISTED_DIRECTORIES + """Generate all files under `path` which aren't under BLACKLISTED_DIRECTORIES and have one of the PYTHON_EXTENSIONS. """ for child in path.iterdir(): @@ -1749,7 +1747,13 @@ class Report: @property def return_code(self) -> int: - """Which return code should the app use considering the current state.""" + """Return the exit code that the app should use. + + This considers the current state of changed files and failures: + - if there were any failures, return 123; + - if any files were changed and --check is being used, return 1; + - otherwise return 0. + """ # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with # 126 we have special returncodes reserved by the shell. if self.failure_count: @@ -1761,7 +1765,7 @@ class Report: return 0 def __str__(self) -> str: - """A color report of the current state. + """Render a color report of the current state. Use `click.unstyle` to remove colors. """ @@ -1791,10 +1795,7 @@ class Report: def assert_equivalent(src: str, dst: str) -> None: - """Raises AssertionError if `src` and `dst` aren't equivalent. - - This is a temporary sanity check until Black becomes stable. - """ + """Raise AssertionError if `src` and `dst` aren't equivalent.""" import ast import traceback @@ -1857,10 +1858,7 @@ def assert_equivalent(src: str, dst: str) -> None: def assert_stable(src: str, dst: str, line_length: int) -> None: - """Raises AssertionError if `dst` reformats differently the second time. - - This is a temporary sanity check until Black becomes stable. - """ + """Raise AssertionError if `dst` reformats differently the second time.""" newdst = format_str(dst, line_length=line_length) if dst != newdst: log = dump_to_file( @@ -1876,7 +1874,7 @@ def assert_stable(src: str, dst: str, line_length: int) -> None: def dump_to_file(*output: str) -> str: - """Dumps `output` to a temporary file. Returns path to the file.""" + """Dump `output` to a temporary file. Return path to the file.""" import tempfile with tempfile.NamedTemporaryFile( @@ -1889,7 +1887,7 @@ def dump_to_file(*output: str) -> str: def diff(a: str, b: str, a_name: str, b_name: str) -> str: - """Returns a udiff string between strings `a` and `b`.""" + """Return a unified diff string between strings `a` and `b`.""" import difflib a_lines = [line + '\n' for line in a.split('\n')] -- 2.39.5 From 3ee9ebb0916d76be904c948d62e9b55f569b6f98 Mon Sep 17 00:00:00 2001 From: Josh Holland Date: Thu, 29 Mar 2018 23:21:18 +0100 Subject: [PATCH 13/16] Fix --check with multiple files (#88) Passing multiple files to --check would previously result in the report being printed as if the files had been written to. --- black.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/black.py b/black.py index 434bc33..dab3f00 100644 --- a/black.py +++ b/black.py @@ -181,7 +181,7 @@ async def schedule_formatting( } await asyncio.wait(tasks.values()) cancelled = [] - report = Report() + report = Report(check=not write_back) for src, task in tasks.items(): if not task.done(): report.failed(src, 'timed out, cancelling') -- 2.39.5 From 728c56c986bc5aea4d9897d3fce3159f89991b8e Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Wed, 28 Mar 2018 19:45:48 -0700 Subject: [PATCH 14/16] Remove the test-specific .flake8 file --- tests/.flake8 | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 tests/.flake8 diff --git a/tests/.flake8 b/tests/.flake8 deleted file mode 100644 index 3528ac4..0000000 --- a/tests/.flake8 +++ /dev/null @@ -1,8 +0,0 @@ -# Like the base Black .flake8 but also ignores F811 which is used deliberately -# in test files. - -[flake8] -ignore = E266, E501, F811 -max-line-length = 80 -max-complexity = 12 -select = B,C,E,F,W,T4,B9 -- 2.39.5 From c55d08d0b96c8de8bd867ca315e380d9e9d2d7ec Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Thu, 29 Mar 2018 21:06:18 -0700 Subject: [PATCH 15/16] Remove standalone comment hacks Now Black properly splits standalone comments within bracketed expressions. They are treated as another type of split instead of being bolted on with whitespace prefixes. A related fix: now multiple comments might appear after a given leaf. Fixes #22 --- black.py | 215 +++++++++++++++++++++++++++++++------------- tests/comments3.py | 38 ++++++++ tests/test_black.py | 8 ++ 3 files changed, 198 insertions(+), 63 deletions(-) create mode 100644 tests/comments3.py diff --git a/black.py b/black.py index dab3f00..6499b22 100644 --- a/black.py +++ b/black.py @@ -3,14 +3,25 @@ import asyncio from asyncio.base_events import BaseEventLoop from concurrent.futures import Executor, ProcessPoolExecutor -from functools import partial +from functools import partial, wraps import keyword import os from pathlib import Path import tokenize import sys from typing import ( - Dict, Generic, Iterable, Iterator, List, Optional, Set, Tuple, Type, TypeVar, Union + Callable, + Dict, + Generic, + Iterable, + Iterator, + List, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, ) from attr import dataclass, Factory @@ -32,7 +43,9 @@ Depth = int NodeType = int LeafID = int Priority = int +Index = int LN = Union[Leaf, Node] +SplitFunc = Callable[['Line', bool], Iterator['Line']] out = partial(click.secho, bold=True, err=True) err = partial(click.secho, fg='red', err=True) @@ -520,7 +533,7 @@ class Line: depth: int = 0 leaves: List[Leaf] = Factory(list) - comments: Dict[LeafID, Leaf] = Factory(dict) + comments: List[Tuple[Index, Leaf]] = Factory(list) bracket_tracker: BracketTracker = Factory(BracketTracker) inside_brackets: bool = False has_for: bool = False @@ -549,16 +562,31 @@ class Line: self.bracket_tracker.mark(leaf) self.maybe_remove_trailing_comma(leaf) self.maybe_increment_for_loop_variable(leaf) - if self.maybe_adapt_standalone_comment(leaf): - return if not self.append_comment(leaf): self.leaves.append(leaf) + def append_safe(self, leaf: Leaf, preformatted: bool = False) -> None: + """Like :func:`append()` but disallow invalid standalone comment structure. + + Raises ValueError when any `leaf` is appended after a standalone comment + or when a standalone comment is not the first leaf on the line. + """ + if self.bracket_tracker.depth == 0: + if self.is_comment: + raise ValueError("cannot append to standalone comments") + + if self.leaves and leaf.type == STANDALONE_COMMENT: + raise ValueError( + "cannot append standalone comments to a populated line" + ) + + self.append(leaf, preformatted=preformatted) + @property def is_comment(self) -> bool: """Is this line a standalone comment?""" - return bool(self) and self.leaves[0].type == STANDALONE_COMMENT + return len(self.leaves) == 1 and self.leaves[0].type == STANDALONE_COMMENT @property def is_decorator(self) -> bool: @@ -622,6 +650,15 @@ class Line: and self.leaves[0].value == 'yield' ) + @property + def contains_standalone_comments(self) -> bool: + """If so, needs to be split before emitting.""" + for leaf in self.leaves: + if leaf.type == STANDALONE_COMMENT: + return True + + return False + def maybe_remove_trailing_comma(self, closing: Leaf) -> bool: """Remove trailing comma if there is one and it's safe.""" if not ( @@ -632,13 +669,13 @@ class Line: return False if closing.type == token.RBRACE: - self.leaves.pop() + self.remove_trailing_comma() return True if closing.type == token.RSQB: comma = self.leaves[-1] if comma.parent and comma.parent.type == syms.listmaker: - self.leaves.pop() + self.remove_trailing_comma() return True # For parens let's check if it's safe to remove the comma. If the @@ -666,7 +703,7 @@ class Line: break if commas > 1: - self.leaves.pop() + self.remove_trailing_comma() return True return False @@ -694,52 +731,49 @@ class Line: return False - def maybe_adapt_standalone_comment(self, comment: Leaf) -> bool: - """Hack a standalone comment to act as a trailing comment for line splitting. - - If this line has brackets and a standalone `comment`, we need to adapt - it to be able to still reformat the line. - - This is not perfect, the line to which the standalone comment gets - appended will appear "too long" when splitting. - """ - if not ( + def append_comment(self, comment: Leaf) -> bool: + """Add an inline or standalone comment to the line.""" + if ( comment.type == STANDALONE_COMMENT and self.bracket_tracker.any_open_brackets() ): + comment.prefix = '' return False - comment.type = token.COMMENT - comment.prefix = '\n' + ' ' * (self.depth + 1) - return self.append_comment(comment) - - def append_comment(self, comment: Leaf) -> bool: - """Add an inline comment to the line.""" if comment.type != token.COMMENT: return False - try: - after = id(self.last_non_delimiter()) - except LookupError: + after = len(self.leaves) - 1 + if after == -1: comment.type = STANDALONE_COMMENT comment.prefix = '' return False else: - if after in self.comments: - self.comments[after].value += str(comment) - else: - self.comments[after] = comment + self.comments.append((after, comment)) return True - def last_non_delimiter(self) -> Leaf: - """Return the last non-delimiter on the line. Raise LookupError otherwise.""" - for i in range(len(self.leaves)): - last = self.leaves[-i - 1] - if not is_delimiter(last): - return last + def comments_after(self, leaf: Leaf) -> Iterator[Leaf]: + """Generate comments that should appear directly after `leaf`.""" + for _leaf_index, _leaf in enumerate(self.leaves): + if leaf is _leaf: + break + + else: + return - raise LookupError("No non-delimiters found") + for index, comment_after in self.comments: + if _leaf_index == index: + yield comment_after + + def remove_trailing_comma(self) -> None: + """Remove the trailing comma and moves the comments attached to it.""" + comma_index = len(self.leaves) - 1 + for i in range(len(self.comments)): + comment_index, comment = self.comments[i] + if comment_index == comma_index: + self.comments[i] = (comma_index - 1, comment) + self.leaves.pop() def __str__(self) -> str: """Render the line.""" @@ -752,7 +786,7 @@ class Line: res = f'{first.prefix}{indent}{first.value}' for leaf in leaves: res += str(leaf) - for comment in self.comments.values(): + for _, comment in self.comments: res += str(comment) return res + '\n' @@ -809,10 +843,6 @@ class UnformattedLines(Line): """Does nothing and returns False.""" return False - def maybe_adapt_standalone_comment(self, comment: Leaf) -> bool: - """Does nothing and returns False.""" - return False - @dataclass class EmptyLineTracker: @@ -1439,23 +1469,24 @@ def split_line( If `py36` is True, splitting may generate syntax that is only compatible with Python 3.6 and later. """ - if isinstance(line, UnformattedLines): + if isinstance(line, UnformattedLines) or line.is_comment: yield line return line_str = str(line).strip('\n') - if len(line_str) <= line_length and '\n' not in line_str: + if ( + len(line_str) <= line_length + and '\n' not in line_str # multiline strings + and not line.contains_standalone_comments + ): yield line return + split_funcs: List[SplitFunc] if line.is_def: split_funcs = [left_hand_split] elif line.inside_brackets: - split_funcs = [delimiter_split] - if '\n' not in line_str: - # Only attempt RHS if we don't have multiline strings or comments - # on this line. - split_funcs.append(right_hand_split) + split_funcs = [delimiter_split, standalone_comment_split, right_hand_split] else: split_funcs = [right_hand_split] for split_func in split_funcs: @@ -1464,7 +1495,7 @@ def split_line( # split altogether. result: List[Line] = [] try: - for l in split_func(line, py36=py36): + for l in split_func(line, py36): if str(l).strip('\n') == line_str: raise CannotSplit("Split function returned an unchanged result") @@ -1517,8 +1548,7 @@ def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: ): for leaf in leaves: result.append(leaf, preformatted=True) - comment_after = line.comments.get(id(leaf)) - if comment_after: + for comment_after in line.comments_after(leaf): result.append(comment_after, preformatted=True) bracket_split_succeeded_or_raise(head, body, tail) for result in (head, body, tail): @@ -1557,8 +1587,7 @@ def right_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: ): for leaf in leaves: result.append(leaf, preformatted=True) - comment_after = line.comments.get(id(leaf)) - if comment_after: + for comment_after in line.comments_after(leaf): result.append(comment_after, preformatted=True) bracket_split_succeeded_or_raise(head, body, tail) for result in (head, body, tail): @@ -1592,10 +1621,25 @@ def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None ) +def dont_increase_indentation(split_func: SplitFunc) -> SplitFunc: + """Normalize prefix of the first leaf in every line returned by `split_func`. + + This is a decorator over relevant split functions. + """ + + @wraps(split_func) + def split_wrapper(line: Line, py36: bool = False) -> Iterator[Line]: + for l in split_func(line, py36): + normalize_prefix(l.leaves[0], inside_brackets=True) + yield l + + return split_wrapper + + +@dont_increase_indentation def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]: """Split according to delimiters of the highest priority. - This kind of split doesn't increase indentation. If `py36` is True, the split will add trailing commas also in function signatures that contain `*` and `**`. """ @@ -1615,11 +1659,24 @@ def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]: current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets) lowest_depth = sys.maxsize trailing_comma_safe = True + + def append_to_line(leaf: Leaf) -> Iterator[Line]: + """Append `leaf` to current line or to new line if appending impossible.""" + nonlocal current_line + try: + current_line.append_safe(leaf, preformatted=True) + except ValueError as ve: + yield current_line + + current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets) + current_line.append(leaf) + for leaf in line.leaves: - current_line.append(leaf, preformatted=True) - comment_after = line.comments.get(id(leaf)) - if comment_after: - current_line.append(comment_after, preformatted=True) + yield from append_to_line(leaf) + + for comment_after in line.comments_after(leaf): + yield from append_to_line(comment_after) + lowest_depth = min(lowest_depth, leaf.bracket_depth) if ( leaf.bracket_depth == lowest_depth @@ -1629,7 +1686,6 @@ def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]: trailing_comma_safe = trailing_comma_safe and py36 leaf_priority = delimiters.get(id(leaf)) if leaf_priority == delimiter_priority: - normalize_prefix(current_line.leaves[0], inside_brackets=True) yield current_line current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets) @@ -1640,7 +1696,40 @@ def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]: and trailing_comma_safe ): current_line.append(Leaf(token.COMMA, ',')) - normalize_prefix(current_line.leaves[0], inside_brackets=True) + yield current_line + + +@dont_increase_indentation +def standalone_comment_split(line: Line, py36: bool = False) -> Iterator[Line]: + """Split standalone comments from the rest of the line.""" + for leaf in line.leaves: + if leaf.type == STANDALONE_COMMENT: + if leaf.bracket_depth == 0: + break + + else: + raise CannotSplit("Line does not have any standalone comments") + + current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets) + + def append_to_line(leaf: Leaf) -> Iterator[Line]: + """Append `leaf` to current line or to new line if appending impossible.""" + nonlocal current_line + try: + current_line.append_safe(leaf, preformatted=True) + except ValueError as ve: + yield current_line + + current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets) + current_line.append(leaf) + + for leaf in line.leaves: + yield from append_to_line(leaf) + + for comment_after in line.comments_after(leaf): + yield from append_to_line(comment_after) + + if current_line: yield current_line diff --git a/tests/comments3.py b/tests/comments3.py new file mode 100644 index 0000000..b57f8f3 --- /dev/null +++ b/tests/comments3.py @@ -0,0 +1,38 @@ +def func(): + lcomp3 = [ + # This one is actually too long to fit in a single line. + element.split('\n', 1)[0] + # yup + for element in collection.select_elements() + # right + if element is not None + ] + # Capture each of the exceptions in the MultiError along with each of their causes and contexts + if isinstance(exc_value, MultiError): + embedded = [] + for exc in exc_value.exceptions: + if exc not in _seen: + embedded.append( + # This should be left alone (before) + traceback.TracebackException.from_exception( + exc, + limit=limit, + lookup_lines=lookup_lines, + capture_locals=capture_locals, + # copy the set of _seen exceptions so that duplicates + # shared between sub-exceptions are not omitted + _seen=set(_seen), + ) + # This should be left alone (after) + ) + + # everything is fine if the expression isn't nested + traceback.TracebackException.from_exception( + exc, + limit=limit, + lookup_lines=lookup_lines, + capture_locals=capture_locals, + # copy the set of _seen exceptions so that duplicates + # shared between sub-exceptions are not omitted + _seen=set(_seen), + ) diff --git a/tests/test_black.py b/tests/test_black.py index 759bda5..9d7f579 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -150,6 +150,14 @@ class BlackTestCase(unittest.TestCase): black.assert_equivalent(source, actual) black.assert_stable(source, actual, line_length=ll) + @patch("black.dump_to_file", dump_to_stderr) + def test_comments3(self) -> None: + source, expected = read_data('comments3') + actual = fs(source) + self.assertFormatEqual(expected, actual) + black.assert_equivalent(source, actual) + black.assert_stable(source, actual, line_length=ll) + @patch("black.dump_to_file", dump_to_stderr) def test_cantfit(self) -> None: source, expected = read_data('cantfit') -- 2.39.5 From 17b22642f5f4f3193dfd10fcc86992fb225d6be1 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Thu, 29 Mar 2018 21:19:20 -0700 Subject: [PATCH 16/16] More comments tests --- tests/comments4.py | 76 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_black.py | 8 +++++ 2 files changed, 84 insertions(+) create mode 100644 tests/comments4.py diff --git a/tests/comments4.py b/tests/comments4.py new file mode 100644 index 0000000..e74bf50 --- /dev/null +++ b/tests/comments4.py @@ -0,0 +1,76 @@ +class C: + + @pytest.mark.parametrize( + ("post_data", "message"), + [ + # metadata_version errors. + ( + {}, + "None is an invalid value for Metadata-Version. " + "Error: This field is required. " + "see " + "https://packaging.python.org/specifications/core-metadata", + ), + ( + {"metadata_version": "-1"}, + "'-1' is an invalid value for Metadata-Version. " + "Error: Unknown Metadata Version " + "see " + "https://packaging.python.org/specifications/core-metadata", + ), + # name errors. + ( + {"metadata_version": "1.2"}, + "'' is an invalid value for Name. " + "Error: This field is required. " + "see " + "https://packaging.python.org/specifications/core-metadata", + ), + ( + {"metadata_version": "1.2", "name": "foo-"}, + "'foo-' is an invalid value for Name. " + "Error: Must start and end with a letter or numeral and " + "contain only ascii numeric and '.', '_' and '-'. " + "see " + "https://packaging.python.org/specifications/core-metadata", + ), + # version errors. + ( + {"metadata_version": "1.2", "name": "example"}, + "'' is an invalid value for Version. " + "Error: This field is required. " + "see " + "https://packaging.python.org/specifications/core-metadata", + ), + ( + {"metadata_version": "1.2", "name": "example", "version": "dog"}, + "'dog' is an invalid value for Version. " + "Error: Must start and end with a letter or numeral and " + "contain only ascii numeric and '.', '_' and '-'. " + "see " + "https://packaging.python.org/specifications/core-metadata", + ), + ], + ) + def test_fails_invalid_post_data( + self, pyramid_config, db_request, post_data, message + ): + pyramid_config.testing_securitypolicy(userid=1) + db_request.POST = MultiDict(post_data) + + +def foo(list_a, list_b): + results = ( + User.query.filter(User.foo == 'bar').filter( # Because foo. + db.or_(User.field_a.astext.in_(list_a), User.field_b.astext.in_(list_b)) + ).filter( + User.xyz.is_(None) + ) + # Another comment about the filtering on is_quux goes here. + .filter(db.not_(User.is_pending.astext.cast(db.Boolean).is_(True))).order_by( + User.created_at.desc() + ).with_for_update( + key_share=True + ).all() + ) + return results diff --git a/tests/test_black.py b/tests/test_black.py index 9d7f579..1c22e54 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -158,6 +158,14 @@ class BlackTestCase(unittest.TestCase): black.assert_equivalent(source, actual) black.assert_stable(source, actual, line_length=ll) + @patch("black.dump_to_file", dump_to_stderr) + def test_comments4(self) -> None: + source, expected = read_data('comments4') + actual = fs(source) + self.assertFormatEqual(expected, actual) + black.assert_equivalent(source, actual) + black.assert_stable(source, actual, line_length=ll) + @patch("black.dump_to_file", dump_to_stderr) def test_cantfit(self) -> None: source, expected = read_data('cantfit') -- 2.39.5