Skip to content

Commit 1648291

Browse files
authored
Quick fix for non UTF-8 encodings with new parser (#22066)
Fixes #22055 As discussed in the issue. This is not super-principled, but it works, and also gives nice standard colorized blocker if the fallback doesn't work either.
1 parent baa9db5 commit 1648291

8 files changed

Lines changed: 135 additions & 76 deletions

File tree

‎mypy/build.py‎

Lines changed: 30 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -3191,31 +3191,10 @@ def get_source(self) -> str:
31913191
with self.wrap_context():
31923192
source = self.source
31933193
if self.path and source is None:
3194-
try:
3194+
with self.handle_file_read_errors():
31953195
path = manager.maybe_swap_for_shadow_path(self.path)
31963196
source = decode_python_encoding(manager.fscache.read(path))
31973197
self.source_hash = manager.fscache.hash_digest(path)
3198-
except OSError as ioerr:
3199-
# ioerr.strerror differs for os.stat failures between Windows and
3200-
# other systems, but os.strerror(ioerr.errno) does not, so we use that.
3201-
# (We want the error messages to be platform-independent so that the
3202-
# tests have predictable output.)
3203-
assert ioerr.errno is not None
3204-
raise CompileError(
3205-
[
3206-
"mypy: error: Cannot read file '{}': {}".format(
3207-
self.path.replace(os.getcwd() + os.sep, ""),
3208-
os.strerror(ioerr.errno),
3209-
)
3210-
],
3211-
module_with_blocker=self.id,
3212-
) from ioerr
3213-
except (UnicodeDecodeError, DecodeError) as decodeerr:
3214-
if self.path.endswith(".pyd"):
3215-
err = f"{self.path}: error: Stubgen does not support .pyd files"
3216-
else:
3217-
err = f"{self.path}: error: Cannot decode file: {str(decodeerr)}"
3218-
raise CompileError([err], module_with_blocker=self.id) from decodeerr
32193198
elif self.path and manager.fscache.isdir(self.path):
32203199
source = ""
32213200
self.source_hash = ""
@@ -3229,11 +3208,37 @@ def get_source(self) -> str:
32293208
self.time_spent_us += time_spent_us(t0)
32303209
return source
32313210

3211+
@contextlib.contextmanager
3212+
def handle_file_read_errors(self) -> Iterator[None]:
3213+
try:
3214+
yield
3215+
except OSError as ioerr:
3216+
# ioerr.strerror differs for os.stat failures between Windows and
3217+
# other systems, but os.strerror(ioerr.errno) does not, so we use that.
3218+
# (We want the error messages to be platform-independent so that the
3219+
# tests have predictable output.)
3220+
assert ioerr.errno is not None
3221+
err_path = self.manager.errors.simplify_path(self.xpath)
3222+
raise CompileError(
3223+
[f"{err_path}: error: Cannot read file: {os.strerror(ioerr.errno)}"],
3224+
module_with_blocker=self.id,
3225+
) from ioerr
3226+
except (UnicodeDecodeError, DecodeError) as decodeerr:
3227+
err_path = self.manager.errors.simplify_path(self.xpath)
3228+
if err_path.endswith(".pyd"):
3229+
err = f"{err_path}: error: Stubgen does not support .pyd files"
3230+
else:
3231+
err = f"{err_path}: error: Cannot decode file: {str(decodeerr)}"
3232+
raise CompileError([err], module_with_blocker=self.id) from decodeerr
3233+
32323234
def parse_file_inner(self, source: str | None, raw_data: FileRawData | None = None) -> None:
32333235
t0 = time_ref()
3234-
self.tree = self.manager.parse_file(
3235-
self.id, self.xpath, source, options=self.options, raw_data=raw_data
3236-
)
3236+
# Error handling here matches get_source(), since in case of an error in
3237+
# the new parser we fall back to reading the file manually ourselves.
3238+
with self.handle_file_read_errors():
3239+
self.tree = self.manager.parse_file(
3240+
self.id, self.xpath, source, options=self.options, raw_data=raw_data
3241+
)
32373242
self.time_spent_us += time_spent_us(t0)
32383243

32393244
def parse_file(self, *, temporary: bool = False, raw_data: FileRawData | None = None) -> None:

‎mypy/nativeparse.py‎

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
read_str_opt,
4747
read_tag,
4848
)
49+
from mypy.errors import CompileError
4950
from mypy.nodes import (
5051
ARG_KINDS,
5152
ARG_POS,
@@ -157,7 +158,7 @@
157158
UnionType,
158159
UnpackType,
159160
)
160-
from mypy.util import unnamed_function
161+
from mypy.util import decode_python_encoding, hash_digest, unnamed_function
161162

162163
TypeIgnores = list[tuple[int, list[str]]]
163164

@@ -297,16 +298,40 @@ def parse_to_binary_ast(
297298
source: str | bytes | None = None,
298299
skip_function_bodies: bool = False,
299300
) -> tuple[bytes, list[ParseError], TypeIgnores, bytes, bool, bool, str, list[tuple[int, str]]]:
300-
ast_bytes, errors, ignores, import_bytes, ast_data = ast_serialize.parse(
301-
filename,
302-
source,
303-
skip_function_bodies=skip_function_bodies,
304-
python_version=options.python_version,
305-
platform=options.platform,
306-
always_true=options.always_true,
307-
always_false=options.always_false,
308-
cache_version=5,
309-
)
301+
try:
302+
ast_bytes, errors, ignores, import_bytes, ast_data = ast_serialize.parse(
303+
filename,
304+
source,
305+
skip_function_bodies=skip_function_bodies,
306+
python_version=options.python_version,
307+
platform=options.platform,
308+
always_true=options.always_true,
309+
always_false=options.always_false,
310+
cache_version=5,
311+
)
312+
except ValueError as exc:
313+
if source is None:
314+
# Try reading/encoding manually, since native parser only supports UTF-8.
315+
# If we still cannot decode, decode error will bubble up to caller.
316+
with open(filename, "rb") as f:
317+
source_bytes = f.read()
318+
source = decode_python_encoding(source_bytes)
319+
ast_bytes, errors, ignores, import_bytes, ast_data = ast_serialize.parse(
320+
filename,
321+
source,
322+
skip_function_bodies=skip_function_bodies,
323+
python_version=options.python_version,
324+
platform=options.platform,
325+
always_true=options.always_true,
326+
always_false=options.always_false,
327+
cache_version=5,
328+
)
329+
# Compute hash using original non-UTF-8 bytes, so that
330+
# incremental mode works correctly.
331+
ast_data["source_hash"] = hash_digest(source_bytes)
332+
else:
333+
# Convert everything unexpected to a standard-looking blocker.
334+
raise CompileError([f"{filename}: error: Cannot parse file: {exc}"]) from exc
310335
return (
311336
ast_bytes,
312337
errors,

‎mypy/test/data.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@ def setup(self) -> None:
379379
# Write the first incremental steps
380380
dir = os.path.dirname(path)
381381
os.makedirs(dir, exist_ok=True)
382-
with open(path, "w", encoding="utf8") as f:
382+
with open(path, "w", encoding=choose_file_encoding(path)) as f:
383383
f.write(content)
384384

385385
for num, paths in self.deleted_paths.items():
@@ -831,6 +831,14 @@ def has_stable_flags(testcase: DataDrivenTestCase) -> bool:
831831
return True
832832

833833

834+
def choose_file_encoding(path: str) -> str:
835+
base_name, _ = os.path.splitext(path)
836+
if base_name.endswith("_latin1"):
837+
return "latin-1"
838+
else:
839+
return "utf-8"
840+
841+
834842
class DataSuite:
835843
# option fields - class variables
836844
files: list[str]

‎mypy/test/helpers.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,13 @@
2727
from mypy.main import process_options
2828
from mypy.options import Options
2929
from mypy.test.config import test_data_prefix, test_temp_dir
30-
from mypy.test.data import DataDrivenTestCase, DeleteFile, UpdateFile, fix_cobertura_filename
30+
from mypy.test.data import (
31+
DataDrivenTestCase,
32+
DeleteFile,
33+
UpdateFile,
34+
choose_file_encoding,
35+
fix_cobertura_filename,
36+
)
3137

3238
skip = pytest.mark.skip
3339

@@ -440,7 +446,7 @@ def write_and_fudge_mtime(content: str, target_path: str) -> None:
440446

441447
dir = os.path.dirname(target_path)
442448
os.makedirs(dir, exist_ok=True)
443-
with open(target_path, "w", encoding="utf-8") as target:
449+
with open(target_path, "w", encoding=choose_file_encoding(target_path)) as target:
444450
target.write(content)
445451

446452
if new_time:

‎mypy/test/test_nativeparse.py‎

Lines changed: 28 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,4 @@
1-
"""Tests for the experimental native mypy parser.
2-
3-
To run these, you will need to manually install ast_serialize from
4-
https://github.com/mypyc/ast_serialize first (see the README for the details).
5-
"""
1+
"""Tests for the native mypy parser."""
62

73
from __future__ import annotations
84

@@ -12,6 +8,7 @@
128
import unittest
139
from collections.abc import Iterator
1410

11+
import pytest
1512
from librt.internal import ReadBuffer
1613

1714
from mypy import defaults, nodes
@@ -27,36 +24,24 @@
2724
)
2825
from mypy.config_parser import parse_mypy_comments
2926
from mypy.errors import CompileError
27+
from mypy.nativeparse import (
28+
State,
29+
deserialize_imports,
30+
native_parse,
31+
parse_to_binary_ast,
32+
read_statements,
33+
)
3034
from mypy.nodes import MypyFile, ParseError
3135
from mypy.options import Options
3236
from mypy.test.data import DataDrivenTestCase, DataSuite
3337
from mypy.test.helpers import assert_string_arrays_equal
3438
from mypy.util import get_mypy_comments
3539

36-
# If the experimental ast_serialize module isn't installed, the following import will fail
37-
# and we won't run any native parser tests.
38-
try:
39-
from mypy.nativeparse import (
40-
State,
41-
deserialize_imports,
42-
native_parse,
43-
parse_to_binary_ast,
44-
read_statements,
45-
)
46-
47-
has_nativeparse = True
48-
except ImportError:
49-
has_nativeparse = False
50-
5140

5241
class NativeParserSuite(DataSuite):
5342
required_out_section = True
5443
base_path = "."
55-
files = (
56-
["native-parser.test", "native-parser-python311.test", "native-parser-python312.test"]
57-
if has_nativeparse
58-
else []
59-
)
44+
files = ["native-parser.test", "native-parser-python311.test", "native-parser-python312.test"]
6045

6146
def run_case(self, testcase: DataDrivenTestCase) -> None:
6247
test_parser(testcase)
@@ -65,7 +50,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
6550
class NativeParserImportsSuite(DataSuite):
6651
required_out_section = True
6752
base_path = "."
68-
files = ["native-parser-imports.test"] if has_nativeparse else []
53+
files = ["native-parser-imports.test"]
6954

7055
def run_case(self, testcase: DataDrivenTestCase) -> None:
7156
test_parser_imports(testcase)
@@ -238,7 +223,6 @@ def format_reachable_imports(node: MypyFile) -> list[str]:
238223
return output
239224

240225

241-
@unittest.skipUnless(has_nativeparse, "nativeparse not available")
242226
class TestNativeParserBinaryFormat(unittest.TestCase):
243227
def _assert_trivial_binary_data(self, b: bytes, /) -> None:
244228
# A quick sanity check to ensure the serialized data looks as expected. Only covers
@@ -296,14 +280,27 @@ def test_trivial_binary_data_from_bytes_source(self) -> None:
296280
self._assert_trivial_binary_data(b)
297281

298282
def test_invalid_bytes_raises(self) -> None:
299-
with self.assertRaises(UnicodeDecodeError):
283+
with self.assertRaises(CompileError):
300284
parse_to_binary_ast("", Options(), b"\xff")
301285

302286

287+
class TestNativeParserCustomEncoding(unittest.TestCase):
288+
def test_latin1(self) -> None:
289+
source = "# coding: latin1\nJérôme = False"
290+
with temp_source(source, encoding="latin1") as fnam:
291+
parse_to_binary_ast(fnam, Options())
292+
293+
def test_latin1_broken(self) -> None:
294+
source = "# coding: ascii\nJérôme = False"
295+
with temp_source(source, encoding="latin1") as fnam:
296+
with pytest.raises(UnicodeDecodeError):
297+
parse_to_binary_ast(fnam, Options())
298+
299+
303300
@contextlib.contextmanager
304-
def temp_source(text: str) -> Iterator[str]:
301+
def temp_source(text: str, encoding: str = "utf-8") -> Iterator[str]:
305302
with tempfile.TemporaryDirectory() as temp_dir:
306303
temp_path = os.path.join(temp_dir, "t.py")
307-
with open(temp_path, "w") as f:
308-
f.write(text)
304+
with open(temp_path, "wb") as f:
305+
f.write(text.encode(encoding))
309306
yield temp_path

‎test-data/unit/check-incremental.test‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2334,7 +2334,7 @@ tmp/c.py:1: error: Module "d" has no attribute "x"
23342334
[delete nonexistent.py.2]
23352335
[out]
23362336
[out2]
2337-
mypy: error: Cannot read file 'tmp/nonexistent.py': No such file or directory
2337+
tmp/nonexistent.py: error: Cannot read file: No such file or directory
23382338

23392339
[case testSerializeAbstractPropertyIncremental]
23402340
from abc import abstractmethod
@@ -8275,3 +8275,21 @@ value: str = "x"
82758275
tmp/a.py:9: note: Revealed type is "builtins.list[builtins.int]"
82768276
[out2]
82778277
tmp/a.py:9: note: Revealed type is "builtins.list[builtins.int]"
8278+
8279+
[case testIncrementalWorksCorrectlyWithNonUtf8]
8280+
import a
8281+
[file a.py]
8282+
import b_latin1
8283+
[file a.py.2]
8284+
import b_latin1
8285+
# touch
8286+
[file b_latin1.py]
8287+
# coding: latin-1
8288+
Jérôme = False
8289+
[file b_latin1.py.2]
8290+
# coding: latin-1
8291+
Jérôme = False
8292+
[rechecked a]
8293+
[stale]
8294+
[out]
8295+
[out2]

‎test-data/unit/cmdline.test‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -422,7 +422,7 @@ int_pow.py:11: note: Revealed type is "Any"
422422
[case testMissingFile]
423423
# cmd: mypy nope.py
424424
[out]
425-
mypy: error: Cannot read file 'nope.py': No such file or directory
425+
nope.py: error: Cannot read file: No such file or directory
426426
== Return code: 2
427427

428428
[case testModulesAndPackages]
@@ -673,7 +673,7 @@ c.py:1: error: Name "fail" is not defined
673673
\[mypy]
674674
files = config.py
675675
[out]
676-
mypy: error: Cannot read file 'override.py': No such file or directory
676+
override.py: error: Cannot read file: No such file or directory
677677
== Return code: 2
678678

679679
[case testErrorSummaryOnSuccess]
@@ -730,7 +730,7 @@ Found 2 errors in 2 files (checked 2 source files)
730730
[case testErrorSummaryOnBadUsage]
731731
# cmd: mypy --error-summary missing.py
732732
[out]
733-
mypy: error: Cannot read file 'missing.py': No such file or directory
733+
missing.py: error: Cannot read file: No such file or directory
734734
Found 1 error in 1 file (errors prevented further checking)
735735
== Return code: 2
736736

‎test-data/unit/fine-grained.test‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6145,9 +6145,9 @@ a.py:1: error: "int" not callable
61456145
[file a.py.2]
61466146
1()
61476147
[out]
6148-
mypy: error: Cannot read file 'tmp/nonexistent.py': No such file or directory
6148+
nonexistent.py: error: Cannot read file: No such file or directory
61496149
==
6150-
mypy: error: Cannot read file 'tmp/nonexistent.py': No such file or directory
6150+
nonexistent.py: error: Cannot read file: No such file or directory
61516151

61526152
[case testNonExistentFileOnCommandLine2]
61536153
# cmd: mypy a.py

0 commit comments

Comments
 (0)