Review of (explicit) encoding for files being created in text mode (#1088)

* Fix encoding for files created for building package (issue #1027)

This is minimal set of modifications, which fixed the problem.

There are still more places, where files are open in text mode
without explicit encoding, thus possibly failing on systems not using
UTF-8 console encoding.

* test_api.py use explicit encoding for files.

* get-poetry.py: fix explicit (utf-8) encoding for files open in text mode

* tests: fix explicit (utf-8) encoding for files open in text mode

* poetry init: use explicit utf-8 for created pyproject.toml

* installed: use explicit utf-8 for text files

* layouts: use explicit utf-8 for open text files

* spdx: explicit utf-8 for open text files

* pypi_repository: explicit utf-8 for open text files

* explicit utf-8 for json

* explicit utf-8 for metadata

* explicit utf-8 for requires.txt

* open files with explicit encoding in py27

* fix creation of poetry.bat on Windows

* Make get-poetry.py compatible to py2/3

* Blacked two files to pass linter test

* remove extranous encoding related comments

* rewert broken move of import related to ansible

* removed extraneous comments about opening files in UTF-8

* rewert ansible.release import (keep order of vars unchanged)

* Revert to original content (as it comes from external project)

* all vendored setup.py files in fixtures reverted back to org

Author: vlcinsky

get-poetry.py

@@ -33,7 +33,7 @@
 from contextlib import contextmanager
 from functools import cmp_to_key
 from gzip import GzipFile
-from io import UnsupportedOperation
+from io import UnsupportedOperation, open
 
 try:
     from urllib.error import HTTPError
@@ -58,6 +58,10 @@
 except ImportError:
     winreg = None
 
+try:
+    u = unicode
+except NameError:
+    u = str
 
 WINDOWS = sys.platform.startswith("win") or (sys.platform == "cli" and os.name == "nt")
 
@@ -191,6 +195,7 @@ def expanduser(path):
 
 
 BIN = """#!/usr/bin/env python
+# -*- coding: utf-8 -*-
 import glob
 import sys
 import os
@@ -205,7 +210,7 @@ def expanduser(path):
     main()
 """
 
-BAT = '@echo off\r\npython "{poetry_bin}" %*\r\n'
+BAT = u('@echo off\r\npython "{poetry_bin}" %*\r\n')
 
 
 PRE_MESSAGE = """# Welcome to {poetry}!
@@ -387,7 +392,9 @@ def _compare_versions(x, y):
 
         current_version = None
         if os.path.exists(POETRY_LIB):
-            with open(os.path.join(POETRY_LIB, "poetry", "__version__.py")) as f:
+            with open(
+                os.path.join(POETRY_LIB, "poetry", "__version__.py"), encoding="utf-8"
+            ) as f:
                 version_content = f.read()
 
             current_version_re = re.match(
@@ -563,15 +570,17 @@ def make_bin(self):
         if WINDOWS:
             with open(os.path.join(POETRY_BIN, "poetry.bat"), "w") as f:
                 f.write(
-                    BAT.format(
-                        poetry_bin=os.path.join(POETRY_BIN, "poetry").replace(
-                            os.environ["USERPROFILE"], "%USERPROFILE%"
+                    u(
+                        BAT.format(
+                            poetry_bin=os.path.join(POETRY_BIN, "poetry").replace(
+                                os.environ["USERPROFILE"], "%USERPROFILE%"
+                            )
                         )
                     )
                 )
 
-        with open(os.path.join(POETRY_BIN, "poetry"), "w") as f:
-            f.write(BIN)
+        with open(os.path.join(POETRY_BIN, "poetry"), "w", encoding="utf-8") as f:
+            f.write(u(BIN))
 
         if not WINDOWS:
             # Making the file executable
@@ -583,7 +592,7 @@ def make_env(self):
             return
 
         with open(os.path.join(POETRY_HOME, "env"), "w") as f:
-            f.write(self.get_export_string())
+            f.write(u(self.get_export_string()))
 
     def update_path(self):
         """
@@ -608,7 +617,7 @@ def update_path(self):
 
             if addition not in content:
                 with open(profile, "a") as f:
-                    f.write(addition)
+                    f.write(u(addition))
 
                 updated.append(os.path.relpath(profile, HOME))
 

poetry/console/commands/init.py

@@ -158,7 +158,7 @@ def handle(self):
 
             return 1
 
-        with (Path.cwd() / "pyproject.toml").open("w") as f:
+        with (Path.cwd() / "pyproject.toml").open("w", encoding="utf-8") as f:
             f.write(content)
 
     def _determine_requirements(

poetry/installation/pip_installer.py

@@ -193,7 +193,7 @@ def install_directory(self, package):
             # We also need it for non-PEP-517 packages
             builder = SdistBuilder(Poetry.create(pyproject.parent), NullEnv(), NullIO())
 
-            with open(setup, "w") as f:
+            with open(setup, "w", encoding="utf-8") as f:
                 f.write(decode(builder.build_setup()))
 
         if package.develop:

poetry/json/__init__.py

@@ -3,6 +3,7 @@
 
 import jsonschema
 
+from io import open
 from typing import List
 
 SCHEMA_DIR = os.path.join(os.path.dirname(__file__), "schemas")
@@ -19,7 +20,7 @@ def validate_object(obj, schema_name):  # type: (dict, str) -> List[str]
     if not os.path.exists(schema):
         raise ValueError("Schema {} does not exist.".format(schema_name))
 
-    with open(schema) as f:
+    with open(schema, encoding="utf-8") as f:
         schema = json.loads(f.read())
 
     validator = jsonschema.Draft7Validator(schema)

poetry/layouts/layout.py

@@ -140,7 +140,7 @@ def _create_tests(self, path):
         tests.mkdir()
         tests_init.touch(exist_ok=False)
 
-        with tests_default.open("w") as f:
+        with tests_default.open("w", encoding="utf-8") as f:
             f.write(
                 TESTS_DEFAULT.format(
                     package_name=self._package_name, version=self._version
@@ -152,5 +152,5 @@ def _write_poetry(self, path):
 
         poetry = path / "pyproject.toml"
 
-        with poetry.open("w") as f:
+        with poetry.open("w", encoding="utf-8") as f:
             f.write(content)

poetry/layouts/src.py

@@ -14,5 +14,5 @@ def _create_default(self, path):
 
         package_path.mkdir(parents=True)
 
-        with package_init.open("w") as f:
+        with package_init.open("w", encoding="utf-8") as f:
             f.write(DEFAULT.format(version=self._version))

poetry/layouts/standard.py

@@ -14,5 +14,5 @@ def _create_default(self, path):
 
         package_path.mkdir()
 
-        with package_init.open("w") as f:
+        with package_init.open("w", encoding="utf-8") as f:
             f.write(DEFAULT.format(version=self._version))

poetry/masonry/api.py

@@ -39,13 +39,13 @@ def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None):
     dist_info.mkdir()
 
     if "scripts" in poetry.local_config or "plugins" in poetry.local_config:
-        with (dist_info / "entry_points.txt").open("w") as f:
+        with (dist_info / "entry_points.txt").open("w", encoding="utf-8") as f:
             builder._write_entry_points(f)
 
-    with (dist_info / "WHEEL").open("w") as f:
+    with (dist_info / "WHEEL").open("w", encoding="utf-8") as f:
         builder._write_wheel_file(f)
 
-    with (dist_info / "METADATA").open("w") as f:
+    with (dist_info / "METADATA").open("w", encoding="utf-8") as f:
         builder._write_metadata_file(f)
 
     return dist_info.name

poetry/masonry/metadata.py

@@ -46,7 +46,7 @@ def from_package(cls, package):  # type: (...) -> Metadata
         meta.version = normalize_version(package.version.text)
         meta.summary = package.description
         if package.readme:
-            with package.readme.open() as f:
+            with package.readme.open(encoding="utf-8") as f:
                 meta.description = f.read()
 
         meta.keywords = ",".join(package.keywords)

poetry/puzzle/provider.py

@@ -346,7 +346,7 @@ def search_for_directory(
                     reqs = []
                     requires = egg_info / "requires.txt"
                     if requires.exists():
-                        with requires.open() as f:
+                        with requires.open(encoding="utf-8") as f:
                             reqs = parse_requires(f.read())
             finally:
                 os.chdir(current_dir)

poetry/repositories/pypi_repository.py

@@ -533,7 +533,7 @@ def _get_info_from_sdist(
 
                 requires = egg_info / "requires.txt"
                 if requires.exists():
-                    with requires.open() as f:
+                    with requires.open(encoding="utf-8") as f:
                         info["requires_dist"] = parse_requires(f.read())
 
                         return info
@@ -545,7 +545,7 @@ def _get_info_from_sdist(
 
                 requires = egg_info / "requires.txt"
                 if requires.exists():
-                    with requires.open() as f:
+                    with requires.open(encoding="utf-8") as f:
                         info["requires_dist"] = parse_requires(f.read())
 
                         return info

poetry/spdx/__init__.py

@@ -1,6 +1,8 @@
 import json
 import os
 
+from io import open
+
 from .license import License
 from .updater import Updater
 
@@ -26,7 +28,7 @@ def load_licenses():
 
     licenses_file = os.path.join(os.path.dirname(__file__), "data", "licenses.json")
 
-    with open(licenses_file) as f:
+    with open(licenses_file, encoding="utf-8") as f:
         data = json.loads(f.read())
 
     for name, license in data.items():

poetry/spdx/updater.py

@@ -1,6 +1,8 @@
 import json
 import os
 
+from io import open
+
 try:
     from urllib.request import urlopen
 except ImportError:
@@ -20,7 +22,7 @@ def dump(self, file=None):
 
         licenses_url = self._base_url + "licenses.json"
 
-        with open(file, "w") as f:
+        with open(file, "w", encoding="utf-8") as f:
             f.write(
                 json.dumps(self.get_licenses(licenses_url), indent=2, sort_keys=True)
             )

tests/console/conftest.py

@@ -176,15 +176,15 @@ def repo():
 def poetry(repo):
     p = Poetry.create(Path(__file__).parent.parent / "fixtures" / "simple_project")
 
-    with p.file.path.open() as f:
+    with p.file.path.open(encoding="utf-8") as f:
         content = f.read()
 
     p.pool.remove_repository("pypi")
     p.pool.add_repository(repo)
 
     yield p
 
-    with p.file.path.open("w") as f:
+    with p.file.path.open("w", encoding="utf-8") as f:
         f.write(content)
 
 

tests/masonry/test_api.py

@@ -117,11 +117,11 @@ def test_prepare_metadata_for_build_wheel():
         assert (dist_info / "WHEEL").exists()
         assert (dist_info / "METADATA").exists()
 
-        with (dist_info / "entry_points.txt").open() as f:
+        with (dist_info / "entry_points.txt").open(encoding="utf-8") as f:
             assert entry_points == decode(f.read())
 
-        with (dist_info / "WHEEL").open() as f:
+        with (dist_info / "WHEEL").open(encoding="utf-8") as f:
             assert wheel_data == decode(f.read())
 
-        with (dist_info / "METADATA").open() as f:
+        with (dist_info / "METADATA").open(encoding="utf-8") as f:
             assert metadata == decode(f.read())

tests/repositories/test_legacy_repository.py

@@ -29,7 +29,7 @@ def _get(self, endpoint):
 
         fixture = self.FIXTURES / (name + ".html")
 
-        with fixture.open() as f:
+        with fixture.open(encoding="utf-8") as f:
             return Page(self._url + endpoint, f.read(), {})
 
     def _download(self, url, dest):

tests/repositories/test_pypi_repository.py

@@ -33,7 +33,7 @@ def _get(self, url):
             if not fixture.exists():
                 fixture = self.JSON_FIXTURES / (name + ".json")
 
-        with fixture.open() as f:
+        with fixture.open(encoding="utf-8") as f:
             return json.loads(f.read())
 
     def _download(self, url, dest):