Skip to content

Commit

Permalink
Reduce size of JSON data in Mypy cache (#14808)
Browse files Browse the repository at this point in the history
By default, Python outputs spaces after commas and colons when dumping
JSON data. This is normally fine, but when exporting large amounts of
JSON (such as cache info) it is more space efficient to export JSON
without whitespaces. To do this, simply pass `separators=(",", ":")` to
`json.dumps()`.

Although the space savings aren't massive, they do reduce the size of
the cache folder by a couple of megabytes:

| Method  | Size   |
|---------|--------|
| Current | 31.2MB |
| This PR | 29.3MB |

For comparison, the size of the cache folder with the `--sqlite-cache`
option is about 27.5MB.
  • Loading branch information
dosisod authored Mar 1, 2023
1 parent dc5ff29 commit 9393c22
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 7 deletions.
13 changes: 7 additions & 6 deletions mypy/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -919,7 +919,7 @@ def stats_summary(self) -> Mapping[str, object]:


def deps_to_json(x: dict[str, set[str]]) -> str:
return json.dumps({k: list(v) for k, v in x.items()})
return json.dumps({k: list(v) for k, v in x.items()}, separators=(",", ":"))


# File for storing metadata about all the fine-grained dependency caches
Expand Down Expand Up @@ -987,7 +987,7 @@ def write_deps_cache(

meta = {"snapshot": meta_snapshot, "deps_meta": fg_deps_meta}

if not metastore.write(DEPS_META_FILE, json.dumps(meta)):
if not metastore.write(DEPS_META_FILE, json.dumps(meta, separators=(",", ":"))):
manager.log(f"Error writing fine-grained deps meta JSON file {DEPS_META_FILE}")
error = True

Expand Down Expand Up @@ -1055,7 +1055,8 @@ def generate_deps_for_cache(manager: BuildManager, graph: Graph) -> dict[str, di

def write_plugins_snapshot(manager: BuildManager) -> None:
"""Write snapshot of versions and hashes of currently active plugins."""
if not manager.metastore.write(PLUGIN_SNAPSHOT_FILE, json.dumps(manager.plugins_snapshot)):
snapshot = json.dumps(manager.plugins_snapshot, separators=(",", ":"))
if not manager.metastore.write(PLUGIN_SNAPSHOT_FILE, snapshot):
manager.errors.set_file(_cache_dir_prefix(manager.options), None, manager.options)
manager.errors.report(0, 0, "Error writing plugins snapshot", blocker=True)

Expand Down Expand Up @@ -1487,7 +1488,7 @@ def validate_meta(
if manager.options.debug_cache:
meta_str = json.dumps(meta_dict, indent=2, sort_keys=True)
else:
meta_str = json.dumps(meta_dict)
meta_str = json.dumps(meta_dict, separators=(",", ":"))
meta_json, _, _ = get_cache_names(id, path, manager.options)
manager.log(
"Updating mtime for {}: file {}, meta {}, mtime {}".format(
Expand Down Expand Up @@ -1517,7 +1518,7 @@ def json_dumps(obj: Any, debug_cache: bool) -> str:
if debug_cache:
return json.dumps(obj, indent=2, sort_keys=True)
else:
return json.dumps(obj, sort_keys=True)
return json.dumps(obj, sort_keys=True, separators=(",", ":"))


def write_cache(
Expand Down Expand Up @@ -3639,4 +3640,4 @@ def write_undocumented_ref_info(state: State, metastore: MetadataStore, options:
assert not ref_info_file.startswith(".")

deps_json = get_undocumented_ref_info_json(state.tree)
metastore.write(ref_info_file, json.dumps(deps_json))
metastore.write(ref_info_file, json.dumps(deps_json, separators=(",", ":")))
2 changes: 1 addition & 1 deletion mypyc/codegen/emitmodule.py
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,7 @@ def write_cache(
"src_hashes": hashes[group_map[id]],
}

result.manager.metastore.write(newpath, json.dumps(ir_data))
result.manager.metastore.write(newpath, json.dumps(ir_data, separators=(",", ":")))

result.manager.metastore.commit()

Expand Down

0 comments on commit 9393c22

Please sign in to comment.