Skip to content

Adds additional machine details to query_config #4759

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 20 commits into from
Mar 18, 2025
Merged
Show file tree
Hide file tree
Changes from 18 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions .devcontainer/cesm-build/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"containerEnv": {
"CIME_MODEL": "cesm",
"CIME_MACHINE": "docker",
"SKIP_CIME_UPDATE": "true",
"SKIP_MODEL_SETUP": "true"
},
"build": {
"dockerfile": "../../docker/Dockerfile",
"target": "base"
}, "workspaceFolder": "/src/cime",
"workspaceMount": "source=${localWorkspaceFolder}/..,target=/src,type=bind",
"postAttachCommand": "/entrypoint.sh bash",
"customizations": {
"vscode": {
"extensions": [
"charliermarsh.ruff"
]
}
},
"features": {
"ghcr.io/devcontainers/features/git:1": {}
}
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are these container changes part of this?

Copy link
Collaborator Author

@jasonb5 jasonb5 Mar 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yea I'll pull that out and make a separate PR. It's some additional dev stuff for anyone using vscode to simplify running/testing in a container locally. @jedwards4b @jgfouca Any issue if I add this in a different PR? It doesn't affect anything unless you're using vscode.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I actually don't care if you keep it in this PR as long as we understand why it's there and you add it to the description.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably best to pull it out of this PR. I'll be making a more testing focused PR soon, it'll fit in better there and I can provide a full explanation.

22 changes: 22 additions & 0 deletions .devcontainer/cesm/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
{
"containerEnv": {
"CIME_MODEL": "cesm",
"CIME_MACHINE": "docker",
"SKIP_CIME_UPDATE": "true",
"SKIP_MODEL_SETUP": "true"
},
"image": "ghcr.io/esmci/cime:latest",
"workspaceFolder": "/src/cime",
"workspaceMount": "source=${localWorkspaceFolder}/..,target=/src,type=bind",
"postAttachCommand": "/entrypoint.sh bash",
"customizations": {
"vscode": {
"extensions": [
"charliermarsh.ruff"
]
}
},
"features": {
"ghcr.io/devcontainers/features/git:1": {}
}
}
8 changes: 8 additions & 0 deletions .devcontainer/docs/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"image": "ghcr.io/esmci/cime:latest",
"forwardPorts": [8000],
"mounts": [
"source=${localWorkspaceFolder}/../,target=/src,type=bind"
],
"postStartCommand": "pip install -r doc/requirements.txt; pip install sphinx-autobuild; cd doc/source; sphinx-autobuild --ignore '**/__pycache__' . ../build/html",
}
25 changes: 25 additions & 0 deletions .devcontainer/e3sm-build/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
{
"containerEnv": {
"CIME_MODEL": "e3sm",
"CIME_MACHINE": "docker",
"SKIP_CIME_UPDATE": "true",
"SKIP_MODEL_SETUP": "true"
},
"build": {
"dockerfile": "../../docker/Dockerfile",
"target": "base"
},
"workspaceFolder": "/src/cime",
"workspaceMount": "source=${localWorkspaceFolder}/..,target=/src,type=bind",
"postAttachCommand": "/entrypoint.sh bash",
"customizations": {
"vscode": {
"extensions": [
"charliermarsh.ruff"
]
}
},
"features": {
"ghcr.io/devcontainers/features/git:1": {}
}
}
22 changes: 22 additions & 0 deletions .devcontainer/e3sm/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
{
"containerEnv": {
"CIME_MODEL": "e3sm",
"CIME_MACHINE": "docker",
"SKIP_CIME_UPDATE": "true",
"SKIP_MODEL_SETUP": "true"
},
"image": "ghcr.io/esmci/cime:latest",
"workspaceFolder": "/src/cime",
"workspaceMount": "source=${localWorkspaceFolder}/..,target=/src,type=bind",
"postAttachCommand": "/entrypoint.sh bash",
"customizations": {
"vscode": {
"extensions": [
"charliermarsh.ruff"
]
}
},
"features": {
"ghcr.io/devcontainers/features/git:1": {}
}
}
21 changes: 21 additions & 0 deletions .vscode/tasks.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "Build docs",
"type": "shell",
"command": "make html",
"options": {
"cwd": "${workspaceFolder}/doc"
},
},
{
"label": "Autobuild docs",
"type": "shell",
"command": "sphinx-autobuild --ignore '**/__pycache__' . ../build/html",
"options": {
"cwd": "${workspaceFolder}/doc/source"
}
}
]
}
223 changes: 200 additions & 23 deletions CIME/XML/machines.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,45 @@
"""
Interface to the config_machines.xml file. This class inherits from GenericXML.py
"""

from CIME.XML.standard_module_setup import *
from CIME.XML.generic_xml import GenericXML
from CIME.XML.files import Files
from CIME.utils import convert_to_unknown_type, get_cime_config
from CIME.utils import CIMEError, expect, convert_to_unknown_type, get_cime_config

import re
import logging
import socket
from functools import partial
from pathlib import Path

logger = logging.getLogger(__name__)


def match_value_by_attribute_regex(element, attribute_name, value):
"""Checks element contains attribute whose pattern matches a value.

If the element does not have the attribute it's considered a match.

Args:
element (CIME.XML.generic_xml._Element): XML element to check attributes.
attribute_name (str): Name of attribute with regex value.
value (str): Value that is matched against attributes regex value.

Returns:
bool: True if attribute regex matches the target value otherwise False.
"""
attribute_value = element.attrib.get(attribute_name, None)

return (
True
if value is None
or attribute_value is None
or re.match(attribute_value, value) is not None
else False
)


class Machines(GenericXML):
def __init__(
self,
Expand Down Expand Up @@ -484,30 +512,179 @@ def set_value(self, vid, value, subgroup=None, ignore_type=True):
# A temporary cache only
self.custom_settings[vid] = value

def print_values(self):
# write out machines
machines = self.get_children("machine")
logger.info("Machines")
for machine in machines:
name = self.get(machine, "MACH")
desc = self.get_child("DESC", root=machine)
os_ = self.get_child("OS", root=machine)
compilers = self.get_child("COMPILERS", root=machine)
max_tasks_per_node = self.get_child("MAX_TASKS_PER_NODE", root=machine)
max_mpitasks_per_node = self.get_child(
"MAX_MPITASKS_PER_NODE", root=machine
def print_values(self, compiler=None):
"""Prints machine values.

Args:
compiler (str, optional): Name of the compiler to print extra details for. Defaults to None.
"""
current = self.probe_machine_name(False)

if self.machine_node is None:
for machine in self.get_children("machine"):
self._print_machine_values(machine, current)
else:
self._print_machine_values(self.machine_node, current, compiler)

def _print_machine_values(self, machine, current=None, compiler=None):
"""Prints a machines details.

Args:
machine (CIME.XML.machines.Machine): Machine object.
current (str, optional): Name of the current machine. Defaults to None.
compiler (str, optional): If not None, then modules and environment variables matching compiler are printed. Defaults to None.

Raises:
CIMEError: If `compiler` is not valid.
"""
name = self.get(machine, "MACH")
if current is not None and current == name:
name = f"{name} (current)"
desc = self.text(self.get_child("DESC", root=machine))
os_ = self.text(self.get_child("OS", root=machine))

compilers = self.text(self.get_child("COMPILERS", root=machine))
if compiler is not None and compiler not in compilers.split(","):
raise CIMEError(
f"Compiler {compiler!r} is not a valid choice from ({compilers})"
)
max_gpus_per_node = self.get_child("MAX_GPUS_PER_NODE", root=machine)

print(" {} : {} ".format(name, self.text(desc)))
print(" os ", self.text(os_))
print(" compilers ", self.text(compilers))
if max_mpitasks_per_node is not None:
print(" pes/node ", self.text(max_mpitasks_per_node))
if max_tasks_per_node is not None:
print(" max_tasks/node ", self.text(max_tasks_per_node))
if max_gpus_per_node is not None:
print(" max_gpus/node ", self.text(max_gpus_per_node))
mpilibs_nodes = self._get_children_filter_attribute_regex(
"MPILIBS", "compiler", compiler, root=machine
)
mpilibs = set([y for x in mpilibs_nodes for y in self.text(x).split(",")])

max_tasks_per_node = self.text(
self.get_child("MAX_TASKS_PER_NODE", root=machine)
)
max_mpitasks_per_node = self.text(
self.get_child("MAX_MPITASKS_PER_NODE", root=machine)
)
max_gpus_per_node = self.get_optional_child("MAX_GPUS_PER_NODE", root=machine)
max_gpus_per_node_text = (
self.text(max_gpus_per_node) if max_gpus_per_node else 0
)

if compiler is not None:
name = f"{name} ({compiler})"

print(" {} : {} ".format(name, desc))
print(" os ", os_)
print(" compilers ", compilers)
print(" mpilibs ", ",".join(mpilibs))
print(" pes/node ", max_mpitasks_per_node)
print(" max_tasks/node ", max_tasks_per_node)
print(" max_gpus/node ", max_gpus_per_node_text)
print("")

if compiler is not None:
module_system_node = self.get_child("module_system", root=machine)

def command_formatter(node):
if node.text is None:
return f"{node.attrib['name']}"
else:
return f"{node.attrib['name']} {node.text}"

print(" Module commands:")
for requirements, commands in self._filter_children_by_compiler(
"modules", "command", compiler, command_formatter, module_system_node
):
indent = "" if requirements == "" else " "
if requirements != "":
print(f" (with {requirements})")
for x in commands:
print(f" {indent}{x}")
print("")

def env_formatter(node, machines=None):
return f"{node.attrib['name']}: {machines._get_resolved_environment_variable(node.text)}"

print(" Environment variables:")
for requirements, variables in self._filter_children_by_compiler(
"environment_variables",
"env",
compiler,
partial(env_formatter, machines=self),
machine,
):
indent = "" if requirements == "" else " "
if requirements != "":
print(f" (with {requirements})")
for x in variables:
print(f" {indent}{x}")

def _filter_children_by_compiler(self, parent, child, compiler, formatter, root):
"""Filters parent nodes and returns requirements and children of filtered nodes.

Example of a yielded values:

"mpilib=openmpi DEBUG=true", ["HOME: /home/dev", "NETCDF_C_PATH: ../netcdf"]

Args:
parent (str): Name of the nodes to filter.
child (str): Name of the children nodes from filtered parent nodes.
compiler (str): Name of the compiler that will be matched against the regex.
formatter (function): Function to format the child nodes from the parents that match.
root (CIME.XML.generic_xml._Element): Root node to filter parent nodes from.

Yields:
str, list: Requirements for parent node and list of formated child nodes.
"""
nodes = self._get_children_filter_attribute_regex(
parent, "compiler", compiler, root=root
)

for x in nodes:
attrib = {**x.attrib}
attrib.pop("compiler", None)

requirements = " ".join([f"{y}={z!r}" for y, z in attrib.items()])
values = [formatter(y) for y in self.get_children(child, root=x)]

yield requirements, values

def _get_children_filter_attribute_regex(self, name, attribute_name, value, root):
"""Filter children nodes using regex.

Uses regex from attribute of children nodes to match a value.

Args:
name (str): Name of the children nodes.
attribute_name (str): Name of the attribute on the child nodes to build regex from.
value (str): Value that is matched using regex from attribute.
root (CIME.XML.generic_xml._Element): Root node to query children nodes from.

Returns:
list: List of children whose regex attribute matches the value.
"""
return [
x
for x in self.get_children(name, root=root)
if match_value_by_attribute_regex(x, attribute_name, value)
]

def _get_resolved_environment_variable(self, text):
"""Attempts to resolve machines environment variable.

Args:
text (str): Environment variable value.

Returns:
str: Resolved value or error message.
"""
if text is None:
return ""

try:
value = self.get_resolved_value(text, allow_unresolved_envvars=True)
except Exception as e:
return f"Failed to resolve {text!r} with: {e!s}"

if value == text and "$" in text:
value = f"Failed to resolve {text!r}"

return value

def return_values(self):
"""return a dictionary of machine info
Expand Down
Loading
Loading