Skip to content

Commit 09a0fb5

Browse files
andy31415pull[bot]
authored andcommitted
Python code capable of parsing matter IDL files (including some unit tests) (#13725)
* A IDL parser: Can parse current IDL format (but that may change). Has working unit tests. * one more test * minor comment * make the structs a bit more compact: easier to read * one more tests * more tests, fixed one bug * Add unit test for cluster commands * Unit test for cluster enums * Unit test for cluster events * Rename "structure_member" to field since that seems easier to type and is a good term * Match the newest attribute format for IDLs * Allow test_parsing to be run stand alone and hope that this fix also fixes mac * Rename "parser" to a more specific name: the name parser is used in python and is too generic to attempt a top level import on it * Restyle fixes * Add support for global tag parsing after idl updated in master * Add support for datatype sizes and unit tests * Add test for sized strings in structs as well * Ran restyler
1 parent f48f37f commit 09a0fb5

9 files changed

+768
-0
lines changed

BUILD.gn

+1
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
160160
deps = [
161161
"//:fake_platform_tests",
162162
"//scripts/build:build_examples.tests",
163+
"//scripts/idl:idl.tests",
163164
"//src:tests_run",
164165
]
165166
}

scripts/idl/BUILD.gn

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Copyright (c) 2022 Project CHIP Authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import("//build_overrides/build.gni")
16+
import("//build_overrides/chip.gni")
17+
18+
import("//build_overrides/pigweed.gni")
19+
import("$dir_pw_build/python.gni")
20+
21+
pw_python_package("idl") {
22+
setup = [ "setup.py" ]
23+
inputs = [
24+
# Dependency grammar
25+
"matter_grammar.lark",
26+
]
27+
28+
sources = [
29+
"__init__.py",
30+
"matter_idl_parser.py",
31+
"matter_idl_types.py",
32+
]
33+
34+
tests = [ "test_matter_idl_parser.py" ]
35+
}

scripts/idl/__init__.py

Whitespace-only changes.

scripts/idl/matter_grammar.lark

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
struct: "struct"i id "{" struct_field* "}"
2+
enum: "enum"i id ":" data_type "{" enum_entry* "}"
3+
4+
event: event_priority "event"i id "=" number "{" struct_field* "}"
5+
6+
?event_priority: "critical"i -> critical_priority
7+
| "info"i -> info_priority
8+
| "debug"i -> debug_priority
9+
10+
attribute: attribute_tag* "attribute"i field
11+
attribute_tag: "readonly"i -> attr_readonly
12+
| "global"i -> attr_global
13+
14+
request_struct: "request"i struct
15+
response_struct: "response"i struct
16+
17+
command: "command"i id "(" id? ")" ":" id "=" number ";"
18+
19+
cluster: cluster_side "cluster"i id "=" number "{" (enum|event|attribute|struct|request_struct|response_struct|command)* "}"
20+
?cluster_side: "server"i -> server_cluster
21+
| "client"i -> client_cluster
22+
23+
endpoint: "endpoint"i number "{" endpoint_cluster* "}"
24+
endpoint_cluster: endpoint_cluster_type "cluster"i id ";"
25+
26+
?endpoint_cluster_type: "server"i -> endpoint_server_cluster
27+
| "binding"i -> endpoint_binding_to_cluster
28+
29+
enum_entry: id "=" number ";"
30+
number: POSITIVE_INTEGER | HEX_INTEGER
31+
32+
struct_field: member_attribute* field
33+
34+
member_attribute: "optional"i -> optional
35+
| "nullable"i -> nullable
36+
37+
field: data_type id list_marker? "=" number ";"
38+
list_marker: "[" "]"
39+
40+
data_type: type ("<" number ">")?
41+
42+
id: ID
43+
type: ID
44+
45+
COMMENT: "{" /(.|\n)+/ "}"
46+
| "//" /.*/
47+
48+
POSITIVE_INTEGER: /\d+/
49+
HEX_INTEGER: /0x[A-Fa-f0-9]+/
50+
ID: /[a-zA-Z_][a-zA-Z0-9_]*/
51+
52+
idl: (struct|enum|cluster|endpoint)*
53+
54+
%import common.WS
55+
%import common.C_COMMENT
56+
%import common.CPP_COMMENT
57+
%ignore WS
58+
%ignore C_COMMENT
59+
%ignore CPP_COMMENT

scripts/idl/matter_idl_parser.py

+240
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
#!/usr/bin/env python
2+
3+
import logging
4+
5+
from lark import Lark
6+
from lark.visitors import Transformer, v_args
7+
8+
try:
9+
from .matter_idl_types import *
10+
except:
11+
import os
12+
import sys
13+
sys.path.append(os.path.abspath(os.path.dirname(__file__)))
14+
15+
from matter_idl_types import *
16+
17+
18+
class MatterIdlTransformer(Transformer):
19+
"""A transformer capable to transform data
20+
parsed by Lark according to matter_grammar.lark
21+
"""
22+
23+
def number(self, tokens):
24+
"""Numbers in the grammar are integers or hex numbers.
25+
"""
26+
if len(tokens) != 1:
27+
raise Error("Unexpected argument counts")
28+
29+
n = tokens[0].value
30+
if n.startswith('0x'):
31+
return int(n[2:], 16)
32+
else:
33+
return int(n)
34+
35+
def id(self, tokens):
36+
"""An id is a string containing an identifier
37+
"""
38+
if len(tokens) != 1:
39+
raise Error("Unexpected argument counts")
40+
return tokens[0].value
41+
42+
def type(self, tokens):
43+
"""A type is just a string for the type
44+
"""
45+
if len(tokens) != 1:
46+
raise Error("Unexpected argument counts")
47+
return tokens[0].value
48+
49+
def data_type(self, tokens):
50+
if len(tokens) == 1:
51+
return DataType(name=tokens[0])
52+
# Just a string for data type
53+
elif len(tokens) == 2:
54+
return DataType(name=tokens[0], max_length=tokens[1])
55+
else:
56+
raise Error("Unexpected size for data type")
57+
58+
@v_args(inline=True)
59+
def enum_entry(self, id, number):
60+
return EnumEntry(name=id, code=number)
61+
62+
@v_args(inline=True)
63+
def enum(self, id, type, *entries):
64+
return Enum(name=id, base_type=type, entries=list(entries))
65+
66+
def field(self, args):
67+
data_type, name = args[0], args[1]
68+
is_list = (len(args) == 4)
69+
code = args[-1]
70+
71+
return Field(data_type=data_type, name=name, code=code, is_list=is_list)
72+
73+
def optional(self, _):
74+
return FieldAttribute.OPTIONAL
75+
76+
def nullable(self, _):
77+
return FieldAttribute.NULLABLE
78+
79+
def attr_readonly(self, _):
80+
return AttributeTag.READABLE
81+
82+
def attr_global(self, _):
83+
return AttributeTag.GLOBAL
84+
85+
def critical_priority(self, _):
86+
return EventPriority.CRITICAL
87+
88+
def info_priority(self, _):
89+
return EventPriority.INFO
90+
91+
def debug_priority(self, _):
92+
return EventPriority.DEBUG
93+
94+
def endpoint_server_cluster(self, _):
95+
return EndpointContentType.SERVER_CLUSTER
96+
97+
def endpoint_binding_to_cluster(self, _):
98+
return EndpointContentType.CLIENT_BINDING
99+
100+
def struct_field(self, args):
101+
# Last argument is the named_member, the rest
102+
# are attributes
103+
field = args[-1]
104+
field.attributes = set(args[:-1])
105+
return field
106+
107+
def server_cluster(self, _):
108+
return ClusterSide.SERVER
109+
110+
def client_cluster(self, _):
111+
return ClusterSide.CLIENT
112+
113+
def command(self, args):
114+
# A command has 3 arguments if no input or
115+
# 4 arguments if input parameter is available
116+
param_in = None
117+
if len(args) > 3:
118+
param_in = args[1]
119+
return Command(name=args[0], input_param=param_in, output_param=args[-2], code=args[-1])
120+
121+
def event(self, args):
122+
return Event(priority=args[0], name=args[1], code=args[2], fields=args[3:], )
123+
124+
def attribute(self, args):
125+
tags = set(args[:-1])
126+
# until we support write only (and need a bit of a reshuffle)
127+
# if the 'attr_readonly == READABLE' is not in the list, we make things
128+
# read/write
129+
if AttributeTag.READABLE not in tags:
130+
tags.add(AttributeTag.READABLE)
131+
tags.add(AttributeTag.WRITABLE)
132+
133+
return Attribute(definition=args[-1], tags=tags)
134+
135+
@v_args(inline=True)
136+
def struct(self, id, *fields):
137+
return Struct(name=id, fields=list(fields))
138+
139+
@v_args(inline=True)
140+
def request_struct(self, value):
141+
value.tag = StructTag.REQUEST
142+
return value
143+
144+
@v_args(inline=True)
145+
def response_struct(self, value):
146+
value.tag = StructTag.RESPONSE
147+
return value
148+
149+
@v_args(inline=True)
150+
def endpoint(self, number, *clusters):
151+
endpoint = Endpoint(number=number)
152+
153+
for t, name in clusters:
154+
if t == EndpointContentType.CLIENT_BINDING:
155+
endpoint.client_bindings.append(name)
156+
elif t == EndpointContentType.SERVER_CLUSTER:
157+
endpoint.server_clusters.append(name)
158+
else:
159+
raise Error("Unknown endpoint content: %r" % t)
160+
161+
return endpoint
162+
163+
@v_args(inline=True)
164+
def endpoint_cluster(self, t, id):
165+
return (t, id)
166+
167+
@v_args(inline=True)
168+
def cluster(self, side, name, code, *content):
169+
result = Cluster(side=side, name=name, code=code)
170+
171+
for item in content:
172+
if type(item) == Enum:
173+
result.enums.append(item)
174+
elif type(item) == Event:
175+
result.events.append(item)
176+
elif type(item) == Attribute:
177+
result.attributes.append(item)
178+
elif type(item) == Struct:
179+
result.structs.append(item)
180+
elif type(item) == Command:
181+
result.commands.append(item)
182+
else:
183+
raise Error("UNKNOWN cluster content item: %r" % item)
184+
185+
return result
186+
187+
def idl(self, items):
188+
idl = Idl()
189+
190+
for item in items:
191+
if type(item) == Enum:
192+
idl.enums.append(item)
193+
elif type(item) == Struct:
194+
idl.structs.append(item)
195+
elif type(item) == Cluster:
196+
idl.clusters.append(item)
197+
elif type(item) == Endpoint:
198+
idl.endpoints.append(item)
199+
else:
200+
raise Error("UNKNOWN idl content item: %r" % item)
201+
202+
return idl
203+
204+
205+
def CreateParser():
206+
return Lark.open('matter_grammar.lark', rel_to=__file__, start='idl', parser='lalr', transformer=MatterIdlTransformer())
207+
208+
209+
if __name__ == '__main__':
210+
import click
211+
import coloredlogs
212+
213+
# Supported log levels, mapping string values required for argument
214+
# parsing into logging constants
215+
__LOG_LEVELS__ = {
216+
'debug': logging.DEBUG,
217+
'info': logging.INFO,
218+
'warn': logging.WARN,
219+
'fatal': logging.FATAL,
220+
}
221+
222+
@click.command()
223+
@click.option(
224+
'--log-level',
225+
default='INFO',
226+
type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False),
227+
help='Determines the verbosity of script output.')
228+
@click.argument('filename')
229+
def main(log_level, filename=None):
230+
coloredlogs.install(level=__LOG_LEVELS__[
231+
log_level], fmt='%(asctime)s %(levelname)-7s %(message)s')
232+
233+
logging.info("Starting to parse ...")
234+
data = CreateParser().parse(open(filename).read())
235+
logging.info("Parse completed")
236+
237+
logging.info("Data:")
238+
print(data)
239+
240+
main()

0 commit comments

Comments
 (0)