Skip to content

Commit ffec188

Browse files
justin-tayjustin-tay
authoredApr 16, 2024
Add options to control caching of schemas (#1018)
1 parent 0a350ff commit ffec188

File tree

9 files changed

+11247
-36
lines changed

9 files changed

+11247
-36
lines changed
 

‎README.md

+13-10
Original file line numberDiff line numberDiff line change
@@ -461,16 +461,19 @@ The following is sample output from the Hierarchical format.
461461

462462
### Schema Validators Configuration
463463

464-
| Name | Description | Default Value
465-
|--------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------
466-
| `pathType` | The path type to use for reporting the instance location and evaluation path. Set to `PathType.JSON_POINTER` to use JSON Pointer. | `PathType.DEFAULT`
467-
| `ecma262Validator` | Whether to use the ECMA 262 `joni` library to validate the `pattern` keyword. This requires the dependency to be manually added to the project or a `ClassNotFoundException` will be thrown. | `false`
468-
| `executionContextCustomizer` | This can be used to customize the `ExecutionContext` generated by the `JsonSchema` for each validation run. | `null`
469-
| `schemaIdValidator` | This is used to customize how the `$id` values are validated. Note that the default implementation allows non-empty fragments where no base IRI is specified and also allows non-absolute IRI `$id` values in the root schema. | `JsonSchemaIdValidator.DEFAULT`
470-
| `messageSource` | This is used to retrieve the locale specific messages. | `DefaultMessageSource.getInstance()`
471-
| `locale` | The locale to use for generating messages in the `ValidationMessage`. | `Locale.getDefault()`
472-
| `failFast` | Whether to return failure immediately when an assertion is generated. | `false`
473-
| `formatAssertionsEnabled` | The default is to generate format assertions from Draft 4 to Draft 7 and to only generate annotations from Draft 2019-09. Setting to `true` or `false` will override the default behavior. | `null`
464+
| Name | Description | Default Value
465+
|---------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------
466+
| `pathType` | The path type to use for reporting the instance location and evaluation path. Set to `PathType.JSON_POINTER` to use JSON Pointer. | `PathType.DEFAULT`
467+
| `ecma262Validator` | Whether to use the ECMA 262 `joni` library to validate the `pattern` keyword. This requires the dependency to be manually added to the project or a `ClassNotFoundException` will be thrown. | `false`
468+
| `executionContextCustomizer` | This can be used to customize the `ExecutionContext` generated by the `JsonSchema` for each validation run. | `null`
469+
| `schemaIdValidator` | This is used to customize how the `$id` values are validated. Note that the default implementation allows non-empty fragments where no base IRI is specified and also allows non-absolute IRI `$id` values in the root schema. | `JsonSchemaIdValidator.DEFAULT`
470+
| `messageSource` | This is used to retrieve the locale specific messages. | `DefaultMessageSource.getInstance()`
471+
| `preloadJsonSchema` | Whether the schema will be preloaded before processing any input. This will use memory but the execution of the validation will be faster. | `true`
472+
| `preloadJsonSchemaRefMaxNestingDepth` | The max depth of the evaluation path to preload when preloading refs. | `40`
473+
| `cacheRefs` | Whether the schemas loaded from refs will be cached and reused for subsequent runs. Setting this to `false` will affect performance but may be neccessary to prevent high memory usage for the cache if multiple nested applicators like `anyOf`, `oneOf` and `allOf` are used. | `true`
474+
| `locale` | The locale to use for generating messages in the `ValidationMessage`. | `Locale.getDefault()`
475+
| `failFast` | Whether to return failure immediately when an assertion is generated. | `false`
476+
| `formatAssertionsEnabled` | The default is to generate format assertions from Draft 4 to Draft 7 and to only generate annotations from Draft 2019-09. Setting to `true` or `false` will override the default behavior. | `null`
474477

475478
## Performance Considerations
476479

‎src/main/java/com/networknt/schema/DynamicRefValidator.java

+14-5
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020
import org.slf4j.Logger;
2121
import org.slf4j.LoggerFactory;
2222

23-
import java.util.*;
23+
import java.util.Collections;
24+
import java.util.Set;
25+
import java.util.function.Supplier;
2426

2527
/**
2628
* {@link JsonValidator} that resolves $dynamicRef.
@@ -39,7 +41,7 @@ public DynamicRefValidator(SchemaLocation schemaLocation, JsonNodePath evaluatio
3941
static JsonSchemaRef getRefSchema(JsonSchema parentSchema, ValidationContext validationContext, String refValue,
4042
JsonNodePath evaluationPath) {
4143
String ref = resolve(parentSchema, refValue);
42-
return new JsonSchemaRef(new CachedSupplier<>(() -> {
44+
return new JsonSchemaRef(getSupplier(() -> {
4345
JsonSchema refSchema = validationContext.getDynamicAnchors().get(ref);
4446
if (refSchema == null) { // This is a $dynamicRef without a matching $dynamicAnchor
4547
// A $dynamicRef without a matching $dynamicAnchor in the same schema resource
@@ -73,9 +75,13 @@ static JsonSchemaRef getRefSchema(JsonSchema parentSchema, ValidationContext val
7375
refSchema = refSchema.fromRef(parentSchema, evaluationPath);
7476
}
7577
return refSchema;
76-
}));
78+
}, validationContext.getConfig().isCacheRefs()));
7779
}
78-
80+
81+
static <T> Supplier<T> getSupplier(Supplier<T> supplier, boolean cache) {
82+
return cache ? new CachedSupplier<>(supplier) : supplier;
83+
}
84+
7985
private static String resolve(JsonSchema parentSchema, String refValue) {
8086
// $ref prevents a sibling $id from changing the base uri
8187
JsonSchema base = parentSchema;
@@ -153,14 +159,17 @@ public void preloadJsonSchema() {
153159
SchemaLocation schemaLocation = jsonSchema.getSchemaLocation();
154160
JsonSchema check = jsonSchema;
155161
boolean circularDependency = false;
162+
int depth = 0;
156163
while (check.getEvaluationParentSchema() != null) {
164+
depth++;
157165
check = check.getEvaluationParentSchema();
158166
if (check.getSchemaLocation().equals(schemaLocation)) {
159167
circularDependency = true;
160168
break;
161169
}
162170
}
163-
if (!circularDependency) {
171+
if (this.validationContext.getConfig().isCacheRefs() && !circularDependency
172+
&& depth < this.validationContext.getConfig().getPreloadJsonSchemaRefMaxNestingDepth()) {
164173
jsonSchema.initializeValidators();
165174
}
166175
}

‎src/main/java/com/networknt/schema/JsonNodePath.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -227,8 +227,8 @@ public boolean equals(Object obj) {
227227
if (getClass() != obj.getClass())
228228
return false;
229229
JsonNodePath other = (JsonNodePath) obj;
230-
return Objects.equals(parent, other.parent) && Objects.equals(pathSegment, other.pathSegment)
231-
&& pathSegmentIndex == other.pathSegmentIndex && type == other.type;
230+
return Objects.equals(pathSegment, other.pathSegment) && pathSegmentIndex == other.pathSegmentIndex
231+
&& type == other.type && Objects.equals(parent, other.parent);
232232
}
233233

234234
@Override

‎src/main/java/com/networknt/schema/JsonSchemaFactory.java

+26-3
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,17 @@ protected JsonSchema newJsonSchema(final SchemaLocation schemaUri, final JsonNod
287287
final ValidationContext validationContext = createValidationContext(schemaNode, config);
288288
JsonSchema jsonSchema = doCreate(validationContext, getSchemaLocation(schemaUri),
289289
new JsonNodePath(validationContext.getConfig().getPathType()), schemaNode, null, false);
290+
preload(jsonSchema, config);
291+
return jsonSchema;
292+
}
293+
294+
/**
295+
* Preloads the json schema if the configuration option is set.
296+
*
297+
* @param jsonSchema the schema to preload
298+
* @param config containing the configuration option
299+
*/
300+
private void preload(JsonSchema jsonSchema, SchemaValidatorsConfig config) {
290301
if (config.isPreloadJsonSchema()) {
291302
try {
292303
/*
@@ -302,7 +313,6 @@ protected JsonSchema newJsonSchema(final SchemaLocation schemaUri, final JsonNod
302313
*/
303314
}
304315
}
305-
return jsonSchema;
306316
}
307317

308318
public JsonSchema create(ValidationContext validationContext, SchemaLocation schemaLocation, JsonNodePath evaluationPath, JsonNode schemaNode, JsonSchema parentSchema) {
@@ -471,7 +481,7 @@ public JsonSchema getSchema(final InputStream schemaStream, final SchemaValidato
471481
public JsonSchema getSchema(final InputStream schemaStream) {
472482
return getSchema(schemaStream, createSchemaValidatorsConfig());
473483
}
474-
484+
475485
/**
476486
* Gets the schema.
477487
*
@@ -480,6 +490,19 @@ public JsonSchema getSchema(final InputStream schemaStream) {
480490
* @return the schema
481491
*/
482492
public JsonSchema getSchema(final SchemaLocation schemaUri, final SchemaValidatorsConfig config) {
493+
JsonSchema schema = loadSchema(schemaUri, config);
494+
preload(schema, config);
495+
return schema;
496+
}
497+
498+
/**
499+
* Loads the schema.
500+
*
501+
* @param schemaUri the absolute IRI of the schema which can map to the retrieval IRI.
502+
* @param config the config
503+
* @return the schema
504+
*/
505+
protected JsonSchema loadSchema(final SchemaLocation schemaUri, final SchemaValidatorsConfig config) {
483506
if (enableSchemaCache) {
484507
// ConcurrentHashMap computeIfAbsent does not allow calls that result in a
485508
// recursive update to the map.
@@ -500,7 +523,7 @@ public JsonSchema getSchema(final SchemaLocation schemaUri, final SchemaValidato
500523
}
501524
return getMappedSchema(schemaUri, config);
502525
}
503-
526+
504527
protected ObjectMapper getYamlMapper() {
505528
return this.yamlMapper != null ? this.yamlMapper : YamlMapperFactory.getInstance();
506529
}

‎src/main/java/com/networknt/schema/RecursiveRefValidator.java

+14-5
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020
import org.slf4j.Logger;
2121
import org.slf4j.LoggerFactory;
2222

23-
import java.util.*;
23+
import java.util.Collections;
24+
import java.util.Set;
25+
import java.util.function.Supplier;
2426

2527
/**
2628
* {@link JsonValidator} that resolves $recursiveRef.
@@ -47,11 +49,15 @@ public RecursiveRefValidator(SchemaLocation schemaLocation, JsonNodePath evaluat
4749

4850
static JsonSchemaRef getRefSchema(JsonSchema parentSchema, ValidationContext validationContext, String refValue,
4951
JsonNodePath evaluationPath) {
50-
return new JsonSchemaRef(new CachedSupplier<>(() -> {
52+
return new JsonSchemaRef(getSupplier(() -> {
5153
return getSchema(parentSchema, validationContext, refValue, evaluationPath);
52-
}));
54+
}, validationContext.getConfig().isCacheRefs()));
5355
}
54-
56+
57+
static <T> Supplier<T> getSupplier(Supplier<T> supplier, boolean cache) {
58+
return cache ? new CachedSupplier<>(supplier) : supplier;
59+
}
60+
5561
static JsonSchema getSchema(JsonSchema parentSchema, ValidationContext validationContext, String refValue,
5662
JsonNodePath evaluationPath) {
5763
JsonSchema refSchema = parentSchema.findSchemaResourceRoot(); // Get the document
@@ -150,14 +156,17 @@ public void preloadJsonSchema() {
150156
SchemaLocation schemaLocation = jsonSchema.getSchemaLocation();
151157
JsonSchema check = jsonSchema;
152158
boolean circularDependency = false;
159+
int depth = 0;
153160
while (check.getEvaluationParentSchema() != null) {
161+
depth++;
154162
check = check.getEvaluationParentSchema();
155163
if (check.getSchemaLocation().equals(schemaLocation)) {
156164
circularDependency = true;
157165
break;
158166
}
159167
}
160-
if (!circularDependency) {
168+
if (this.validationContext.getConfig().isCacheRefs() && !circularDependency
169+
&& depth < this.validationContext.getConfig().getPreloadJsonSchemaRefMaxNestingDepth()) {
161170
jsonSchema.initializeValidators();
162171
}
163172
}

‎src/main/java/com/networknt/schema/RefValidator.java

+22-11
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020
import org.slf4j.Logger;
2121
import org.slf4j.LoggerFactory;
2222

23-
import java.util.*;
23+
import java.util.Collections;
24+
import java.util.Set;
25+
import java.util.function.Supplier;
2426

2527
/**
2628
* {@link JsonValidator} that resolves $ref.
@@ -58,10 +60,10 @@ static JsonSchemaRef getRefSchema(JsonSchema parentSchema, ValidationContext val
5860
String schemaUriFinal = resolve(parentSchema, refUri);
5961
SchemaLocation schemaLocation = SchemaLocation.of(schemaUriFinal);
6062
// This should retrieve schemas regardless of the protocol that is in the uri.
61-
return new JsonSchemaRef(new CachedSupplier<>(() -> {
63+
return new JsonSchemaRef(getSupplier(() -> {
6264
JsonSchema schemaResource = validationContext.getSchemaResources().get(schemaUriFinal);
6365
if (schemaResource == null) {
64-
schemaResource = validationContext.getJsonSchemaFactory().getSchema(schemaLocation, validationContext.getConfig());
66+
schemaResource = validationContext.getJsonSchemaFactory().loadSchema(schemaLocation, validationContext.getConfig());
6567
if (schemaResource != null) {
6668
copySchemaResources(validationContext, schemaResource);
6769
}
@@ -89,12 +91,12 @@ static JsonSchemaRef getRefSchema(JsonSchema parentSchema, ValidationContext val
8991
}
9092
return schemaResource.fromRef(parentSchema, evaluationPath);
9193
}
92-
}));
94+
}, validationContext.getConfig().isCacheRefs()));
9395

9496
} else if (SchemaLocation.Fragment.isAnchorFragment(refValue)) {
9597
String absoluteIri = resolve(parentSchema, refValue);
9698
// Schema resource needs to update the parent and evaluation path
97-
return new JsonSchemaRef(new CachedSupplier<>(() -> {
99+
return new JsonSchemaRef(getSupplier(() -> {
98100
JsonSchema schemaResource = validationContext.getSchemaResources().get(absoluteIri);
99101
if (schemaResource == null) {
100102
schemaResource = validationContext.getDynamicAnchors().get(absoluteIri);
@@ -106,15 +108,21 @@ static JsonSchemaRef getRefSchema(JsonSchema parentSchema, ValidationContext val
106108
return null;
107109
}
108110
return schemaResource.fromRef(parentSchema, evaluationPath);
109-
}));
111+
}, validationContext.getConfig().isCacheRefs()));
110112
}
111113
if (refValue.equals(REF_CURRENT)) {
112-
return new JsonSchemaRef(new CachedSupplier<>(
113-
() -> parentSchema.findSchemaResourceRoot().fromRef(parentSchema, evaluationPath)));
114+
return new JsonSchemaRef(
115+
getSupplier(() -> parentSchema.findSchemaResourceRoot().fromRef(parentSchema, evaluationPath),
116+
validationContext.getConfig().isCacheRefs()));
114117
}
115-
return new JsonSchemaRef(new CachedSupplier<>(
118+
return new JsonSchemaRef(getSupplier(
116119
() -> getJsonSchema(parentSchema, validationContext, refValue, refValueOriginal, evaluationPath)
117-
.fromRef(parentSchema, evaluationPath)));
120+
.fromRef(parentSchema, evaluationPath),
121+
validationContext.getConfig().isCacheRefs()));
122+
}
123+
124+
static <T> Supplier<T> getSupplier(Supplier<T> supplier, boolean cache) {
125+
return cache ? new CachedSupplier<>(supplier) : supplier;
118126
}
119127

120128
private static void copySchemaResources(ValidationContext validationContext, JsonSchema schemaResource) {
@@ -235,14 +243,17 @@ public void preloadJsonSchema() {
235243
SchemaLocation schemaLocation = jsonSchema.getSchemaLocation();
236244
JsonSchema check = jsonSchema;
237245
boolean circularDependency = false;
246+
int depth = 0;
238247
while (check.getEvaluationParentSchema() != null) {
248+
depth++;
239249
check = check.getEvaluationParentSchema();
240250
if (check.getSchemaLocation().equals(schemaLocation)) {
241251
circularDependency = true;
242252
break;
243253
}
244254
}
245-
if (!circularDependency) {
255+
if (this.validationContext.getConfig().isCacheRefs() && !circularDependency
256+
&& depth < this.validationContext.getConfig().getPreloadJsonSchemaRefMaxNestingDepth()) {
246257
jsonSchema.initializeValidators();
247258
}
248259
}

0 commit comments

Comments
 (0)