Skip to content

Commit 5614efb

Browse files
mpcenmpcen
authored
Merge pull request #976 from qtomlinson/qt/trimmed_mongo
Add TrimmedMongoDefinitionStore
2 parents ccec159 + 9e7a58c commit 5614efb

12 files changed

+839
-398
lines changed

‎providers/index.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ module.exports = {
88
definition: {
99
azure: require('../providers/stores/azblobConfig').definition,
1010
file: require('../providers/stores/fileConfig').definition,
11-
mongo: require('../providers/stores/mongoConfig'),
11+
mongo: require('../providers/stores/mongoConfig').definitionPaged,
12+
mongoTrimmed: require('../providers/stores/mongoConfig').definitionTrimmed,
1213
dispatch: require('../providers/stores/dispatchConfig')
1314
},
1415
attachment: {

‎providers/stores/abstractMongoDefinitionStore.js

+221
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
// (c) Copyright 2023, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
2+
// SPDX-License-Identifier: MIT
3+
4+
const MongoClient = require('mongodb').MongoClient
5+
const promiseRetry = require('promise-retry')
6+
const EntityCoordinates = require('../../lib/entityCoordinates')
7+
const logger = require('../logging/logger')
8+
const { get } = require('lodash')
9+
const base64 = require('base-64')
10+
11+
const sortOptions = {
12+
type: ['coordinates.type'],
13+
provider: ['coordinates.provider'],
14+
name: ['coordinates.name', 'coordinates.revision'],
15+
namespace: ['coordinates.namespace', 'coordinates.name', 'coordinates.revision'],
16+
revision: ['coordinates.revision'],
17+
license: ['licensed.declared'],
18+
releaseDate: ['described.releaseDate'],
19+
licensedScore: ['licensed.score.total'],
20+
describedScore: ['described.score.total'],
21+
effectiveScore: ['scores.effective'],
22+
toolScore: ['scores.tool']
23+
}
24+
25+
const valueTransformers = {
26+
'licensed.score.total': value => value && parseInt(value),
27+
'described.score.total': value => value && parseInt(value),
28+
'scores.effective': value => value && parseInt(value),
29+
'scores.tool': value => value && parseInt(value)
30+
}
31+
32+
const SEPARATOR = '&'
33+
34+
class AbstractMongoDefinitionStore {
35+
constructor(options) {
36+
this.logger = options.logger || logger()
37+
this.options = options
38+
}
39+
40+
initialize() {
41+
return promiseRetry(async retry => {
42+
try {
43+
this.client = await MongoClient.connect(this.options.connectionString, { useNewUrlParser: true })
44+
this.db = this.client.db(this.options.dbName)
45+
this.collection = this.db.collection(this.options.collectionName)
46+
} catch (error) {
47+
retry(error)
48+
}
49+
})
50+
}
51+
52+
async close() {
53+
await this.client.close()
54+
}
55+
56+
/**
57+
* List all of the matching components for the given coordinates.
58+
* Accepts partial coordinates.
59+
*
60+
* @param {EntityCoordinates} coordinates
61+
* @returns A list of matching coordinates i.e. [ 'npm/npmjs/-/JSONStream/1.3.3' ]
62+
*/
63+
// eslint-disable-next-line no-unused-vars
64+
async list(coordinates) {
65+
throw new Error('Unsupported Operation')
66+
}
67+
68+
/**
69+
* Get and return the object at the given coordinates.
70+
*
71+
* @param {Coordinates} coordinates - The coordinates of the object to get
72+
* @returns The loaded object
73+
*/
74+
// eslint-disable-next-line no-unused-vars
75+
async get(coordinates) {
76+
throw new Error('Unsupported Operation')
77+
}
78+
79+
/**
80+
* Query and return the objects based on the query
81+
*
82+
* @param {object} query - The filters and sorts for the request
83+
* @returns The data and continuationToken if there is more results
84+
*/
85+
async find(query, continuationToken = '', pageSize = 100, projection) {
86+
const sort = this._buildSort(query)
87+
const combinedFilters = this._buildQueryWithPaging(query, continuationToken, sort)
88+
this.logger.debug(`filter: ${JSON.stringify(combinedFilters)}\nsort: ${JSON.stringify(sort)}`)
89+
const cursor = await this.collection.find(combinedFilters, {
90+
projection,
91+
sort,
92+
limit: pageSize
93+
})
94+
const data = await cursor.toArray()
95+
continuationToken = this._getContinuationToken(pageSize, data, sort)
96+
return { data, continuationToken }
97+
}
98+
99+
// eslint-disable-next-line no-unused-vars
100+
async store(definition) {
101+
throw new Error('Unsupported Operation')
102+
}
103+
104+
// eslint-disable-next-line no-unused-vars
105+
async delete(coordinates) {
106+
throw new Error('Unsupported Operation')
107+
}
108+
109+
getId(coordinates) {
110+
if (!coordinates) return ''
111+
return EntityCoordinates.fromObject(coordinates)
112+
.toString()
113+
.toLowerCase()
114+
}
115+
116+
buildQuery(parameters) {
117+
const filter = { }
118+
if (parameters.type) filter['coordinates.type'] = parameters.type
119+
if (parameters.provider) filter['coordinates.provider'] = parameters.provider
120+
if (parameters.namespace) filter['coordinates.namespace'] = parameters.namespace
121+
if (parameters.name) filter['coordinates.name'] = parameters.name
122+
if (parameters.type === null) filter['coordinates.type'] = null
123+
if (parameters.provider === null) filter['coordinates.provider'] = null
124+
if (parameters.name === null) filter['coordinates.name'] = null
125+
if (parameters.namespace === null) filter['coordinates.namespace'] = null
126+
if (parameters.license) filter['licensed.declared'] = parameters.license
127+
if (parameters.releasedAfter) filter['described.releaseDate'] = { $gt: parameters.releasedAfter }
128+
if (parameters.releasedBefore) filter['described.releaseDate'] = { $lt: parameters.releasedBefore }
129+
if (parameters.minEffectiveScore) filter['scores.effective'] = { $gt: parseInt(parameters.minEffectiveScore) }
130+
if (parameters.maxEffectiveScore) filter['scores.effective'] = { $lt: parseInt(parameters.maxEffectiveScore) }
131+
if (parameters.minToolScore) filter['scores.tool'] = { $gt: parseInt(parameters.minToolScore) }
132+
if (parameters.maxToolScore) filter['scores.tool'] = { $lt: parseInt(parameters.maxToolScore) }
133+
if (parameters.minLicensedScore) filter['licensed.score.total'] = { $gt: parseInt(parameters.minLicensedScore) }
134+
if (parameters.maxLicensedScore) filter['licensed.score.total'] = { $lt: parseInt(parameters.maxLicensedScore) }
135+
if (parameters.minDescribedScore) filter['described.score.total'] = { $gt: parseInt(parameters.minDescribedScore) }
136+
if (parameters.maxDescribedScore) filter['described.score.total'] = { $lt: parseInt(parameters.maxDescribedScore) }
137+
return filter
138+
}
139+
140+
_buildSort(parameters) {
141+
const sort = sortOptions[parameters.sort] || []
142+
const clause = {}
143+
sort.forEach(item => clause[item] = parameters.sortDesc ? -1 : 1)
144+
//Always sort ascending on partitionKey for continuation token
145+
const coordinateKey = this.getCoordinatesKey()
146+
clause[coordinateKey] = 1
147+
return clause
148+
}
149+
150+
_buildQueryWithPaging(query, continuationToken, sort) {
151+
const filter = this.buildQuery(query)
152+
const paginationFilter = this._buildPaginationQuery(continuationToken, sort)
153+
return paginationFilter ? { $and: [filter, paginationFilter] } : filter
154+
}
155+
156+
_buildPaginationQuery(continuationToken, sort) {
157+
if (!continuationToken.length) return
158+
const queryExpressions = this._buildQueryExpressions(continuationToken, sort)
159+
return queryExpressions.length <= 1 ?
160+
queryExpressions [0] :
161+
{ $or: [ ...queryExpressions ] }
162+
}
163+
164+
_buildQueryExpressions(continuationToken, sort) {
165+
const lastValues = base64.decode(continuationToken)
166+
const sortValues = lastValues.split(SEPARATOR).map(value => value.length ? value : null)
167+
168+
const queryExpressions = []
169+
const sortConditions = Object.entries(sort)
170+
for (let nSorts = 1; nSorts <= sortConditions.length; nSorts++) {
171+
const subList = sortConditions.slice(0, nSorts)
172+
queryExpressions.push(this._buildQueryExpression(subList, sortValues))
173+
}
174+
return queryExpressions
175+
}
176+
177+
_buildQueryExpression(sortConditions, sortValues) {
178+
return sortConditions.reduce((filter, [sortField, sortDirection], index) => {
179+
const transform = valueTransformers[sortField]
180+
let sortValue = sortValues[index]
181+
sortValue = transform ? transform(sortValue) : sortValue
182+
const isLast = index === sortConditions.length - 1
183+
const filterForSort = this._buildQueryForSort(isLast, sortField, sortValue, sortDirection)
184+
return { ...filter, ...filterForSort}
185+
}, {})
186+
}
187+
188+
_buildQueryForSort(isTieBreaker, sortField, sortValue, sortDirection) {
189+
let operator = '$eq'
190+
if (isTieBreaker) {
191+
if (sortDirection === 1) {
192+
operator = sortValue === null ? '$ne' : '$gt'
193+
} else {
194+
operator = '$lt'
195+
}
196+
}
197+
const filter = { [sortField]: { [operator]: sortValue } }
198+
199+
//Less than non null value should include null as well
200+
if (operator === '$lt' && sortValue) {
201+
return {
202+
$or: [
203+
filter,
204+
{ [sortField]: null }
205+
]
206+
}
207+
}
208+
return filter
209+
}
210+
211+
_getContinuationToken(pageSize, data, sortClause) {
212+
if (data.length !== pageSize) return ''
213+
const lastItem = data[data.length - 1]
214+
const lastValues = Object.keys(sortClause)
215+
.map(key => get(lastItem, key))
216+
.join(SEPARATOR)
217+
return base64.encode(lastValues)
218+
}
219+
220+
}
221+
module.exports = AbstractMongoDefinitionStore

‎providers/stores/dispatchDefinitionStore.js

+18-8
Original file line numberDiff line numberDiff line change
@@ -6,34 +6,34 @@ const logger = require('../logging/logger')
66
class DispatchDefinitionStore {
77
constructor(options) {
88
this.stores = options.stores
9-
this.logger = logger()
9+
this.logger = options.logger || logger()
1010
}
1111

1212
initialize() {
13-
return this._perform(store => store.initialize())
13+
return this._performInParallel(store => store.initialize())
1414
}
1515

1616
get(coordinates) {
17-
return this._perform(store => store.get(coordinates), true)
17+
return this._performInSequence(store => store.get(coordinates))
1818
}
1919

2020
list(coordinates) {
21-
return this._perform(store => store.list(coordinates), true)
21+
return this._performInSequence(store => store.list(coordinates))
2222
}
2323

2424
store(definition) {
25-
return this._perform(store => store.store(definition))
25+
return this._performInParallel(store => store.store(definition))
2626
}
2727

2828
delete(coordinates) {
29-
return this._perform(store => store.delete(coordinates))
29+
return this._performInParallel(store => store.delete(coordinates))
3030
}
3131

3232
find(query, continuationToken = '') {
33-
return this._perform(store => store.find(query, continuationToken), true)
33+
return this._performInSequence(store => store.find(query, continuationToken))
3434
}
3535

36-
async _perform(operation, first = false) {
36+
async _performInSequence(operation, first = true) {
3737
let result = null
3838
for (let i = 0; i < this.stores.length; i++) {
3939
const store = this.stores[i]
@@ -47,6 +47,16 @@ class DispatchDefinitionStore {
4747
}
4848
return result
4949
}
50+
51+
async _performInParallel(operation) {
52+
const opPromises = this.stores.map(store => operation(store))
53+
const results = await Promise.allSettled(opPromises)
54+
results
55+
.filter(result => result.status === 'rejected')
56+
.forEach(result => this.logger.error('DispatchDefinitionStore failure', result.reason))
57+
const fulfilled = results.find(result => result.status === 'fulfilled')
58+
return fulfilled?.value
59+
}
5060
}
5161

5262
module.exports = options => new DispatchDefinitionStore(options)

0 commit comments

Comments
 (0)