Skip to content

Commit

Permalink
Merge branch 'master' into 1927-design-implement-contract-deletion-op…
Browse files Browse the repository at this point in the history 
…_contract_delete
  • Loading branch information
@corrideat
corrideat committed Feb 20, 2025
2 parents e82aa78 + 0fdb7ad commit 2204cbd
Show file tree
Hide file tree
Showing 35 changed files with 56,153 additions and 290 deletions.
22 changes: 11 additions & 11 deletions backend/database.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ export default ((sbp('sbp/selectors/register', {
}
// Number of entries pushed.
let counter = 0
let currentHash = await sbp('chelonia/db/get', `_private_hidx=${contractID}#${height}`)
let currentHash = await sbp('chelonia.db/get', `_private_hidx=${contractID}#${height}`)
let prefix = '['
let ended = false
// NOTE: if this ever stops working you can also try Readable.from():
Expand All @@ -69,7 +69,7 @@ export default ((sbp('sbp/selectors/register', {
const currentPrefix = prefix
prefix = ','
counter++
currentHash = await sbp('chelonia/db/get', `_private_hidx=${contractID}#${entry.height() + 1}`)
currentHash = await sbp('chelonia.db/get', `_private_hidx=${contractID}#${entry.height() + 1}`)
this.push(`${currentPrefix}"${strToB64(entry.serialize())}"`)
} else {
this.push(counter > 0 ? ']' : '[]')
Expand Down Expand Up @@ -109,11 +109,11 @@ export default ((sbp('sbp/selectors/register', {
}
// otherwise it is a Boom.notFound(), proceed ahead
}
await sbp('chelonia/db/set', namespaceKey(name), value)
await sbp('chelonia.db/set', namespaceKey(name), value)
return { name, value }
},
'backend/db/lookupName': async function (name: string): Promise<string | Error> {
const value = await sbp('chelonia/db/get', namespaceKey(name))
const value = await sbp('chelonia.db/get', namespaceKey(name))
return value || Boom.notFound()
}
}): any): string[])
Expand All @@ -125,7 +125,7 @@ function namespaceKey (name: string): string {
export const initDB = async () => {
// If persistence must be enabled:
// - load and initialize the selected storage backend
// - then overwrite 'chelonia/db/get' and '-set' to use it with an LRU cache
// - then overwrite 'chelonia.db/get' and '-set' to use it with an LRU cache
if (persistence) {
const { initStorage, readData, writeData, deleteData } = await import(`./database-${persistence}.js`)

Expand All @@ -137,7 +137,7 @@ export const initDB = async () => {
})

sbp('sbp/selectors/overwrite', {
'chelonia/db/get': async function (prefixableKey: string): Promise<Buffer | string | void> {
'chelonia.db/get': async function (prefixableKey: string): Promise<Buffer | string | void> {
const lookupValue = cache.get(prefixableKey)
if (lookupValue !== undefined) {
return lookupValue
Expand All @@ -151,7 +151,7 @@ export const initDB = async () => {
cache.set(prefixableKey, value)
return value
},
'chelonia/db/set': async function (key: string, value: Buffer | string): Promise<void> {
'chelonia.db/set': async function (key: string, value: Buffer | string): Promise<void> {
checkKey(key)
if (key.startsWith('_private_immutable')) {
const existingValue = await readData(key)
Expand All @@ -162,7 +162,7 @@ export const initDB = async () => {
await writeData(key, value)
cache.set(key, value)
},
'chelonia/db/delete': async function (key: string): Promise<void> {
'chelonia.db/delete': async function (key: string): Promise<void> {
checkKey(key)
if (key.startsWith('_private_immutable')) {
throw new Error('Cannot delete immutable key')
Expand All @@ -171,7 +171,7 @@ export const initDB = async () => {
cache.delete(key)
}
})
sbp('sbp/selectors/lock', ['chelonia/db/get', 'chelonia/db/set', 'chelonia/db/delete'])
sbp('sbp/selectors/lock', ['chelonia.db/get', 'chelonia.db/set', 'chelonia.db/delete'])
}
// TODO: Update this to only run when persistence is disabled when `chel deploy` can target SQLite.
if (persistence !== 'fs' || options.fs.dirname !== dbRootPath) {
Expand All @@ -196,11 +196,11 @@ export const initDB = async () => {
console.info('[chelonia.db] Preloading...')
for (const key of keys) {
// Skip keys which are already in the DB.
if (!persistence || !await sbp('chelonia/db/get', key)) {
if (!persistence || !await sbp('chelonia.db/get', key)) {
const value = await readFile(path.join(dataFolder, key), 'utf8')
// Load only contract source files and contract manifests.
if (value.startsWith(CONTRACT_MANIFEST_MAGIC) || value.startsWith(CONTRACT_SOURCE_MAGIC)) {
await sbp('chelonia/db/set', key, value)
await sbp('chelonia.db/set', key, value)
numNewKeys++
}
}
Expand Down
14 changes: 7 additions & 7 deletions backend/push.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,27 +10,27 @@ const { PUSH_SERVER_ACTION_TYPE, REQUEST_TYPE, createMessage } = require('../sha

const addSubscriptionToIndex = async (subcriptionId: string) => {
await sbp('okTurtles.eventQueue/queueEvent', 'update-webpush-indices', async () => {
const currentIndex = await sbp('chelonia/db/get', '_private_webpush_index')
const currentIndex = await sbp('chelonia.db/get', '_private_webpush_index')
// Add the current subscriptionId to the subscription index. Entries in the
// index are separated by \x00 (NUL). The index itself is used to know
// which entries to load.
const updatedIndex = `${currentIndex ? `${currentIndex}\x00` : ''}${subcriptionId}`
await sbp('chelonia/db/set', '_private_webpush_index', updatedIndex)
await sbp('chelonia.db/set', '_private_webpush_index', updatedIndex)
})
}

const deleteSubscriptionFromIndex = async (subcriptionId: string) => {
await sbp('okTurtles.eventQueue/queueEvent', 'update-webpush-indices', async () => {
const currentIndex = await sbp('chelonia/db/get', '_private_webpush_index')
const currentIndex = await sbp('chelonia.db/get', '_private_webpush_index')
const index = currentIndex.indexOf(subcriptionId)
if (index === -1) return
const updatedIndex = currentIndex.slice(0, index > 1 ? index - 1 : 0) + currentIndex.slice(index + subcriptionId.length)
await sbp('chelonia/db/set', '_private_webpush_index', updatedIndex)
await sbp('chelonia.db/set', '_private_webpush_index', updatedIndex)
})
}

const saveSubscription = (server, subscriptionId) => {
return sbp('chelonia/db/set', `_private_webpush_${subscriptionId}`, JSON.stringify({
return sbp('chelonia.db/set', `_private_webpush_${subscriptionId}`, JSON.stringify({
subscription: server.pushSubscriptions[subscriptionId],
channelIDs: [...server.pushSubscriptions[subscriptionId].subscriptions]
})).catch(e => {
Expand Down Expand Up @@ -173,7 +173,7 @@ const removeSubscription = async (server, subscriptionId) => {
})
}
await deleteSubscriptionFromIndex(subscriptionId)
await sbp('chelonia/db/delete', `_private_webpush_${subscriptionId}`)
await sbp('chelonia.db/delete', `_private_webpush_${subscriptionId}`)
} catch (e) {
console.error(e, 'Error removing subscription', subscriptionId)
}
Expand Down Expand Up @@ -277,7 +277,7 @@ export const pushServerActionhandlers: any = {
// store it in memory.
server.pushSubscriptions[subscriptionId] = subscriptionInfoWrapper(subscriptionId, subscription)
addSubscriptionToIndex(subscriptionId).then(() => {
return sbp('chelonia/db/set', `_private_webpush_${subscriptionId}`, JSON.stringify({ subscription: subscription, channelIDs: [] }))
return sbp('chelonia.db/set', `_private_webpush_${subscriptionId}`, JSON.stringify({ subscription: subscription, channelIDs: [] }))
.catch(async e => {
console.error(e, 'removing subscription from index because of error saving subscription', subscriptionId)
await deleteSubscriptionFromIndex(subscriptionId)
Expand Down
59 changes: 47 additions & 12 deletions backend/routes.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ route.POST('/event', {
const credentials = request.auth.credentials
// Only allow identity contracts to be created without attribution
if (!credentials?.billableContractID && deserializedHEAD.isFirstMessage) {
const manifest = await sbp('chelonia/db/get', deserializedHEAD.head.manifest)
const manifest = await sbp('chelonia.db/get', deserializedHEAD.head.manifest)
const parsedManifest = JSON.parse(manifest)
const { name } = JSON.parse(parsedManifest.body)
if (name !== 'gi.contracts/identity') return Boom.unauthorized('This contract type requires ownership information', 'shelter')
Expand Down Expand Up @@ -353,7 +353,7 @@ if (process.env.NODE_ENV === 'development') {
console.error(`hash(${hash}) != ourHash(${ourHash})`)
return Boom.badRequest('bad hash!')
}
await sbp('chelonia/db/set', hash, data)
await sbp('chelonia.db/set', hash, data)
return '/file/' + hash
} catch (err) {
return logger(err)
Expand Down Expand Up @@ -443,19 +443,19 @@ route.POST('/file', {
// different file F2) and then request to delete their file F1, which would
// result in corrupting F2.
// Ensure that the manifest doesn't exist
if (await sbp('chelonia/db/get', manifestHash)) {
if (await sbp('chelonia.db/get', manifestHash)) {
throw new Error(`Manifest ${manifestHash} already exists`)
}
// Ensure that the chunks do not exist
await Promise.all(chunks.map(async ([cid]) => {
const exists = !!(await sbp('chelonia/db/get', cid))
const exists = !!(await sbp('chelonia.db/get', cid))
if (exists) {
throw new Error(`Chunk ${cid} already exists`)
}
}))
// Now, store all chunks and the manifest
await Promise.all(chunks.map(([cid, data]) => sbp('chelonia/db/set', cid, data)))
await sbp('chelonia/db/set', manifestHash, manifestMeta.payload)
await Promise.all(chunks.map(([cid, data]) => sbp('chelonia.db/set', cid, data)))
await sbp('chelonia.db/set', manifestHash, manifestMeta.payload)
// Store attribution information
await sbp('backend/server/saveOwner', credentials.billableContractID, manifestHash)
// Store size information
Expand Down Expand Up @@ -499,7 +499,7 @@ route.POST('/deleteFile/{hash}', {
const { hash } = request.params
const strategy = request.auth.strategy
if (!hash || hash.startsWith('_private')) return Boom.notFound()
const owner = await sbp('chelonia/db/get', `_private_owner_${hash}`)
const owner = await sbp('chelonia.db/get', `_private_owner_${hash}`)
if (!owner) {
return Boom.notFound()
}
Expand All @@ -510,7 +510,7 @@ route.POST('/deleteFile/{hash}', {
let count = 0
// Walk up the ownership tree
do {
const owner = await sbp('chelonia/db/get', `_private_owner_${ultimateOwner}`)
const owner = await sbp('chelonia.db/get', `_private_owner_${ultimateOwner}`)
if (owner) {
ultimateOwner = owner
count++
Expand All @@ -527,7 +527,7 @@ route.POST('/deleteFile/{hash}', {
break
}
case 'chel-bearer': {
const expectedToken = await sbp('chelonia/db/get', `_private_deletionToken_${hash}`)
const expectedToken = await sbp('chelonia.db/get', `_private_deletionToken_${hash}`)
if (!expectedToken) {
return Boom.notFound()
}
Expand All @@ -545,6 +545,7 @@ route.POST('/deleteFile/{hash}', {

// Authentication passed, now proceed to delete the file and its associated
// keys
<<<<<<< HEAD
try {
await sbp('backend/deleteFile', hash)
return h.response()
Expand All @@ -562,6 +563,40 @@ route.POST('/deleteFile/{hash}', {
}
}
})
=======
const rawManifest = await sbp('chelonia.db/get', hash)
if (!rawManifest) return Boom.notFound()
try {
const manifest = JSON.parse(rawManifest)
if (!manifest || typeof manifest !== 'object') return Boom.badData('manifest format is invalid')
if (manifest.version !== '1.0.0') return Boom.badData('unsupported manifest version')
if (!Array.isArray(manifest.chunks) || !manifest.chunks.length) return Boom.badData('missing chunks')
// Delete all chunks
await Promise.all(manifest.chunks.map(([, cid]) => sbp('chelonia.db/delete', cid)))
} catch (e) {
console.warn(e, `Error parsing manifest for ${hash}. It's probably not a file manifest.`)
return Boom.notFound()
}
// The keys to be deleted are not read from or updated, so they can be deleted
// without using a queue
await sbp('chelonia.db/delete', hash)
await sbp('chelonia.db/delete', `_private_owner_${hash}`)
await sbp('chelonia.db/delete', `_private_size_${hash}`)
await sbp('chelonia.db/delete', `_private_deletionToken_${hash}`)
const resourcesKey = `_private_resources_${owner}`
// Use a queue for atomicity
await sbp('okTurtles.eventQueue/queueEvent', resourcesKey, async () => {
const existingResources = await sbp('chelonia.db/get', resourcesKey)
if (!existingResources) return
if (existingResources.endsWith(hash)) {
await sbp('chelonia.db/set', resourcesKey, existingResources.slice(0, -hash.length - 1))
return
}
const hashIndex = existingResources.indexOf(hash + '\x00')
if (hashIndex === -1) return
await sbp('chelonia.db/set', resourcesKey, existingResources.slice(0, hashIndex) + existingResources.slice(hashIndex + hash.length + 1))
})
>>>>>>> master

route.POST('/deleteContract/{hash}', {
auth: {
Expand Down Expand Up @@ -660,7 +695,7 @@ route.POST('/kv/{contractID}/{key}', {
return Boom.unauthorized(null, 'shelter')
}

const existing = await sbp('chelonia/db/get', `_private_kv_${contractID}_${key}`)
const existing = await sbp('chelonia.db/get', `_private_kv_${contractID}_${key}`)

// Some protection against accidental overwriting by implementing the if-match
// header
Expand Down Expand Up @@ -717,7 +752,7 @@ route.POST('/kv/{contractID}/{key}', {
}

const existingSize = existing ? Buffer.from(existing).byteLength : 0
await sbp('chelonia/db/set', `_private_kv_${contractID}_${key}`, request.payload)
await sbp('chelonia.db/set', `_private_kv_${contractID}_${key}`, request.payload)
await sbp('backend/server/updateSize', contractID, request.payload.byteLength - existingSize)
await appendToIndexFactory(`_private_kvIdx_${contractID}`)(key)
await sbp('backend/server/broadcastKV', contractID, key, request.payload.toString())
Expand All @@ -742,7 +777,7 @@ route.GET('/kv/{contractID}/{key}', {
return Boom.unauthorized(null, 'shelter')
}

const result = await sbp('chelonia/db/get', `_private_kv_${contractID}_${key}`)
const result = await sbp('chelonia.db/get', `_private_kv_${contractID}_${key}`)
if (!result) {
return notFoundNoCache(h)
}
Expand Down
16 changes: 8 additions & 8 deletions backend/server.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ sbp('sbp/selectors/register', {
// the foreign key is rotated or deleted. For this to work reliably, we'd
// need to ensure that the state for both contract B and contract A are
// saved when the foreign key gets added to contract B.
await sbp('chelonia/db/set', '_private_cheloniaState_' + contractID, JSON.stringify(state))
await sbp('chelonia.db/set', '_private_cheloniaState_' + contractID, JSON.stringify(state))
}
// If this is a new contract, we also need to add it to the index, which
// is used when starting up the server to know which keys to fetch.
Expand Down Expand Up @@ -197,11 +197,11 @@ sbp('sbp/selectors/register', {
// Use a queue to ensure atomic updates
await sbp('okTurtles.eventQueue/queueEvent', sizeKey, async () => {
// Size is stored as a decimal value
const existingSize = parseInt(await sbp('chelonia/db/get', sizeKey, 10) ?? '0')
const existingSize = parseInt(await sbp('chelonia.db/get', sizeKey, 10) ?? '0')
if (!(existingSize >= 0)) {
throw new TypeError(`Invalid stored size ${existingSize} for ${resourceID}`)
}
await sbp('chelonia/db/set', sizeKey, (existingSize + size).toString(10))
await sbp('chelonia.db/set', sizeKey, (existingSize + size).toString(10))
})
},
'backend/server/saveDeletionToken': async function (resourceID: string) {
Expand All @@ -210,7 +210,7 @@ sbp('sbp/selectors/register', {
crypto.getRandomValues(deletionTokenRaw)
// $FlowFixMe[incompatible-call]
const deletionToken = Buffer.from(deletionTokenRaw).toString('base64url')
await sbp('chelonia/db/set', `_private_deletionToken_${resourceID}`, deletionToken)
await sbp('chelonia.db/set', `_private_deletionToken_${resourceID}`, deletionToken)
return deletionToken
},
'backend/server/stop': function () {
Expand Down Expand Up @@ -434,15 +434,15 @@ sbp('okTurtles.data/set', PUBSUB_INSTANCE, createServer(hapi.listener, {
})
// Load the saved Chelonia state
// First, get the contract index
const savedStateIndex = await sbp('chelonia/db/get', '_private_cheloniaState_index')
const savedStateIndex = await sbp('chelonia.db/get', '_private_cheloniaState_index')
if (savedStateIndex) {
// Now, we contract the contract state by reading each contract state
// partition
const recoveredState = Object.create(null)
recoveredState.contracts = Object.create(null)
const channels = sbp('okTurtles.data/get', PUBSUB_INSTANCE).channels
await Promise.all(savedStateIndex.split('\x00').map(async (contractID) => {
const cpSerialized = await sbp('chelonia/db/get', `_private_cheloniaState_${contractID}`)
const cpSerialized = await sbp('chelonia.db/get', `_private_cheloniaState_${contractID}`)
if (!cpSerialized) {
console.warn(`[server] missing state for contractID ${contractID} - skipping setup for this contract`)
return
Expand All @@ -456,11 +456,11 @@ sbp('okTurtles.data/set', PUBSUB_INSTANCE, createServer(hapi.listener, {
Object.assign(sbp('chelonia/rootState'), recoveredState)
}
// Then, load push subscriptions
const savedWebPushIndex = await sbp('chelonia/db/get', '_private_webpush_index')
const savedWebPushIndex = await sbp('chelonia.db/get', '_private_webpush_index')
if (savedWebPushIndex) {
const { pushSubscriptions, subscribersByChannelID } = sbp('okTurtles.data/get', PUBSUB_INSTANCE)
await Promise.all(savedWebPushIndex.split('\x00').map(async (subscriptionId) => {
const subscriptionSerialized = await sbp('chelonia/db/get', `_private_webpush_${subscriptionId}`)
const subscriptionSerialized = await sbp('chelonia.db/get', `_private_webpush_${subscriptionId}`)
if (!subscriptionSerialized) {
console.warn(`[server] missing state for subscriptionId ${subscriptionId} - skipping setup for this subscription`)
return
Expand Down
4 changes: 2 additions & 2 deletions backend/vapid.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ if (!process.env.VAPID_EMAIL) {
const vapid = { VAPID_EMAIL: process.env.VAPID_EMAIL || 'mailto:test@example.com' }

export const initVapid = async () => {
const vapidKeyPair = await sbp('chelonia/db/get', '_private_immutable_vapid_key').then(async (vapidKeyPair: string): Promise<[Object, string]> => {
const vapidKeyPair = await sbp('chelonia.db/get', '_private_immutable_vapid_key').then(async (vapidKeyPair: string): Promise<[Object, string]> => {
if (!vapidKeyPair) {
console.info('Generating new VAPID keypair...')
// Generate a new ECDSA key pair
Expand All @@ -35,7 +35,7 @@ export const initVapid = async () => {
)
])

return sbp('chelonia/db/set', '_private_immutable_vapid_key', JSON.stringify(serializedKeyPair)).then(() => {
return sbp('chelonia.db/set', '_private_immutable_vapid_key', JSON.stringify(serializedKeyPair)).then(() => {
console.info('Successfully saved newly generated VAPID keys')
return [keyPair.privateKey, serializedKeyPair[1]]
})
Expand Down
Loading

0 comments on commit 2204cbd

Please sign in to comment.