From b597fc1e403a5d81c945d510b639337dc583f966 Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Wed, 11 Dec 2024 17:44:43 +0100 Subject: [PATCH 1/6] Add support for Vyper cbor auxdata in bytecode-utils and consequently in VyperCheckedContract --- .vscode/launch.json | 20 + package-lock.json | 5 +- packages/bytecode-utils/README.md | 4 +- packages/bytecode-utils/package.json | 3 +- packages/bytecode-utils/src/lib/bytecode.ts | 354 ++++++++++++++---- packages/bytecode-utils/test/bytecode.spec.ts | 81 +++- .../test/bytecodes/vyper-cbor-no-array.hex | 1 + .../test/bytecodes/vyper-integrity.hex | 1 + .../bytecodes/vyper-no-auxdata-length.hex | 1 + .../test/bytecodes/vyper-no-integrity.hex | 1 + .../src/lib/AbstractCheckedContract.ts | 3 + .../src/lib/SolidityCheckedContract.ts | 11 +- .../src/lib/VyperCheckedContract.ts | 69 ++++ packages/lib-sourcify/src/lib/verification.ts | 33 +- services/monitor/src/ChainMonitor.ts | 7 +- services/monitor/src/util.ts | 4 +- .../session-state/session-state.handlers.ts | 10 +- 17 files changed, 507 insertions(+), 101 deletions(-) create mode 100644 packages/bytecode-utils/test/bytecodes/vyper-cbor-no-array.hex create mode 100644 packages/bytecode-utils/test/bytecodes/vyper-integrity.hex create mode 100644 packages/bytecode-utils/test/bytecodes/vyper-no-auxdata-length.hex create mode 100644 packages/bytecode-utils/test/bytecodes/vyper-no-integrity.hex diff --git a/.vscode/launch.json b/.vscode/launch.json index 2053b33b3..228b3fe36 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -130,6 +130,26 @@ "console": "integratedTerminal" // "internalConsoleOptions": "neverOpen" }, + { + "type": "node", + "request": "launch", + "name": "Mocha - bytecode-utils", + "cwd": "${workspaceFolder}/packages/bytecode-utils", + "program": "${workspaceRoot}/node_modules/.bin/mocha", + "args": [ + "-r", + "ts-node/register", + "./test/**/*.spec.ts", + "--no-timeout", + // Run a single test when debugging + // "--grep=v0.6.12", + "--exit" + ], + "sourceMaps": true, + "smartStep": true, + "console": "integratedTerminal" + // "internalConsoleOptions": "neverOpen" + }, { "type": "node", "request": "launch", diff --git a/package-lock.json b/package-lock.json index 13e636577..1390add89 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15811,6 +15811,8 @@ }, "node_modules/semver": { "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -18270,7 +18272,8 @@ "@ethersproject/bytes": "5.7.0", "base-x": "4.0.0", "bs58": "5.0.0", - "cbor-x": "1.6.0" + "cbor-x": "1.6.0", + "semver": "^7.6.3" }, "devDependencies": { "@types/chai": "4.3.20", diff --git a/packages/bytecode-utils/README.md b/packages/bytecode-utils/README.md index 72c61ea42..6a1d41dba 100644 --- a/packages/bytecode-utils/README.md +++ b/packages/bytecode-utils/README.md @@ -11,11 +11,11 @@ yarn add @ethereum-sourcify/bytecode-utils ## Usage ```ts -import { decode } from "@ethereum-sourcify/bytecode-utils"; +import { decode, AuxdataStyle } from "@ethereum-sourcify/bytecode-utils"; const bytecodeRaw = "0x60806040526004361061003f5760003560e01...7265206c656e677468a2646970667358221220dceca8706b29e917dacf25fceef95acac8d90d765ac926663ce4096195952b6164736f6c634300060b0033" -decode(bytecodeRaw); +decode(bytecodeRaw, AuxdataStyle.SOLIDITY); ``` ### Result diff --git a/packages/bytecode-utils/package.json b/packages/bytecode-utils/package.json index 193947343..e82227026 100644 --- a/packages/bytecode-utils/package.json +++ b/packages/bytecode-utils/package.json @@ -41,7 +41,8 @@ "@ethersproject/bytes": "5.7.0", "base-x": "4.0.0", "bs58": "5.0.0", - "cbor-x": "1.6.0" + "cbor-x": "1.6.0", + "semver": "^7.6.3" }, "devDependencies": { "@types/chai": "4.3.20", diff --git a/packages/bytecode-utils/src/lib/bytecode.ts b/packages/bytecode-utils/src/lib/bytecode.ts index d574aa1b6..11232e696 100644 --- a/packages/bytecode-utils/src/lib/bytecode.ts +++ b/packages/bytecode-utils/src/lib/bytecode.ts @@ -1,13 +1,14 @@ import { arrayify, hexlify } from '@ethersproject/bytes'; import bs58 from 'bs58'; import * as CBOR from 'cbor-x'; +import semver from 'semver'; type CBOR = { bytes: string; length: number; }; -export type DecodedObject = { +export type SolidityDecodedObject = { // Known CBOR fields that are defined in the spec ipfs?: string; solcVersion?: string; @@ -18,12 +19,34 @@ export type DecodedObject = { [key: string]: string | Uint8Array | undefined | boolean; }; +export type VyperDecodedObject = { + integrity?: string; + runtimeSize?: number; + dataSizes?: number[]; + immutableSize?: number; + compiler: { + vyper: [number, number, number]; + }; +}; + +export enum AuxdataStyle { + SOLIDITY = 'solidity', + VYPER = 'vyper', + VYPER_LT_0_3_10 = 'vyper_lt_0_3_10', + VYPER_LT_0_3_5 = 'vyper_lt_0_3_5', +} + /** * Decode contract's bytecode * @param bytecode - hex of the bytecode with 0x prefix * @returns Object describing the contract */ -export const decode = (bytecode: string): DecodedObject => { +export const decode = ( + bytecode: string, + auxdataStyle: T, +): T extends AuxdataStyle.SOLIDITY + ? SolidityDecodedObject + : VyperDecodedObject => { if (bytecode.length === 0) { throw Error('Bytecode cannot be null'); } @@ -32,92 +55,291 @@ export const decode = (bytecode: string): DecodedObject => { } // split auxdata - const [, auxdata] = splitAuxdata(bytecode); + const [, auxdata] = splitAuxdata(bytecode, auxdataStyle); + // See more here: https://github.com/vyperlang/vyper/pull/3010 + if (auxdataStyle === AuxdataStyle.VYPER) { + if (!auxdata) { + // Attempt fallback auxdata styles for older Vyper versions + const fallbackStyles = [ + AuxdataStyle.VYPER_LT_0_3_10, // Vyper bytecode before version 0.3.10 uses SOLIDITY AuxdataStyle + AuxdataStyle.VYPER_LT_0_3_5, // Vyper bytecode before version 0.3.5 doesn't include auxdata length + ]; - if (!auxdata) { - throw Error('Auxdata is not in the execution bytecode'); - } + for (const style of fallbackStyles) { + const [, fallbackAuxdata] = splitAuxdata(bytecode, style); + if (fallbackAuxdata) { + // cbor decode the object and get a json + const cborDecodedObject = CBOR.decode( + arrayify(`0x${fallbackAuxdata}`), + ); + return { + compiler: cborDecodedObject.vyper.join('.'), + } as any; + } + } + + throw Error('Auxdata is not in the execution bytecode'); + } - // cbor decode the object and get a json - const cborDecodedObject = CBOR.decode(arrayify(`0x${auxdata}`)); + // cbor decode the object and get a json + const cborDecodedObject = CBOR.decode(arrayify(`0x${auxdata}`)); - const result: DecodedObject = {}; + // Starting with version 0.3.10, Vyper stores the auxdata as an array + // See more here: https://github.com/vyperlang/vyper/pull/3584 + if (cborDecodedObject instanceof Array) { + // read the last element from array, it contains the compiler version + const compilerVersion = + cborDecodedObject[cborDecodedObject.length - 1].vyper.join('.'); - // Decode all the parameters from the json - Object.keys(cborDecodedObject).forEach((key: string) => { - switch (key) { - case 'ipfs': { - const ipfsCID = bs58.encode(cborDecodedObject.ipfs); - result.ipfs = ipfsCID; - break; + if (semver.gte(compilerVersion, '0.4.1')) { + // Starting with version 0.4.1 Vyper added the integrity field + // See more here: https://github.com/vyperlang/vyper/pull/4234 + return { + integrity: cborDecodedObject[0], + runtimeSize: cborDecodedObject[1], + dataSizes: cborDecodedObject[2], + immutableSize: cborDecodedObject[3], + compiler: compilerVersion, + } as any; + } else if (semver.gte(compilerVersion, '0.3.10')) { + return { + runtimeSize: cborDecodedObject[0], + dataSizes: cborDecodedObject[1], + immutableSize: cborDecodedObject[2], + compiler: compilerVersion, + } as any; } - case 'solc': { - // nightly builds are string encoded - if (typeof cborDecodedObject.solc === 'string') { - result.solcVersion = cborDecodedObject.solc; - } else { - result.solcVersion = cborDecodedObject.solc.join('.'); + } + throw Error('This version of Vyper is not supported'); + } else if (auxdataStyle === AuxdataStyle.SOLIDITY) { + // split auxdata + const [, auxdata] = splitAuxdata(bytecode, auxdataStyle); + + if (!auxdata) { + throw Error('Auxdata is not in the execution bytecode'); + } + + // cbor decode the object and get a json + const cborDecodedObject = CBOR.decode(arrayify(`0x${auxdata}`)); + + const result: SolidityDecodedObject = {}; + // Decode all the parameters from the json + Object.keys(cborDecodedObject).forEach((key: string) => { + switch (key) { + case 'ipfs': { + const ipfsCID = bs58.encode(cborDecodedObject.ipfs); + result.ipfs = ipfsCID; + break; + } + case 'solc': { + // nightly builds are string encoded + if (typeof cborDecodedObject.solc === 'string') { + result.solcVersion = cborDecodedObject.solc; + } else { + result.solcVersion = cborDecodedObject.solc.join('.'); + } + break; + } + case 'experimental': { + result.experimental = cborDecodedObject.experimental; + break; + } + case 'bzzr0': + case 'bzzr1': + default: { + result[key] = hexlify(cborDecodedObject[key]); + break; } - break; - } - case 'experimental': { - result.experimental = cborDecodedObject.experimental; - break; - } - case 'bzzr0': - case 'bzzr1': - default: { - result[key] = hexlify(cborDecodedObject[key]); - break; } - } - }); + }); - return result; + return result as any; + } else { + throw Error('Invalid auxdata style'); + } }; /** - * Splits bytecode into execution bytecode and auxdata - * If the bytecode has no CBOR encoded part, returns the whole bytecode - * @param bytecode - hex of the bytecode with 0x prefix - * @returns string[] - [ executionBytecode, auxdata?, cborBytesLength?] all as hexStrings + * Splits the bytecode into execution bytecode and auxdata. + * If the bytecode does not contain CBOR-encoded auxdata, returns the whole bytecode. + * + * @param bytecode - Hex string of the bytecode with 0x prefix + * @param auxdataStyle - The style of auxdata (Solidity or Vyper) + * @returns An array containing execution bytecode and optionally auxdata and its length */ -export const splitAuxdata = (bytecode: string): string[] => { +export const splitAuxdata = ( + bytecode: string, + auxdataStyle: AuxdataStyle, +): string[] => { + validateBytecode(bytecode); + bytecode = ensureHexPrefix(bytecode); + + const bytesLength = 4; + const cborBytesLength = getCborBytesLength( + bytecode, + auxdataStyle, + bytesLength, + ); + + if (isCborLengthInvalid(bytecode, cborBytesLength, bytesLength)) { + return [bytecode]; + } + + const auxdata = extractAuxdata( + bytecode, + auxdataStyle, + cborBytesLength, + bytesLength, + ); + const executionBytecode = extractExecutionBytecode( + bytecode, + cborBytesLength, + bytesLength, + ); + + if (isCborEncoded(auxdata)) { + const cborLengthHex = getCborLengthHex(bytecode, auxdataStyle, bytesLength); + return [executionBytecode, auxdata, cborLengthHex]; + } + + return [bytecode]; +}; + +/** + * Validates that the bytecode is not empty. + * + * @param bytecode - The bytecode string to validate + */ +const validateBytecode = (bytecode: string) => { if (bytecode.length === 0) { throw Error('Bytecode cannot be null'); } - if (bytecode.substring(0, 2) !== '0x') { - bytecode = '0x' + bytecode; - } +}; - const bytesLength = 4; +/** + * Ensures the bytecode string starts with '0x'. + * + * @param bytecode - The bytecode string + * @returns The bytecode string with '0x' prefix + */ +const ensureHexPrefix = (bytecode: string): string => { + return bytecode.startsWith('0x') ? bytecode : `0x${bytecode}`; +}; - // Take latest 2 bytes of the bytecode (length of the cbor object) - const cborLengthHex = `${bytecode.slice(-bytesLength)}`; - const cborLength = parseInt(cborLengthHex, 16); - const cborBytesLength = cborLength * 2; +/** + * Determines the length of the CBOR auxdata in bytes. + * + * @param bytecode - The complete bytecode string + * @param auxdataStyle - The style of auxdata + * @param bytesLength - The length of bytes used to encode the CBOR length + * @returns An object containing the CBOR bytes length and a flag for legacy Vyper + */ +const getCborBytesLength = ( + bytecode: string, + auxdataStyle: AuxdataStyle, + bytesLength: number, +): number => { + if (auxdataStyle === AuxdataStyle.VYPER_LT_0_3_5) { + return 22; + } + const cborLengthHex = bytecode.slice(-bytesLength); + return parseInt(cborLengthHex, 16) * 2; +}; - // If the length of the cbor is more or equal to the length of the execution bytecode, it means there is no cbor - if (bytecode.length - bytesLength - cborBytesLength <= 0) { - return [bytecode]; +/** + * Checks if the CBOR length is invalid based on the bytecode length. + * + * @param bytecode - The complete bytecode string + * @param cborBytesLength - The length of CBOR auxdata in bytes + * @param bytesLength - The length of bytes used to encode the CBOR length + * @returns True if the CBOR length is invalid, otherwise false + */ +const isCborLengthInvalid = ( + bytecode: string, + cborBytesLength: number, + bytesLength: number, +): boolean => { + return bytecode.length - bytesLength - cborBytesLength <= 0; +}; + +/** + * Extracts the auxdata from the bytecode based on the auxdata style. + * + * @param bytecode - The complete bytecode string + * @param auxdataStyle - The style of auxdata + * @param cborBytesLength - The length of CBOR auxdata in bytes + * @param bytesLength - The length of bytes used to encode the CBOR length + * @returns The extracted auxdata as a hex string + */ +const extractAuxdata = ( + bytecode: string, + auxdataStyle: AuxdataStyle, + cborBytesLength: number, + bytesLength: number, +): string => { + switch (auxdataStyle) { + case AuxdataStyle.VYPER_LT_0_3_10: + case AuxdataStyle.SOLIDITY: + return bytecode.substring( + bytecode.length - bytesLength - cborBytesLength, + bytecode.length - bytesLength, + ); + case AuxdataStyle.VYPER: + return bytecode.substring( + bytecode.length - cborBytesLength, + bytecode.length - bytesLength, + ); + case AuxdataStyle.VYPER_LT_0_3_5: + return bytecode.substring(bytecode.length - 22, bytecode.length); + default: + throw Error('Unsupported auxdata style'); } - // Extract the cbor object using the extracted length - const auxdata = bytecode.substring( - bytecode.length - bytesLength - cborBytesLength, - bytecode.length - bytesLength, - ); +}; - // Extract exection bytecode - const executionBytecode = bytecode.substring( - 0, - bytecode.length - bytesLength - cborBytesLength, - ); +/** + * Extracts the execution bytecode from the complete bytecode string. + * + * @param bytecode - The complete bytecode string + * @param cborBytesLength - The length of CBOR auxdata in bytes + * @param bytesLength - The length of bytes used to encode the CBOR length + * @returns The execution bytecode as a hex string + */ +const extractExecutionBytecode = ( + bytecode: string, + cborBytesLength: number, + bytesLength: number, +): string => { + return bytecode.substring(0, bytecode.length - bytesLength - cborBytesLength); +}; +/** + * Attempts to decode the auxdata to verify if it's CBOR-encoded. + * + * @param auxdata - The auxdata string to decode + * @returns True if auxdata is CBOR-encoded, otherwise false + */ +const isCborEncoded = (auxdata: string): boolean => { try { - // return the complete array only if the auxdata is actually cbor encoded CBOR.decode(arrayify(`0x${auxdata}`)); - return [executionBytecode, auxdata, cborLengthHex]; - } catch (e) { - return [bytecode]; + return true; + } catch { + return false; } }; + +/** + * Retrieves the CBOR length from the bytecode based on the auxdata style. + * + * @param bytecode - The complete bytecode string + * @param auxdataStyle - The style of auxdata + * @param bytesLength - The length of bytes used to encode the CBOR length + * @returns The CBOR length as a hex string + */ +const getCborLengthHex = ( + bytecode: string, + auxdataStyle: AuxdataStyle, + bytesLength: number, +): string => { + if (auxdataStyle === AuxdataStyle.VYPER_LT_0_3_5) return ''; + return bytecode.slice(-bytesLength); +}; diff --git a/packages/bytecode-utils/test/bytecode.spec.ts b/packages/bytecode-utils/test/bytecode.spec.ts index 84ae02751..a2062c763 100644 --- a/packages/bytecode-utils/test/bytecode.spec.ts +++ b/packages/bytecode-utils/test/bytecode.spec.ts @@ -2,7 +2,7 @@ import chai from 'chai'; import { readFileSync } from 'fs'; import path from 'path'; -import { decode, splitAuxdata } from '../src/lib/bytecode'; +import { AuxdataStyle, decode, splitAuxdata } from '../src/lib/bytecode'; type Error = { message: string; @@ -21,17 +21,35 @@ const BYTECODE_WITHOUT0X = readFileSync( const BYTECODE_WITHOUTAUXDATA = readFileSync( `${BYTECODES_FOLDER}/withoutauxdata.hex`, ).toString(); +const BYTECODE_VYPER_INTEGRITY = readFileSync( + `${BYTECODES_FOLDER}/vyper-integrity.hex`, +).toString(); +const BYTECODE_VYPER_NO_INTEGRITY = readFileSync( + `${BYTECODES_FOLDER}/vyper-no-integrity.hex`, +).toString(); +const BYTECODE_VYPER_NO_ARRAY = readFileSync( + `${BYTECODES_FOLDER}/vyper-cbor-no-array.hex`, +).toString(); +const BYTECODE_VYPER_NO_AUXDATA_LENGTH = readFileSync( + `${BYTECODES_FOLDER}/vyper-no-auxdata-length.hex`, +).toString(); describe('bytecode utils', function () { it("return the whole bytecode when the bytecode that doesn't contain auxdata", () => { - const [execution, auxadata, length] = splitAuxdata(BYTECODE_WITHOUTAUXDATA); + const [execution, auxadata, length] = splitAuxdata( + BYTECODE_WITHOUTAUXDATA, + AuxdataStyle.SOLIDITY, + ); chai.expect(auxadata).to.be.undefined; chai.expect(length).to.be.undefined; chai.expect(`${execution}`).to.equal(BYTECODE_WITHOUTAUXDATA); }); it('split succesfully bytecode into execution bytecode and auxadata', () => { - const [execution, auxadata, length] = splitAuxdata(BYTECODE_IPFS); + const [execution, auxadata, length] = splitAuxdata( + BYTECODE_IPFS, + AuxdataStyle.SOLIDITY, + ); chai .expect(auxadata) .to.equal( @@ -42,25 +60,70 @@ describe('bytecode utils', function () { it('bytecode decode cbor with `ipfs` property', () => { chai - .expect(decode(BYTECODE_IPFS).ipfs) + .expect(decode(BYTECODE_IPFS, AuxdataStyle.SOLIDITY).ipfs) .to.equal('QmdD3hpMj6mEFVy9DP4QqjHaoeYbhKsYvApX1YZNfjTVWp'); }); it('bytecode decode cbor with `bzzr1` property', () => { chai - .expect(decode(BYTECODE_BZZR1).bzzr1) + .expect(decode(BYTECODE_BZZR1, AuxdataStyle.SOLIDITY).bzzr1) .to.equal( '0x71e0c183217ae3e9a1406ae7b58c2f36e09f2b16b10e19d46ceb821f3ee6abad', ); }); it('bytecode decode cbor with `experimental` property', () => { - chai.expect(decode(BYTECODE_EXPERIMENTAL).experimental).to.be.true; + chai.expect( + decode(BYTECODE_EXPERIMENTAL, AuxdataStyle.SOLIDITY).experimental, + ).to.be.true; + }); + + it('bytecode decode Vyper cbor auxdata for version >= 0.4.1', () => { + chai + .expect(decode(BYTECODE_VYPER_INTEGRITY, AuxdataStyle.VYPER)) + .to.deep.equal({ + integrity: new Uint8Array([ + 5, 183, 84, 197, 139, 46, 84, 10, 20, 171, 166, 241, 103, 23, 171, 44, + 48, 237, 199, 73, 54, 200, 152, 93, 119, 177, 82, 205, 151, 136, 126, + 7, + ]), + runtimeSize: 143, + dataSizes: [], + immutableSize: 0, + compiler: '0.4.1', + }); + }); + + it('bytecode decode Vyper cbor auxdata for version >= 0.3.10 and < 0.4.1', () => { + chai + .expect(decode(BYTECODE_VYPER_NO_INTEGRITY, AuxdataStyle.VYPER)) + .to.deep.equal({ + runtimeSize: 143, + dataSizes: [], + immutableSize: 0, + compiler: '0.3.10', + }); + }); + + it('bytecode decode Vyper cbor auxdata for version < 0.3.10', () => { + chai + .expect(decode(BYTECODE_VYPER_NO_ARRAY, AuxdataStyle.VYPER)) + .to.deep.equal({ + compiler: '0.3.8', + }); + }); + + it('bytecode decode Vyper cbor auxdata for version < 0.3.5', () => { + chai + .expect(decode(BYTECODE_VYPER_NO_AUXDATA_LENGTH, AuxdataStyle.VYPER)) + .to.deep.equal({ + compiler: '0.3.4', + }); }); it('bytecode decode should fail gracefully when input is undefined', () => { try { - decode(''); + decode('', AuxdataStyle.SOLIDITY); } catch (e) { chai.expect((e as Error).message).to.equal('Bytecode cannot be null'); } @@ -68,13 +131,13 @@ describe('bytecode utils', function () { it('decode a bytecode not starting with 0x', () => { chai - .expect(decode(BYTECODE_WITHOUT0X).ipfs) + .expect(decode(BYTECODE_WITHOUT0X, AuxdataStyle.SOLIDITY).ipfs) .to.equal('QmbFc3AoHDC977j2UH2WwYSwsSRrBGj8bsiiyigXhHzyuZ'); }); it('bytecode decode should fail gracefully when input is corrupted', () => { try { - decode(BYTECODE_WRONG); + decode(BYTECODE_WRONG, AuxdataStyle.SOLIDITY); } catch (e) { chai .expect((e as Error).message) diff --git a/packages/bytecode-utils/test/bytecodes/vyper-cbor-no-array.hex b/packages/bytecode-utils/test/bytecodes/vyper-cbor-no-array.hex new file mode 100644 index 000000000..4ae86f69c --- /dev/null +++ b/packages/bytecode-utils/test/bytecodes/vyper-cbor-no-array.hex @@ -0,0 +1 @@ +0x6100a361000f6000396100a36000f360003560e01c346100915763c605f76c811861008a57602080608052600c6040527f48656c6c6f20576f726c6421000000000000000000000000000000000000000060605260408160800181516020830160208301815181525050808252508051806020830101601f82600003163682375050601f19601f8251602001011690509050810190506080f35b5060006000fd5b600080fda165767970657283000308000b \ No newline at end of file diff --git a/packages/bytecode-utils/test/bytecodes/vyper-integrity.hex b/packages/bytecode-utils/test/bytecodes/vyper-integrity.hex new file mode 100644 index 000000000..a49c87cd2 --- /dev/null +++ b/packages/bytecode-utils/test/bytecodes/vyper-integrity.hex @@ -0,0 +1 @@ +0x61008f61000f60003961008f6000f360003560e01c63c605f76c8118610084573461008a57602080608052600c6040527f48656c6c6f20576f726c6421000000000000000000000000000000000000000060605260408160800181518152602082015160208201528051806020830101601f82600003163682375050601f19601f8251602001011690509050810190506080f35b60006000fd5b600080fd85582005b754c58b2e540a14aba6f16717ab2c30edc74936c8985d77b152cd97887e07188f8000a1657679706572830004010034 \ No newline at end of file diff --git a/packages/bytecode-utils/test/bytecodes/vyper-no-auxdata-length.hex b/packages/bytecode-utils/test/bytecodes/vyper-no-auxdata-length.hex new file mode 100644 index 000000000..22160ace9 --- /dev/null +++ b/packages/bytecode-utils/test/bytecodes/vyper-no-auxdata-length.hex @@ -0,0 +1 @@ +0x6100b761000f6000396100b76000f36003361161000c576100a1565b60003560e01c346100a75763c605f76c811861009f57600436186100a757602080608052600c6040527f48656c6c6f20576f726c6421000000000000000000000000000000000000000060605260408160800181518082526020830160208301815181525050508051806020830101601f82600003163682375050601f19601f8251602001011690509050810190506080f35b505b60006000fd5b600080fda165767970657283000304 \ No newline at end of file diff --git a/packages/bytecode-utils/test/bytecodes/vyper-no-integrity.hex b/packages/bytecode-utils/test/bytecodes/vyper-no-integrity.hex new file mode 100644 index 000000000..11c41a634 --- /dev/null +++ b/packages/bytecode-utils/test/bytecodes/vyper-no-integrity.hex @@ -0,0 +1 @@ +0x61008f61000f60003961008f6000f360003560e01c63c605f76c8118610084573461008a57602080608052600c6040527f48656c6c6f20576f726c6421000000000000000000000000000000000000000060605260408160800181518152602082015160208201528051806020830101601f82600003163682375050601f19601f8251602001011690509050810190506080f35b60006000fd5b600080fd84188f8000a16576797065728300030a0012 \ No newline at end of file diff --git a/packages/lib-sourcify/src/lib/AbstractCheckedContract.ts b/packages/lib-sourcify/src/lib/AbstractCheckedContract.ts index de9872168..42abc365b 100644 --- a/packages/lib-sourcify/src/lib/AbstractCheckedContract.ts +++ b/packages/lib-sourcify/src/lib/AbstractCheckedContract.ts @@ -30,4 +30,7 @@ export abstract class AbstractCheckedContract { * @param forceEmscripten Whether to force using emscripten for compilation */ abstract recompile(forceEmscripten?: boolean): Promise; + abstract generateCborAuxdataPositions( + forceEmscripten?: boolean, + ): Promise; } diff --git a/packages/lib-sourcify/src/lib/SolidityCheckedContract.ts b/packages/lib-sourcify/src/lib/SolidityCheckedContract.ts index 26c0b3307..16e482888 100644 --- a/packages/lib-sourcify/src/lib/SolidityCheckedContract.ts +++ b/packages/lib-sourcify/src/lib/SolidityCheckedContract.ts @@ -19,6 +19,7 @@ import semver from 'semver'; import { fetchWithBackoff } from './utils'; import { storeByHash } from './validation'; import { + AuxdataStyle, decode as decodeBytecode, splitAuxdata, } from '@ethereum-sourcify/bytecode-utils'; @@ -47,6 +48,8 @@ export class SolidityCheckedContract extends AbstractCheckedContract { solcJsonInput: any; compilerOutput?: SolidityOutput; + auxdataStyle: AuxdataStyle.SOLIDITY = AuxdataStyle.SOLIDITY; + /** Checks whether this contract is valid or not. * This is a static method due to persistence issues. * @@ -110,7 +113,7 @@ export class SolidityCheckedContract extends AbstractCheckedContract { ): Promise { let decodedAuxdata; try { - decodedAuxdata = decodeBytecode(runtimeBytecode); + decodedAuxdata = decodeBytecode(runtimeBytecode, this.auxdataStyle); } catch (err) { // There is no auxdata at all in this contract return null; @@ -289,6 +292,7 @@ export class SolidityCheckedContract extends AbstractCheckedContract { // Extract the auxdata from the end of the recompiled runtime bytecode const [, runtimeAuxdataCbor, runtimeCborLengthHex] = splitAuxdata( this.runtimeBytecode, + this.auxdataStyle, ); const auxdataFromRawRuntimeBytecode = `${runtimeAuxdataCbor}${runtimeCborLengthHex}`; @@ -310,7 +314,7 @@ export class SolidityCheckedContract extends AbstractCheckedContract { offset: this.runtimeBytecode.substring(2).length / 2 - parseInt(runtimeCborLengthHex, 16) - - 2, + 2, // bytecode has 2 bytes of cbor length prefix at the end value: `0x${auxdataFromRawRuntimeBytecode}`, }, }; @@ -318,6 +322,7 @@ export class SolidityCheckedContract extends AbstractCheckedContract { // Try to extract the auxdata from the end of the recompiled creation bytecode const [, creationAuxdataCbor, creationCborLengthHex] = splitAuxdata( this.creationBytecode, + this.auxdataStyle, ); // If we can find the auxdata at the end of the bytecode return; otherwise continue with `generateEditedContract` @@ -330,7 +335,7 @@ export class SolidityCheckedContract extends AbstractCheckedContract { offset: this.creationBytecode.substring(2).length / 2 - parseInt(creationCborLengthHex, 16) - - 2, + 2, // bytecode has 2 bytes of cbor length prefix at the end value: `0x${auxdataFromRawCreationBytecode}`, }, }; diff --git a/packages/lib-sourcify/src/lib/VyperCheckedContract.ts b/packages/lib-sourcify/src/lib/VyperCheckedContract.ts index 48b952ad7..dd1c026f9 100644 --- a/packages/lib-sourcify/src/lib/VyperCheckedContract.ts +++ b/packages/lib-sourcify/src/lib/VyperCheckedContract.ts @@ -8,6 +8,7 @@ import { } from './IVyperCompiler'; import { AbstractCheckedContract } from './AbstractCheckedContract'; import { id } from 'ethers'; +import { AuxdataStyle, splitAuxdata } from '@ethereum-sourcify/bytecode-utils'; /** * Abstraction of a checked vyper contract. With metadata and source (vyper) files. @@ -18,6 +19,7 @@ export class VyperCheckedContract extends AbstractCheckedContract { vyperSettings: VyperSettings; vyperJsonInput!: VyperJsonInput; compilerOutput?: VyperOutput; + auxdataStyle: AuxdataStyle.VYPER = AuxdataStyle.VYPER; generateMetadata(output?: VyperOutput) { let outputMetadata: MetadataOutput; @@ -181,4 +183,71 @@ export class VyperCheckedContract extends AbstractCheckedContract { runtimeLinkReferences: {}, }; } + + public async generateCborAuxdataPositions() { + if ( + this.creationBytecode === undefined || + this.runtimeBytecode === undefined + ) { + return false; + } + + if (this.compilerOutput === undefined) { + return false; + } + + // Extract the auxdata from the end of the recompiled runtime bytecode + const [, runtimeAuxdataCbor, runtimeCborLengthHex] = splitAuxdata( + this.runtimeBytecode, + this.auxdataStyle, + ); + + this.runtimeBytecodeCborAuxdata = {}; + if (runtimeAuxdataCbor) { + const auxdataFromRawRuntimeBytecode = `${runtimeAuxdataCbor}${runtimeCborLengthHex}`; + + // we divide by 2 because we store the length in bytes (without 0x) + this.runtimeBytecodeCborAuxdata = { + '1': { + offset: + this.runtimeBytecode.substring(2).length / 2 - + parseInt( + runtimeCborLengthHex || + '0' /** handles vyper lower than 0.3.5 in which runtimeCborLengthHex is '' */, + 16, + ) - + 2, + value: `0x${auxdataFromRawRuntimeBytecode}`, + }, + }; + } + + // Try to extract the auxdata from the end of the recompiled creation bytecode + const [, creationAuxdataCbor, creationCborLengthHex] = splitAuxdata( + this.creationBytecode, + this.auxdataStyle, + ); + + this.creationBytecodeCborAuxdata = {}; + // If we can find the auxdata at the end of the bytecode return; otherwise continue with `generateEditedContract` + if (creationAuxdataCbor) { + const auxdataFromRawCreationBytecode = `${creationAuxdataCbor}${creationCborLengthHex}`; + + // we divide by 2 because we store the length in bytes (without 0x) + this.creationBytecodeCborAuxdata = { + '1': { + offset: + this.creationBytecode.substring(2).length / 2 - + parseInt( + creationCborLengthHex || + '0' /** handles vyper lower than 0.3.5 in which creationCborLengthHex is '' */, + 16, + ), + value: `0x${auxdataFromRawCreationBytecode}`, + }, + }; + return true; + } + return true; + } } diff --git a/packages/lib-sourcify/src/lib/verification.ts b/packages/lib-sourcify/src/lib/verification.ts index ae9198ec8..b9c9f98c9 100644 --- a/packages/lib-sourcify/src/lib/verification.ts +++ b/packages/lib-sourcify/src/lib/verification.ts @@ -17,6 +17,7 @@ import { LinkReferences, } from './types'; import { + AuxdataStyle, decode as bytecodeDecode, splitAuxdata, } from '@ethereum-sourcify/bytecode-utils'; @@ -99,10 +100,7 @@ export async function verifyDeployed( } const generateRuntimeCborAuxdataPositions = async () => { - if ( - checkedContract instanceof SolidityCheckedContract && - !checkedContract.runtimeBytecodeCborAuxdata - ) { + if (!checkedContract.runtimeBytecodeCborAuxdata) { await checkedContract.generateCborAuxdataPositions(); } return checkedContract.runtimeBytecodeCborAuxdata || {}; @@ -156,10 +154,7 @@ export async function verifyDeployed( } const generateCreationCborAuxdataPositions = async () => { - if ( - checkedContract instanceof SolidityCheckedContract && - !checkedContract.creationBytecodeCborAuxdata - ) { + if (!checkedContract.creationBytecodeCborAuxdata) { await checkedContract.generateCborAuxdataPositions(); } return checkedContract.creationBytecodeCborAuxdata || {}; @@ -230,14 +225,26 @@ export async function verifyDeployed( try { if ( checkedContract instanceof SolidityCheckedContract && - splitAuxdata(match.onchainRuntimeBytecode || '')[1] === - splitAuxdata(checkedContract.runtimeBytecode || '')[1] && + splitAuxdata( + match.onchainRuntimeBytecode || '', + AuxdataStyle.SOLIDITY, + )[1] === + splitAuxdata( + checkedContract.runtimeBytecode || '', + AuxdataStyle.SOLIDITY, + )[1] && match.runtimeMatch === null && match.creationMatch === null && checkedContract.metadata.settings.optimizer?.enabled ) { - const [, deployedAuxdata] = splitAuxdata(runtimeBytecode); - const [, recompiledAuxdata] = splitAuxdata(recompiled.runtimeBytecode); + const [, deployedAuxdata] = splitAuxdata( + runtimeBytecode, + AuxdataStyle.SOLIDITY, + ); + const [, recompiledAuxdata] = splitAuxdata( + recompiled.runtimeBytecode, + AuxdataStyle.SOLIDITY, + ); // Metadata hashes match but bytecodes don't match. if (deployedAuxdata === recompiledAuxdata) { (match as Match).runtimeMatch = 'extra-file-input-bug'; @@ -888,7 +895,7 @@ const saltToHex = (salt: string) => { function endsWithMetadataHash(bytecode: string) { let endsWithMetadata: boolean; try { - const decodedCBOR = bytecodeDecode(bytecode); + const decodedCBOR = bytecodeDecode(bytecode, AuxdataStyle.SOLIDITY); endsWithMetadata = !!decodedCBOR.ipfs || !!decodedCBOR['bzzr0'] || !!decodedCBOR['bzzr1']; } catch (e) { diff --git a/services/monitor/src/ChainMonitor.ts b/services/monitor/src/ChainMonitor.ts index 21b088993..06c540ea0 100644 --- a/services/monitor/src/ChainMonitor.ts +++ b/services/monitor/src/ChainMonitor.ts @@ -2,7 +2,10 @@ import { FileHash } from "./util"; import { Block, TransactionResponse, getCreateAddress } from "ethers"; import assert from "assert"; import { EventEmitter } from "stream"; -import { decode as bytecodeDecode } from "@ethereum-sourcify/bytecode-utils"; +import { + AuxdataStyle, + decode as bytecodeDecode, +} from "@ethereum-sourcify/bytecode-utils"; import { SourcifyChain } from "@ethereum-sourcify/lib-sourcify"; import logger from "./logger"; import { @@ -261,7 +264,7 @@ export default class ChainMonitor extends EventEmitter { return; } try { - const cborData = bytecodeDecode(bytecode); + const cborData = bytecodeDecode(bytecode, AuxdataStyle.SOLIDITY); metadataHash = FileHash.fromCborData(cborData); } catch (err: any) { this.chainLogger.info("Error extracting cborAuxdata or metadata hash", { diff --git a/services/monitor/src/util.ts b/services/monitor/src/util.ts index cc52d2517..5d4baf50c 100644 --- a/services/monitor/src/util.ts +++ b/services/monitor/src/util.ts @@ -1,4 +1,4 @@ -import { DecodedObject } from "@ethereum-sourcify/bytecode-utils"; +import { SolidityDecodedObject } from "@ethereum-sourcify/bytecode-utils"; import { DecentralizedStorageOrigin } from "./types"; export type FetchedFileCallback = (fetchedFile: string) => any; @@ -45,7 +45,7 @@ export class FileHash { return null; } - static fromCborData(cborData: DecodedObject): FileHash { + static fromCborData(cborData: SolidityDecodedObject): FileHash { for (const origin of KNOWN_CBOR_ORIGINS) { const fileHash = cborData[origin]; if (fileHash) { diff --git a/services/server/src/server/controllers/verification/session-state/session-state.handlers.ts b/services/server/src/server/controllers/verification/session-state/session-state.handlers.ts index cff09681e..e4f501be7 100644 --- a/services/server/src/server/controllers/verification/session-state/session-state.handlers.ts +++ b/services/server/src/server/controllers/verification/session-state/session-state.handlers.ts @@ -18,7 +18,10 @@ import { import { BadRequestError } from "../../../../common/errors"; import { StatusCodes } from "http-status-codes"; -import { decode as bytecodeDecode } from "@ethereum-sourcify/bytecode-utils"; +import { + AuxdataStyle, + decode as bytecodeDecode, +} from "@ethereum-sourcify/bytecode-utils"; import logger from "../../../../common/logger"; import { Services } from "../../../services/services"; import { ChainRepository } from "../../../../sourcify-chain-repository"; @@ -91,7 +94,10 @@ export async function addInputContractEndpoint(req: Request, res: Response) { const bytecode = await sourcifyChain.getBytecode(address); - const { ipfs: metadataIpfsCid } = bytecodeDecode(bytecode); + const { ipfs: metadataIpfsCid } = bytecodeDecode( + bytecode, + AuxdataStyle.SOLIDITY, + ); if (!metadataIpfsCid) { throw new BadRequestError("The contract doesn't have a metadata IPFS CID"); From 79cc4543ee41f12e57f6e9e2c288a24ff62eebe2 Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Thu, 12 Dec 2024 08:43:01 +0100 Subject: [PATCH 2/6] handle vyper incorrect semver versions, support auxdata position also for versions < 0.3.10 --- .../src/lib/VyperCheckedContract.ts | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/lib-sourcify/src/lib/VyperCheckedContract.ts b/packages/lib-sourcify/src/lib/VyperCheckedContract.ts index dd1c026f9..066616ab2 100644 --- a/packages/lib-sourcify/src/lib/VyperCheckedContract.ts +++ b/packages/lib-sourcify/src/lib/VyperCheckedContract.ts @@ -9,6 +9,7 @@ import { import { AbstractCheckedContract } from './AbstractCheckedContract'; import { id } from 'ethers'; import { AuxdataStyle, splitAuxdata } from '@ethereum-sourcify/bytecode-utils'; +import semver from 'semver'; /** * Abstraction of a checked vyper contract. With metadata and source (vyper) files. @@ -19,7 +20,10 @@ export class VyperCheckedContract extends AbstractCheckedContract { vyperSettings: VyperSettings; vyperJsonInput!: VyperJsonInput; compilerOutput?: VyperOutput; - auxdataStyle: AuxdataStyle.VYPER = AuxdataStyle.VYPER; + auxdataStyle: + | AuxdataStyle.VYPER + | AuxdataStyle.VYPER_LT_0_3_10 + | AuxdataStyle.VYPER_LT_0_3_5; generateMetadata(output?: VyperOutput) { let outputMetadata: MetadataOutput; @@ -107,6 +111,27 @@ export class VyperCheckedContract extends AbstractCheckedContract { super(); this.vyperCompiler = vyperCompiler; this.compilerVersion = vyperCompilerVersion; + + // Vyper beta and rc versions are not semver compliant, so we need to handle them differently + let compilerVersionForComparison = this.compilerVersion; + if (!semver.valid(this.compilerVersion)) { + // Check for beta or release candidate versions + if (this.compilerVersion.match(/\d+\.\d+\.\d+(b\d+|rc\d+)/)) { + compilerVersionForComparison = `${this.compilerVersion + .split('+')[0] + .replace(/(b\d+|rc\d+)$/, '')}+${this.compilerVersion.split('+')[1]}`; + } else { + throw new Error('Invalid Vyper compiler version'); + } + } + // Vyper version support for auxdata is different for each version + if (semver.lt(compilerVersionForComparison, '0.3.5')) { + this.auxdataStyle = AuxdataStyle.VYPER_LT_0_3_5; + } else if (semver.lt(compilerVersionForComparison, '0.3.10')) { + this.auxdataStyle = AuxdataStyle.VYPER_LT_0_3_10; + } else { + this.auxdataStyle = AuxdataStyle.VYPER; + } this.compiledPath = compiledPath; this.name = name; this.sources = sources; From 712e91c4f10f9786718fe63eb1517dba4ab8b798 Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Mon, 16 Dec 2024 10:16:10 +0100 Subject: [PATCH 3/6] Refactor Solidity and Vyper contract handling to improve auxdata extraction - Changed `auxdataStyle` in `SolidityCheckedContract` to be a static readonly property. - Updated bytecode decoding in both `SolidityCheckedContract` and `VyperCheckedContract` to use the static `auxdataStyle`. - Simplified `generateCborAuxdataPositions` method in `VyperCheckedContract` by introducing a helper function for auxdata position generation. - Added comments to clarify the use of `AuxdataStyle.SOLIDITY` for bytecode decoding in `ChainMonitor` and session state handlers. - Ensured that `generateCborAuxdataPositions` is called when necessary in `AbstractDatabaseService` to maintain data integrity. --- .../src/lib/SolidityCheckedContract.ts | 20 ++-- .../src/lib/VyperCheckedContract.ts | 92 +++++++++---------- services/monitor/src/ChainMonitor.ts | 6 ++ .../session-state/session-state.handlers.ts | 6 ++ .../AbstractDatabaseService.ts | 6 +- 5 files changed, 72 insertions(+), 58 deletions(-) diff --git a/packages/lib-sourcify/src/lib/SolidityCheckedContract.ts b/packages/lib-sourcify/src/lib/SolidityCheckedContract.ts index 16e482888..e796eefbd 100644 --- a/packages/lib-sourcify/src/lib/SolidityCheckedContract.ts +++ b/packages/lib-sourcify/src/lib/SolidityCheckedContract.ts @@ -48,7 +48,7 @@ export class SolidityCheckedContract extends AbstractCheckedContract { solcJsonInput: any; compilerOutput?: SolidityOutput; - auxdataStyle: AuxdataStyle.SOLIDITY = AuxdataStyle.SOLIDITY; + static readonly auxdataStyle: AuxdataStyle.SOLIDITY = AuxdataStyle.SOLIDITY; /** Checks whether this contract is valid or not. * This is a static method due to persistence issues. @@ -113,7 +113,10 @@ export class SolidityCheckedContract extends AbstractCheckedContract { ): Promise { let decodedAuxdata; try { - decodedAuxdata = decodeBytecode(runtimeBytecode, this.auxdataStyle); + decodedAuxdata = decodeBytecode( + runtimeBytecode, + SolidityCheckedContract.auxdataStyle, + ); } catch (err) { // There is no auxdata at all in this contract return null; @@ -264,16 +267,13 @@ export class SolidityCheckedContract extends AbstractCheckedContract { */ public async generateCborAuxdataPositions(forceEmscripten = false) { if ( - this.creationBytecode === undefined || - this.runtimeBytecode === undefined + !this.creationBytecode || + !this.runtimeBytecode || + !this.compilerOutput ) { return false; } - if (this.compilerOutput === undefined) { - return false; - } - // Auxdata array extracted from the compiler's `legacyAssembly` field const auxdatasFromCompilerOutput = findAuxdatasInLegacyAssembly( this.compilerOutput.contracts[this.compiledPath][this.name].evm @@ -292,7 +292,7 @@ export class SolidityCheckedContract extends AbstractCheckedContract { // Extract the auxdata from the end of the recompiled runtime bytecode const [, runtimeAuxdataCbor, runtimeCborLengthHex] = splitAuxdata( this.runtimeBytecode, - this.auxdataStyle, + SolidityCheckedContract.auxdataStyle, ); const auxdataFromRawRuntimeBytecode = `${runtimeAuxdataCbor}${runtimeCborLengthHex}`; @@ -322,7 +322,7 @@ export class SolidityCheckedContract extends AbstractCheckedContract { // Try to extract the auxdata from the end of the recompiled creation bytecode const [, creationAuxdataCbor, creationCborLengthHex] = splitAuxdata( this.creationBytecode, - this.auxdataStyle, + SolidityCheckedContract.auxdataStyle, ); // If we can find the auxdata at the end of the bytecode return; otherwise continue with `generateEditedContract` diff --git a/packages/lib-sourcify/src/lib/VyperCheckedContract.ts b/packages/lib-sourcify/src/lib/VyperCheckedContract.ts index 066616ab2..77e720f26 100644 --- a/packages/lib-sourcify/src/lib/VyperCheckedContract.ts +++ b/packages/lib-sourcify/src/lib/VyperCheckedContract.ts @@ -1,4 +1,9 @@ -import { MetadataOutput, RecompilationResult, StringMap } from './types'; +import { + CompiledContractCborAuxdata, + MetadataOutput, + RecompilationResult, + StringMap, +} from './types'; import { logInfo, logSilly, logWarn } from './logger'; import { IVyperCompiler, @@ -211,68 +216,61 @@ export class VyperCheckedContract extends AbstractCheckedContract { public async generateCborAuxdataPositions() { if ( - this.creationBytecode === undefined || - this.runtimeBytecode === undefined + !this.creationBytecode || + !this.runtimeBytecode || + !this.compilerOutput ) { return false; } - if (this.compilerOutput === undefined) { - return false; - } - - // Extract the auxdata from the end of the recompiled runtime bytecode const [, runtimeAuxdataCbor, runtimeCborLengthHex] = splitAuxdata( this.runtimeBytecode, this.auxdataStyle, ); - this.runtimeBytecodeCborAuxdata = {}; - if (runtimeAuxdataCbor) { - const auxdataFromRawRuntimeBytecode = `${runtimeAuxdataCbor}${runtimeCborLengthHex}`; - - // we divide by 2 because we store the length in bytes (without 0x) - this.runtimeBytecodeCborAuxdata = { - '1': { - offset: - this.runtimeBytecode.substring(2).length / 2 - - parseInt( - runtimeCborLengthHex || - '0' /** handles vyper lower than 0.3.5 in which runtimeCborLengthHex is '' */, - 16, - ) - - 2, - value: `0x${auxdataFromRawRuntimeBytecode}`, - }, - }; - } + this.runtimeBytecodeCborAuxdata = this.tryGenerateCborAuxdataPosition( + this.runtimeBytecode, + runtimeAuxdataCbor, + runtimeCborLengthHex, + ); - // Try to extract the auxdata from the end of the recompiled creation bytecode const [, creationAuxdataCbor, creationCborLengthHex] = splitAuxdata( this.creationBytecode, this.auxdataStyle, ); - this.creationBytecodeCborAuxdata = {}; - // If we can find the auxdata at the end of the bytecode return; otherwise continue with `generateEditedContract` - if (creationAuxdataCbor) { - const auxdataFromRawCreationBytecode = `${creationAuxdataCbor}${creationCborLengthHex}`; + this.creationBytecodeCborAuxdata = this.tryGenerateCborAuxdataPosition( + this.creationBytecode, + creationAuxdataCbor, + creationCborLengthHex, + ); + + return true; + } - // we divide by 2 because we store the length in bytes (without 0x) - this.creationBytecodeCborAuxdata = { - '1': { - offset: - this.creationBytecode.substring(2).length / 2 - - parseInt( - creationCborLengthHex || - '0' /** handles vyper lower than 0.3.5 in which creationCborLengthHex is '' */, - 16, - ), - value: `0x${auxdataFromRawCreationBytecode}`, - }, - }; - return true; + private tryGenerateCborAuxdataPosition( + bytecode: string, + auxdataCbor: string, + cborLengthHex: string, + ): CompiledContractCborAuxdata { + if (!auxdataCbor) { + return {}; } - return true; + + const auxdataFromRawBytecode = `${auxdataCbor}${cborLengthHex}`; + + return { + '1': { + offset: + // we divide by 2 because we store the length in bytes (without 0x) + bytecode.substring(2).length / 2 - + parseInt( + cborLengthHex || + '0' /** handles vyper lower than 0.3.5 in which cborLengthHex is '' */, + 16, + ), + value: `0x${auxdataFromRawBytecode}`, + }, + }; } } diff --git a/services/monitor/src/ChainMonitor.ts b/services/monitor/src/ChainMonitor.ts index 06c540ea0..38749d15c 100644 --- a/services/monitor/src/ChainMonitor.ts +++ b/services/monitor/src/ChainMonitor.ts @@ -264,6 +264,12 @@ export default class ChainMonitor extends EventEmitter { return; } try { + /** + * We decode the bytecode using `AuxdataStyle.SOLIDITY` since Solidity is currently + * the only smart contract language that includes metadata information in its bytecode. + * This metadata contains an IPFS CID that points to a JSON file with the contract's + * source code and compiler settings. + */ const cborData = bytecodeDecode(bytecode, AuxdataStyle.SOLIDITY); metadataHash = FileHash.fromCborData(cborData); } catch (err: any) { diff --git a/services/server/src/server/controllers/verification/session-state/session-state.handlers.ts b/services/server/src/server/controllers/verification/session-state/session-state.handlers.ts index e4f501be7..47909eb90 100644 --- a/services/server/src/server/controllers/verification/session-state/session-state.handlers.ts +++ b/services/server/src/server/controllers/verification/session-state/session-state.handlers.ts @@ -94,6 +94,12 @@ export async function addInputContractEndpoint(req: Request, res: Response) { const bytecode = await sourcifyChain.getBytecode(address); + /** + * We decode the bytecode using `AuxdataStyle.SOLIDITY` since Solidity is currently + * the only smart contract language that includes metadata information in its bytecode. + * This metadata contains an IPFS CID that points to a JSON file with the contract's + * source code and compiler settings. + */ const { ipfs: metadataIpfsCid } = bytecodeDecode( bytecode, AuxdataStyle.SOLIDITY, diff --git a/services/server/src/server/services/storageServices/AbstractDatabaseService.ts b/services/server/src/server/services/storageServices/AbstractDatabaseService.ts index 2b45c88c7..e83861858 100644 --- a/services/server/src/server/services/storageServices/AbstractDatabaseService.ts +++ b/services/server/src/server/services/storageServices/AbstractDatabaseService.ts @@ -149,7 +149,11 @@ export default abstract class AbstractDatabaseService { recompiledContract.compiledPath ][recompiledContract.name]; - if (recompiledContract instanceof SolidityCheckedContract) { + // If during verification `generateCborAuxdataPositions` was not called, we call it now + if ( + recompiledContract.runtimeBytecodeCborAuxdata === undefined && + recompiledContract.creationBytecodeCborAuxdata === undefined + ) { if (!(await recompiledContract.generateCborAuxdataPositions())) { throw new Error( `cannot generate contract artifacts address=${match.address} chainId=${match.chainId}`, From 6cd8914543d7c426cdd2de4038d3612bb2612b68 Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Mon, 16 Dec 2024 11:01:59 +0100 Subject: [PATCH 4/6] add comment to explain `generateCborAuxdataPositions` return conditions --- packages/lib-sourcify/src/lib/SolidityCheckedContract.ts | 2 +- packages/lib-sourcify/src/lib/VyperCheckedContract.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/lib-sourcify/src/lib/SolidityCheckedContract.ts b/packages/lib-sourcify/src/lib/SolidityCheckedContract.ts index e796eefbd..b711ddfad 100644 --- a/packages/lib-sourcify/src/lib/SolidityCheckedContract.ts +++ b/packages/lib-sourcify/src/lib/SolidityCheckedContract.ts @@ -263,7 +263,7 @@ export class SolidityCheckedContract extends AbstractCheckedContract { /** * Finds the positions of the auxdata in the runtime and creation bytecodes. * Saves the CborAuxdata position (offset) and value in the runtime- and creationBytecodeCborAuxdata fields. - * + * @returns false if the auxdata positions cannot be generated or if the auxdata in legacyAssembly differs from the auxdata in the bytecode, true otherwise. */ public async generateCborAuxdataPositions(forceEmscripten = false) { if ( diff --git a/packages/lib-sourcify/src/lib/VyperCheckedContract.ts b/packages/lib-sourcify/src/lib/VyperCheckedContract.ts index 77e720f26..11784fddf 100644 --- a/packages/lib-sourcify/src/lib/VyperCheckedContract.ts +++ b/packages/lib-sourcify/src/lib/VyperCheckedContract.ts @@ -214,6 +214,10 @@ export class VyperCheckedContract extends AbstractCheckedContract { }; } + /** + * Generate the cbor auxdata positions for the creation and runtime bytecodes. + * @returns false if the auxdata positions cannot be generated, true otherwise. + */ public async generateCborAuxdataPositions() { if ( !this.creationBytecode || From 3c2fb82c0a8216094361fa59ff2b0bf9a3b92a33 Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Mon, 16 Dec 2024 13:58:08 +0100 Subject: [PATCH 5/6] Refactor bytecode decoding to enhance auxdata handling for Vyper versions. Updated the `decode` function to support various auxdata styles, including specific handling for Vyper versions < 0.3.10 and < 0.3.5. Improved error messages for missing auxdata in bytecode. Updated tests to reflect changes in auxdata style usage. --- packages/bytecode-utils/src/lib/bytecode.ts | 48 +++++++------------ packages/bytecode-utils/test/bytecode.spec.ts | 8 ++-- 2 files changed, 22 insertions(+), 34 deletions(-) diff --git a/packages/bytecode-utils/src/lib/bytecode.ts b/packages/bytecode-utils/src/lib/bytecode.ts index 11232e696..2b67a9f9d 100644 --- a/packages/bytecode-utils/src/lib/bytecode.ts +++ b/packages/bytecode-utils/src/lib/bytecode.ts @@ -39,9 +39,10 @@ export enum AuxdataStyle { /** * Decode contract's bytecode * @param bytecode - hex of the bytecode with 0x prefix + * @param auxdataStyle - The style of auxdata, check AuxdataStyle enum for more info * @returns Object describing the contract */ -export const decode = ( +export const decode = ( bytecode: string, auxdataStyle: T, ): T extends AuxdataStyle.SOLIDITY @@ -56,35 +57,18 @@ export const decode = ( // split auxdata const [, auxdata] = splitAuxdata(bytecode, auxdataStyle); + if (!auxdata) { + throw Error('Auxdata is not in the bytecode'); + } + // See more here: https://github.com/vyperlang/vyper/pull/3010 if (auxdataStyle === AuxdataStyle.VYPER) { - if (!auxdata) { - // Attempt fallback auxdata styles for older Vyper versions - const fallbackStyles = [ - AuxdataStyle.VYPER_LT_0_3_10, // Vyper bytecode before version 0.3.10 uses SOLIDITY AuxdataStyle - AuxdataStyle.VYPER_LT_0_3_5, // Vyper bytecode before version 0.3.5 doesn't include auxdata length - ]; - - for (const style of fallbackStyles) { - const [, fallbackAuxdata] = splitAuxdata(bytecode, style); - if (fallbackAuxdata) { - // cbor decode the object and get a json - const cborDecodedObject = CBOR.decode( - arrayify(`0x${fallbackAuxdata}`), - ); - return { - compiler: cborDecodedObject.vyper.join('.'), - } as any; - } - } - - throw Error('Auxdata is not in the execution bytecode'); - } - // cbor decode the object and get a json const cborDecodedObject = CBOR.decode(arrayify(`0x${auxdata}`)); // Starting with version 0.3.10, Vyper stores the auxdata as an array + // after 0.3.10: [integrity, runtimesize, datasize,immutablesize,version_cbor_object] + // after 0.4.1: [runtimesize, datasize,immutablesize,version_cbor_object] // See more here: https://github.com/vyperlang/vyper/pull/3584 if (cborDecodedObject instanceof Array) { // read the last element from array, it contains the compiler version @@ -111,14 +95,16 @@ export const decode = ( } } throw Error('This version of Vyper is not supported'); + } else if ( + auxdataStyle === AuxdataStyle.VYPER_LT_0_3_10 || + auxdataStyle === AuxdataStyle.VYPER_LT_0_3_5 + ) { + // cbor decode the object and get a json + const cborDecodedObject = CBOR.decode(arrayify(`0x${auxdata}`)); + return { + compiler: cborDecodedObject.vyper.join('.'), + } as any; } else if (auxdataStyle === AuxdataStyle.SOLIDITY) { - // split auxdata - const [, auxdata] = splitAuxdata(bytecode, auxdataStyle); - - if (!auxdata) { - throw Error('Auxdata is not in the execution bytecode'); - } - // cbor decode the object and get a json const cborDecodedObject = CBOR.decode(arrayify(`0x${auxdata}`)); diff --git a/packages/bytecode-utils/test/bytecode.spec.ts b/packages/bytecode-utils/test/bytecode.spec.ts index a2062c763..6295a2c81 100644 --- a/packages/bytecode-utils/test/bytecode.spec.ts +++ b/packages/bytecode-utils/test/bytecode.spec.ts @@ -107,7 +107,7 @@ describe('bytecode utils', function () { it('bytecode decode Vyper cbor auxdata for version < 0.3.10', () => { chai - .expect(decode(BYTECODE_VYPER_NO_ARRAY, AuxdataStyle.VYPER)) + .expect(decode(BYTECODE_VYPER_NO_ARRAY, AuxdataStyle.VYPER_LT_0_3_10)) .to.deep.equal({ compiler: '0.3.8', }); @@ -115,7 +115,9 @@ describe('bytecode utils', function () { it('bytecode decode Vyper cbor auxdata for version < 0.3.5', () => { chai - .expect(decode(BYTECODE_VYPER_NO_AUXDATA_LENGTH, AuxdataStyle.VYPER)) + .expect( + decode(BYTECODE_VYPER_NO_AUXDATA_LENGTH, AuxdataStyle.VYPER_LT_0_3_5), + ) .to.deep.equal({ compiler: '0.3.4', }); @@ -141,7 +143,7 @@ describe('bytecode utils', function () { } catch (e) { chai .expect((e as Error).message) - .to.equal('Auxdata is not in the execution bytecode'); + .to.equal('Auxdata is not in the bytecode'); } }); }); From 029ff347176d06589d79c708f80285de913a1a33 Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Mon, 16 Dec 2024 14:29:46 +0100 Subject: [PATCH 6/6] Refactor bytecode decoding to replace `compiler` with `vyperVersion` in Vyper-related types and tests. --- packages/bytecode-utils/src/lib/bytecode.ts | 10 ++++------ packages/bytecode-utils/test/bytecode.spec.ts | 8 ++++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/bytecode-utils/src/lib/bytecode.ts b/packages/bytecode-utils/src/lib/bytecode.ts index 2b67a9f9d..586822da4 100644 --- a/packages/bytecode-utils/src/lib/bytecode.ts +++ b/packages/bytecode-utils/src/lib/bytecode.ts @@ -24,9 +24,7 @@ export type VyperDecodedObject = { runtimeSize?: number; dataSizes?: number[]; immutableSize?: number; - compiler: { - vyper: [number, number, number]; - }; + vyperVersion: string; }; export enum AuxdataStyle { @@ -83,14 +81,14 @@ export const decode = ( runtimeSize: cborDecodedObject[1], dataSizes: cborDecodedObject[2], immutableSize: cborDecodedObject[3], - compiler: compilerVersion, + vyperVersion: compilerVersion, } as any; } else if (semver.gte(compilerVersion, '0.3.10')) { return { runtimeSize: cborDecodedObject[0], dataSizes: cborDecodedObject[1], immutableSize: cborDecodedObject[2], - compiler: compilerVersion, + vyperVersion: compilerVersion, } as any; } } @@ -102,7 +100,7 @@ export const decode = ( // cbor decode the object and get a json const cborDecodedObject = CBOR.decode(arrayify(`0x${auxdata}`)); return { - compiler: cborDecodedObject.vyper.join('.'), + vyperVersion: cborDecodedObject.vyper.join('.'), } as any; } else if (auxdataStyle === AuxdataStyle.SOLIDITY) { // cbor decode the object and get a json diff --git a/packages/bytecode-utils/test/bytecode.spec.ts b/packages/bytecode-utils/test/bytecode.spec.ts index 6295a2c81..d281ebe80 100644 --- a/packages/bytecode-utils/test/bytecode.spec.ts +++ b/packages/bytecode-utils/test/bytecode.spec.ts @@ -90,7 +90,7 @@ describe('bytecode utils', function () { runtimeSize: 143, dataSizes: [], immutableSize: 0, - compiler: '0.4.1', + vyperVersion: '0.4.1', }); }); @@ -101,7 +101,7 @@ describe('bytecode utils', function () { runtimeSize: 143, dataSizes: [], immutableSize: 0, - compiler: '0.3.10', + vyperVersion: '0.3.10', }); }); @@ -109,7 +109,7 @@ describe('bytecode utils', function () { chai .expect(decode(BYTECODE_VYPER_NO_ARRAY, AuxdataStyle.VYPER_LT_0_3_10)) .to.deep.equal({ - compiler: '0.3.8', + vyperVersion: '0.3.8', }); }); @@ -119,7 +119,7 @@ describe('bytecode utils', function () { decode(BYTECODE_VYPER_NO_AUXDATA_LENGTH, AuxdataStyle.VYPER_LT_0_3_5), ) .to.deep.equal({ - compiler: '0.3.4', + vyperVersion: '0.3.4', }); });