Merge pull request #416 from ethereum/staging

New release

.vscode/launch.json

@@ -26,7 +26,7 @@
       "type": "node",
       "request": "launch",
       "name": "Monitor",
-      "program": "${workspaceFolder}/scripts/run_monitor.js",
+      "program": "${workspaceFolder}/dist/monitor/monitor.js",
       "envFile": "${workspaceFolder}/environments/.env",
       "preLaunchTask": "npm: build:lerna",
       "outFiles": [

README.md

@@ -123,13 +123,12 @@ The repository is accessible via this [link](https://repo.sourcify.dev/).
 The repository UI currently looks like this:
 <img src="./public/ui_start.png" width="80%"/>
 
-It offers the option to search, donwload or open folders.
+It offers the option to search, download or open folders.
 
 For example to download:
 
 <img src="./public/download.png" width="80%"/>
 
-
 Or if you want to search something:
 
 <img src="./public/search.png" width="80%"/>
@@ -269,3 +268,22 @@ You can find examples in our Postman collection in the root of this project `Sou
 * [Get file tree (full or partial match)](docs/api/server/getAnyTreeByChainAndAddress.md) : `GET /files/tree/any/:chain/:address`
 * [Get source files (full or partial match)](docs/api/server/getAnyByChainAndAddress.md) : `GET /files/any/:chain/:address`
 * [Server health](docs/api/server/health.md) : `GET /health`
+
+### Repository API
+
+This paragraph should be read with the following in mind:
+* `<MATCH_QUALITY>` is either `full_match` or `partial_match`.
+* `<CHAIN_ID>` is the respective ID of the chain (e.g. 1 for Ethereum Mainnet, 5 for Görli). See the full list [here](https://chainid.network/).
+* `<CONTRACT_ADDRESS>` is the hexadecimal address of the contract (40 hex digits prefixed by `0x`).
+* `<FILE_PATH>` is the original path of a source file. Sourcify doesn't flatten the file hierarchy, so everything is provided as during compilation.
+
+The repository (https://repo.sourcify.dev) provides the following GET endpoints:
+* JSON-formatted metadata file (with ABI): `/contracts/<MATCH_QUALITY>/<CHAIN_ID>/<CONTRACT_ADDRESS>/metadata.json`
+* Source file: `/contracts/<MATCH_QUALITY>/<CHAIN_ID>/<CONTRACT_ADDRESS>/sources/<FILE_PATH>`
+* JSON-formatted full and partial match count per chain: `/stats.json`
+* JSON-formatted timestamp and version of the repo: `/manifest.json`
+
+E.g. the following URL fetches `browser/OceanMan.sol`, a source of the contract deployed on Ethereum Mainnet (chain ID: 1) at address 0x00000000064Ecc11c97AC4a5551F279532Bf9E0D.
+* https://repo.sourcify.dev/contracts/full_match/1/0x00000000064Ecc11c97AC4a5551F279532Bf9E0D/sources/browser/OceanMan.sol
+
+In order to fetch from the staging repository, replace https://repo.sourcify.dev with https://contractrepo.sourcify.shardlabs.io.

docs/api/server/verification1/check-by-addresses.md

@@ -18,19 +18,22 @@ Checks if contract with the desired chain and address is verified and in the rep
 [
     {
         "address": "0x0000A906D248Cc99FB8CB296C8Ad8C6Df05431c9",
-        "status": "perfect"
+        "status": "perfect",
+        "chains": ["3"]
     },
     {
         "address": "0x0A67477639a71bf98528280D3724f465A1814740",
-        "status": "perfect"
+        "status": "perfect",
+        "chains": ["5", "3"]
     },
     {
         "address": "0x0A67477639a71bf98528280D3724f465A1814741",
         "status": "false"
     },
     {
         "address": "0x0a06cc1Ce1105d90ce01752813449A029906aD7b",
-        "status": "perfect"
+        "status": "perfect",
+        "chains": ["5"]
     }
 ]
 ```

docs/api/server/verification1/verify.md

@@ -27,7 +27,7 @@ If using `application/json`, the files should be in an object under the key `fil
 
 ## Responses
 
-**Condition** : Contract is perfect match.
+**Condition** : The recompiled contract matches the deployed version `perfect`ly.
 
 **Code** : `200 OK`
 
@@ -45,7 +45,7 @@ If using `application/json`, the files should be in an object under the key `fil
 ```
 ### OR
 
-**Condition** : Contract is partial match.
+**Condition** : The recompiled contract matches the deployed version `partial`ly.
 
 **Code** : `200 OK`
 
@@ -62,6 +62,25 @@ If using `application/json`, the files should be in an object under the key `fil
 }
 ```
 
+### OR
+
+**Condition** : The contract at the provided address and chain has already been sourcified at timestamp indicated by `storageTimestamp`.
+
+**Code** : `200 OK`
+
+**Content** :
+```json
+{
+    "result": [
+        {
+            "address": "0x0001Db7722Fb4211C24d4aC5E1127353116323d3",
+            "status": "perfect",
+            "storageTimestamp":"2020-11-10T14:12:15.665Z"
+        }
+    ]
+}
+```
+
 ### OR
 **Condition** : Missing or invalid parameters received.
 
@@ -70,19 +89,93 @@ If using `application/json`, the files should be in an object under the key `fil
 **Content** :
 ```json
 {
-    "error": "Missing body parameters: address, chain"
+    "message": "Validation Error: address, chain",
+    "errors": [
+        {
+            "field": "address",
+            "message": "Invalid value"
+        },
+        {
+            "field": "chain",
+            "message": "Invalid value"
+        }
+    ]
+}
+```
+
+### OR
+**Condition** : Provided valid address and chain input, but no files. This is interpreted as simply checking whether the contract at the provided address and chain has already been sourcified.
+
+**Code** : `404 Not Found`
+
+**Content** :
+```json
+{
+    "error": "The contract at the provided address has not yet been sourcified."
 }
 ```
 
 ### OR
 
-**Condition** : Failed fetching missing files. OR Contract bytecode does not match deployed bytecode.
+**Condition** : Recompiled bytecode does not match the deployed bytecode.
 
 **Code** : `500 Internal Server Error`
 
-**Content** : 
+**Content** :
+```json
+{
+    "error": "The deployed and recompiled bytecode don't match."
+}
+```
+
+### OR
+
+**Condition** : The provided chain does not have a contract deployed at the provided address.
+
+**Code** : `500 Internal Server Error`
+
+**Content** :
+```json
+{
+    "error":"Contract name: RandomName. Ethereum Mainnet does not have a contract deployed at 0x7c90F0C9Eb46391c93d0545dDF4658d3B8DF1866."
+}
+```
+
+### OR
+
+**Condition** : The provided chain is temporarily unavailable.
+
+**Code** : `500 Internal Server Error`
+
+**Content** :
+```json
+{
+    "error":"Contract name: RandomName. Ethereum Mainnet is temporarily unavailable."
+}
+```
+
+### OR
+
+**Condition** : Some resources are missing and could not be fetched.
+
+**Code** : `500 Internal Server Error`
+
+**Content** :
+```json
+{
+    "error":"Resource missing; unsuccessful fetching: browser/RandomName.sol"
+}
+```
+
+### OR
+
+**Condition** : Verifying contracts with immutable variables is not supported for the provided chain.
+
+**Code** : `500 Internal Server Error`
+
+**Content** :
 ```json
 {
-    "error": "Could not match on-chain deployed bytecode to recompiled bytecode for:\n{\n \"browser/ParameterTest.sol\": \"ParameterTest\"\n}\nAddresses checked:\n[\n \"0x0001Db7722Fb4211C24d4aC5E1127353116323d3\"\n]"
+    "error":"Contract name: RandomName. Verifying contracts with immutable variables is not supported for Ethereum Mainnet."
 }
 ```

package.json

@@ -24,8 +24,8 @@
     "dev:server:local": "npm run build:ui && TESTING=true ./scripts/run_server.sh",
     "build:ui": "cd ./ui/ && npm install && npm run build",
     "monitor:build": "npm run clean && npm run build",
-    "monitor:start": "node ./scripts/run_monitor.js",
-    "monitor": "rimraf ./dist && tsc && node ./scripts/run_monitor.js",
+    "monitor:start": "node ./dist/monitor/monitor.js",
+    "monitor": "rimraf ./dist && tsc && node ./dist/monitor/monitor.js",
     "monitor:ci": "scripts/monitor_ci.sh",
     "server:build": "npm run clean && npm run build",
     "server:start": "node ./dist/server/server.js",

services/core/src/utils/CheckedContract.ts

@@ -1,5 +1,5 @@
 import Web3 from 'web3';
-import { StringMap, SourceMap, Metadata } from './types';
+import { StringMap, Metadata } from './types';
 import { isEmpty } from './utils';
 import bunyan from 'bunyan';
 import fetch from 'node-fetch';

services/verification/src/services/Injector.ts

@@ -76,7 +76,7 @@ export class Injector {
         const chainsData = this.infuraPID ? getSupportedChains() : getFullnodeChains();
 
         for (const chain of chainsData) {
-            this.chains[chain.chainId] = {};
+            this.chains[chain.chainId] = { name: chain.name };
             if (this.infuraPID) {
                 const web3 = chain.web3[0].replace('${INFURA_API_KEY}', this.infuraPID);
                 this.chains[chain.chainId].web3 = new Web3(web3);
@@ -105,6 +105,7 @@ export class Injector {
         compiledBytecode: string
     ): Promise<Match> {
         let match: Match = { address: null, status: null };
+        const chainName = this.chains[chain].name || "The chain";
 
         for (let address of addresses) {
             address = Web3.utils.toChecksumAddress(address)
@@ -125,8 +126,16 @@ export class Injector {
             const status = this.compareBytecodes(deployedBytecode, compiledBytecode);
 
             if (status) {
-                match = { address: address, status: status };
+                match = { address, status };
                 break;
+            } else if (addresses.length === 1) {
+                if (!deployedBytecode) {
+                    match.message = `${chainName} is temporarily unavailable.`
+                } else if (deployedBytecode === "0x") {
+                    match.message = `${chainName} does not have a contract deployed at ${address}.`;
+                } else if (deployedBytecode.length === compiledBytecode.length) {
+                    match.message = `Verifying contracts with immutable variables is not supported for ${chainName}.`;
+                }
             }
         }
         return match;
@@ -202,12 +211,6 @@ export class Injector {
 
         let match: Match;
 
-        // Starting from here, we cannot trust the metadata object anymore,
-        // because it is modified inside recompile.
-        const target = Object.assign({}, contract.metadata.settings.compilationTarget);
-
-        // target is expected to have been checked in validation (size 1 assertion)
-
         if (!CheckedContract.isValid(contract)) {
             // eslint-disable-next-line no-useless-catch
             try {
@@ -264,12 +267,8 @@ export class Injector {
             this.storePartialMatchData(this.repositoryPath, chain, match.address, compilationResult, contract.solidity)
 
         } else {
-            const err = new Error(
-                `Could not match on-chain deployed bytecode to recompiled bytecode for:\n` +
-                `${JSON.stringify(target, null, ' ')}\n` +
-                `Addresses checked:\n` +
-                `${JSON.stringify(addresses, null, ' ')}`
-            );
+            const message = match.message || "The deployed and recompiled bytecode don't match."
+            const err = new Error(`Contract name: ${contract.name}. ${message}`);
 
             this.log.info({
                 loc: '[INJECT]',

services/verification/src/utils.ts

@@ -82,22 +82,23 @@ export async function recompile(
         contractName
     } = reformatMetadata(metadata, sources, log);
 
+    const loc = "[RECOMPILE]";
     const version = metadata.compiler.version;
 
     log.info(
-        {
-            loc: '[RECOMPILE]',
-            fileName: fileName,
-            contractName: contractName,
-            version: version
-        },
+        { loc, fileName, contractName, version },
         'Recompiling'
     );
 
     const compiled = await useCompiler(version, input, log);
     const output = JSON.parse(compiled);
-    const contract: any = output.contracts[fileName][contractName];
+    if (!output.contracts || !output.contracts[fileName] || !output.contracts[fileName][contractName]) {
+        const errors = output.errors.filter((e: any) => e.severity === "error").map((e: any) => e.message);
+        log.error({ loc, fileName, contractName, version, errors });
+        throw new Error("Recompilation error (probably caused by invalid metadata)");
+    }
 
+    const contract: any = output.contracts[fileName][contractName];
     return {
         bytecode: contract.evm.bytecode.object,
         deployedBytecode: `0x${contract.evm.deployedBytecode.object}`,
@@ -123,12 +124,14 @@ async function useCompiler(version: string, input: any, log: bunyan) {
     if (solcPath) {
         const logObject = {loc: "[RECOMPILE]", version, solcPath};
         log.info(logObject, "Compiling with external executable");
+
         const shellOutputBuffer = spawnSync(solcPath, ["--standard-json"], {input: inputStringified});
         if (!shellOutputBuffer.stdout) {
             log.error(logObject, shellOutputBuffer.error || "Recompilation error");
             throw new Error("Recompilation error");
         }
         compiled = shellOutputBuffer.stdout.toString();
+
     } else {
         const soljson = await getSolcJs(version, log);
         compiled = soljson.compile(inputStringified);
@@ -202,7 +205,7 @@ function reformatMetadata(
 
     input.settings = metadata.settings;
 
-    // this assumes that the size of copmilationTarget is 1
+    // this assumes that the size of compilationTarget is 1
     for (fileName in metadata.settings.compilationTarget) {
         contractName = metadata.settings.compilationTarget[fileName];
     }
@@ -231,11 +234,9 @@ function reformatMetadata(
         'metadata'
     ];
 
-    return {
-        input: input,
-        fileName: fileName,
-        contractName: contractName
-    }
+    input.settings.libraries = { "": metadata.settings.libraries || {} };
+
+    return { input, fileName, contractName };
 }
 
 

src/Dockerfile.monitor

@@ -6,7 +6,6 @@ COPY services/core ./services/core
 COPY services/validation ./services/validation
 COPY services/verification ./services/verification
 COPY src ./src
-COPY scripts/run_monitor.js ./scripts/
 COPY *.json ./
 RUN npx lerna bootstrap && npx lerna run build
 

src/config.ts

@@ -12,7 +12,7 @@ export default {
     },
     server: {
         port: process.env.SERVER_PORT || 5000,
-        url: process.env.SERVER_URL || "http//localhost:5000"
+        url: process.env.SERVER_URL || "http://localhost:5000"
     },
     localchain: {
         port: process.env.LOCALCHAIN_PORT || 8545,