hardhat e2e for dsm v3

Dockerfile.hardhat.yml

@@ -0,0 +1,12 @@
+FROM node:18.14.2
+
+WORKDIR /app
+
+COPY hardhat.config.ts ./hardhat.config.ts
+COPY package.json ./package.json
+COPY yarn.lock ./yarn.lock
+
+RUN npm install -g hardhat
+
+
+CMD ["npx", "hardhat", "node"]

docker-compose.e2e-v3.yml

@@ -0,0 +1,64 @@
+version: '3.7'
+
+services:
+  e2e_pgdb:
+    container_name: e2e_pgdb
+    image: postgres:14-alpine
+    restart: unless-stopped
+    environment:
+      - POSTGRES_DB=node_operator_keys_service_db
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+    expose:
+       - "5432:5432"
+    ports:
+       - "5433:5432"
+    volumes:
+      - ./.volumes/pgdata-17000/:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+  hardhat_node:
+    container_name: hardhat_node
+    build:
+      context: .
+      dockerfile: Dockerfile.hardhat.yml
+    ports:
+      - "8545:8545"
+    environment:
+      - FORK_URL=${FORK_URL}
+      - FORK_BLOCK=${FORK_BLOCK}
+      - WALLET_PRIVATE_KEY=${WALLET_PRIVATE_KEY}
+    volumes:
+      - .:/app
+    healthcheck:
+      test: ["CMD-SHELL", "curl -s http://127.0.0.1:8545 >/dev/null || exit 1"]
+      interval: 10s
+      retries: 10
+
+  e2e_keys_api:
+    container_name: e2e_keys_api
+    image: lidofinance/lido-keys-api:staging
+    platform: linux/amd64
+    environment:
+      - NODE_ENV=production
+      - DB_NAME=node_operator_keys_service_db
+      - DB_PORT=5432
+      - DB_HOST=e2e_pgdb
+      - DB_USER=postgres
+      - DB_PASSWORD=postgres
+      - PROVIDERS_URLS=http://hardhat_node:8545
+      - VALIDATOR_REGISTRY_ENABLE=false
+      - CHAIN_ID=${CHAIN_ID}
+      - CL_API_URLS=""
+    depends_on:
+      e2e_pgdb:
+        condition: service_healthy
+    ports:
+      - "3000:3000"
+
+    links:
+      - e2e_pgdb

hardhat.config.ts

@@ -0,0 +1,30 @@
+import { HardhatUserConfig } from 'hardhat/config';
+import * as dotenv from 'dotenv';
+dotenv.config();
+
+// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+const RPC_URL = process.env.FORK_URL!;
+// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+const secretKey = process.env.WALLET_PRIVATE_KEY!;
+const CHAIN_ID = process.env.CHAIN_ID!;
+// TODO: read chain from RPC_URL
+
+const config: HardhatUserConfig = {
+  networks: {
+    hardhat: {
+      forking: {
+        url: RPC_URL,
+      },
+      chainId: Number(CHAIN_ID),
+      accounts: [
+        {
+          privateKey: secretKey,
+          balance: (BigInt(1e18) * BigInt(100)).toString(),
+        },
+      ],
+    },
+  },
+  solidity: '0.8.4',
+};
+
+export default config;

package.json

@@ -66,7 +66,9 @@
     "@nestjs/cli": "^8.2.5",
     "@nestjs/schematics": "^8.0.0",
     "@nestjs/testing": "^8.0.11",
+    "@nomicfoundation/hardhat-ethers": "^3.0.6",
     "@typechain/ethers-v5": "^7.1.2",
+    "@typechain/hardhat": "^9.1.0",
     "@types/app-root-path": "^1.2.4",
     "@types/cache-manager": "^3.4.3",
     "@types/glob": "^8.1.0",
@@ -77,19 +79,24 @@
     "@types/ws": "^8.5.3",
     "@typescript-eslint/eslint-plugin": "^4.28.2",
     "@typescript-eslint/parser": "^4.28.2",
+    "concurrently": "^9.0.1",
+    "docker-compose": "^1.1.0",
     "eslint": "^7.30.0",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-prettier": "^3.4.0",
     "ganache": "7.9.0",
+    "hardhat": "^2.22.14",
     "jest": "^27.0.6",
+    "pg": "^8.13.0",
     "prettier": "^2.3.2",
     "supertest": "^6.1.3",
     "ts-jest": "^27.0.3",
     "ts-loader": "^9.2.3",
     "ts-node": "^10.0.0",
     "tsconfig-paths": "^3.10.1",
     "typechain": "^5.1.2",
-    "typescript": "^4.3.5"
+    "typescript": "^4.3.5",
+    "wait-on": "^8.0.1"
   },
   "jest": {
     "moduleFileExtensions": [

run_e2e_setup.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+docker compose -f docker-compose.e2e-v3.yml up -d hardhat_node e2e_pgdb
+
+echo "Running cutKeys script via ts-node..."
+npx ts-node test/helpers/wait-node-cut-keys.ts &> keycutting.log &
+
+echo "Waiting for Key cutting to complete..."
+tail -f keycutting.log | while read LOG_LINE
+do
+   if [[ "${LOG_LINE}" == *"Key cutting completed."* ]]; then
+      echo "Key cutting process completed!"
+      pkill -P $$ tail
+   fi
+done
+
+echo "Key cutting completed. Starting e2e_keys_api..."
+docker compose -f docker-compose.e2e-v3.yml up -d e2e_keys_api
+
+echo "Waiting for e2e_keys_api to be ready..."
+until [ "$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/v1/modules)" == "200" ]; do
+  echo "e2e_keys_api is not ready yet. Retrying..."
+  sleep 5
+done
+
+echo "e2e_keys_api is ready."

src/guardian/guardian-message/guardian-message.service.ts

@@ -74,6 +74,10 @@ export class GuardianMessageService {
     });
   }
 
+  /**
+   * Sends an unvetting message to the message broker
+   * @param message - MessageUnvet object
+   */
   public sendUnvetMessage(message: Omit<MessageUnvet, 'type'>) {
     return this.sendMessageFromGuardian({
       ...message,

src/keys-api/interfaces/SRModuleKeyListResponse.ts

@@ -0,0 +1,13 @@
+import { SRModule } from '.';
+import { ELBlockSnapshot } from './ELBlockSnapshot';
+import { RegistryKey } from './RegistryKey';
+
+export type SRModuleKeyListResponse = {
+  data: {
+    keys: Array<RegistryKey>;
+    module: SRModule;
+  };
+  meta: {
+    elBlockSnapshot: ELBlockSnapshot;
+  };
+};

src/keys-api/keys-api.service.ts

@@ -8,6 +8,7 @@ import { Configuration } from 'common/config';
 import { GroupedByModuleOperatorListResponse } from './interfaces/GroupedByModuleOperatorListResponse';
 import { InconsistentLastChangedBlockHash } from 'common/custom-errors';
 import { SRModuleListResponse } from './interfaces/SRModuleListResponse';
+import { SRModuleKeyListResponse } from './interfaces/SRModuleKeyListResponse';
 
 @Injectable()
 export class KeysApiService {
@@ -93,6 +94,16 @@ export class KeysApiService {
     return result;
   }
 
+  /**
+   * The /v1/modules/{module_id}/keys endpoint returns full list of keys
+   */
+  public async getModuleKeys(moduleId: number, opId: number) {
+    const result = await this.fetch<SRModuleKeyListResponse>(
+      `/v1/modules/${moduleId}/keys?operatorIndex=${opId}`,
+    );
+    return result;
+  }
+
   /**
    * Verifies the consistency of metadata by comparing hashes.
    * @param firstRequestHash - Hash of the first request

src/staking-module-data-collector/staking-module-data-collector.service.ts

@@ -75,6 +75,8 @@ export class StakingModuleDataCollectorService {
         blockData,
       );
 
+    console.log('duplicates=', duplicates);
+
     await Promise.all(
       stakingModulesData.map(async (stakingModuleData) => {
         // identify keys that were front-run withing vetted unused keys
@@ -189,6 +191,8 @@ export class StakingModuleDataCollectorService {
       (key) =>
         !key.used && key.vetted && key.moduleAddress === stakingModuleAddress,
     );
+
+    console.log({ vettedUnusedKeys, stakingModuleAddress });
     return vettedUnusedKeys;
   }
 }

test/constants.ts

@@ -1,4 +1,3 @@
-import { CHAINS } from '@lido-sdk/constants';
 import { SecretKey } from '@chainsafe/blst';
 
 import { fromHexString } from '@chainsafe/ssz';
@@ -9,37 +8,10 @@ export const TESTS_TIMEOUT = 30_000;
 // Needs to be higher on gh actions for reliable runs
 export const SLEEP_FOR_RESULT = 3_000;
 
-// Addresses
-export const SECURITY_MODULE = '0x808DE3b26Be9438F12E9B45528955EA94C17f217';
-// https://holesky.etherscan.io/address/0x808DE3b26Be9438F12E9B45528955EA94C17f217#readContract
-// getOwner
-export const SECURITY_MODULE_OWNER =
-  '0xE92329EC7ddB11D25e25b3c21eeBf11f15eB325d';
-
-export const SECURITY_MODULE_V2 = '0x045dd46212a178428c088573a7d102b9d89a022a';
-// https://holesky.etherscan.io/address/0x045dd46212a178428c088573a7d102b9d89a022a#readContract
-// getOwner
-export const SECURITY_MODULE_OWNER_V2 =
-  '0xDA6bEE5441f2e6b364F3b25E85d5f3C29Bfb669E';
-export const STAKING_ROUTER = '0xd6EbF043D30A7fe46D1Db32BA90a0A51207FE229';
-export const NOP_REGISTRY = '0x595F64Ddc3856a3b5Ff4f4CC1d1fb4B46cFd2bAC';
-export const DEPOSIT_CONTRACT = '0x4242424242424242424242424242424242424242';
-export const SIMPLE_DVT = '0x11a93807078f8BB880c1BD0ee4C387537de4b4b6';
-export const CSM = '0x4562c3e63c2e586cD1651B958C22F88135aCAd4f';
-export const SANDBOX = '0xD6C2ce3BB8bea2832496Ac8b5144819719f343AC';
 // Withdrawal credentials
-export const LIDO_WC =
-  '0x010000000000000000000000f0179dec45a37423ead4fad5fcb136197872ead9';
 export const BAD_WC =
   '0x010000000000000000000000b9d7934878b5fb9610b3fe8a5e441e8fad7e291f';
 
-// Fork node config
-export const CHAIN_ID = CHAINS.Holesky;
-
-export const FORK_BLOCK = 1894357;
-export const FORK_BLOCK_V2 = 1817726;
-export const UNLOCKED_ACCOUNTS = [SECURITY_MODULE_OWNER];
-export const UNLOCKED_ACCOUNTS_V2 = [SECURITY_MODULE_OWNER_V2];
 export const GANACHE_PORT = 8545;
 
 // BLS key for the validator