What is Blockchain NodeOps Engineering? And How It Differs from Mainstream DevOps/SRE

# Ethereum Mainnet
mainnet_full_node:
  execution_client: ~900GB (Geth with PBSS)
  consensus_client: ~200GB
  sync_time: 24-48 hours (snap sync)

mainnet_archive_node:
  execution_client: ~15TB+ 
  consensus_client: ~200GB
  sync_time: 2-4 weeks (full sync)

# VeChainThor
thor_full_node:
  data_size: ~150GB
  sync_time: 8-12 hours

# Solana Validator
solana_validator:
  accounts_db: ~500GB
  ledger: ~2TB (with history)
  iops_required: 100,000+
  sync_time: 12-24 hours (with snapshot)

# This WILL get you slashed on any PoS network

# Node A (primary) - running validator with keys
lighthouse vc --validators-dir /keys \
  --beacon-nodes http://beacon:5052

# Node B (backup) - NEVER run simultaneously with same keys
lighthouse vc --validators-dir /keys \
  --beacon-nodes http://beacon-backup:5052

# Both nodes active = DOUBLE SIGNING = SLASHED

version: '3.8'
services:
  web3signer:
    image: consensys/web3signer:latest
    volumes:
      - ./keystore:/keystore:ro
    command: |
      eth2
      --network=mainnet
      --key-store-path=/keystore
      --slashing-protection-db-url=jdbc:postgresql://db:5432/slashing
      --slashing-protection-db-username=signer
      --slashing-protection-db-password=${DB_PASSWORD}
    networks:
      - internal

  lighthouse-vc:
    image: sigp/lighthouse:latest
    command: |
      lighthouse vc
      --network mainnet
      --beacon-nodes http://beacon:5052
      --web3-signer-url http://web3signer:9000
      --suggested-fee-recipient ${FEE_RECIPIENT}
    depends_on:
      - web3signer
    networks:
      - internal

  db:
    image: postgres:15
    volumes:
      - slashing_db:/var/lib/postgresql/data
    environment:
      POSTGRES_DB: slashing
      POSTGRES_USER: signer
      POSTGRES_PASSWORD: ${DB_PASSWORD}
    networks:
      - internal

networks:
  internal:
    driver: bridge
    internal: true  # No external access

volumes:
  slashing_db:

resource "aws_security_group" "blockchain_node" {
  name        = "blockchain-node-sg"
  description = "Security group for blockchain nodes"
  vpc_id      = var.vpc_id

  # Ethereum P2P (execution layer - devp2p)
  ingress {
    from_port   = 30303
    to_port     = 30303
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Ethereum execution P2P TCP"
  }
  ingress {
    from_port   = 30303
    to_port     = 30303
    protocol    = "udp"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Ethereum execution P2P UDP"
  }

  # Ethereum P2P (consensus layer - libp2p)
  ingress {
    from_port   = 9000
    to_port     = 9000
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Ethereum consensus P2P TCP"
  }
  ingress {
    from_port   = 9000
    to_port     = 9000
    protocol    = "udp"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Ethereum consensus P2P UDP"
  }

  # VeChainThor P2P
  ingress {
    from_port   = 11235
    to_port     = 11235
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
    description = "VeChainThor P2P"
  }

  # RPC (INTERNAL ONLY - never expose to public)
  ingress {
    from_port   = 8545
    to_port     = 8545
    protocol    = "tcp"
    cidr_blocks = [var.vpc_cidr]
    description = "JSON-RPC internal only"
  }

  tags = {
    Name = "blockchain-node-sg"
  }
}

#!/bin/bash
# Pre-fork upgrade procedure

# 1. Check client compatibility matrix
echo "Checking client versions..."
curl -s https://ethereum.github.io/consensus-specs/releases/

# 2. Update execution client (Geth)
docker pull ethereum/client-go:v1.13.14
docker run --rm ethereum/client-go:v1.13.14 version

# 3. Update consensus client (Lighthouse)
docker pull sigp/lighthouse:v5.1.0
lighthouse bn --network mainnet --version

# 4. Verify sync status BEFORE fork
SYNC_STATUS=$(curl -s http://localhost:5052/eth/v1/node/syncing)
SYNC_DISTANCE=$(echo $SYNC_STATUS | jq -r '.data.sync_distance')

if [ "$SYNC_DISTANCE" != "0" ]; then
  echo "WARNING: Node not synced! Distance: $SYNC_DISTANCE"
  exit 1
fi

echo "Node synced and ready for fork"

# 5. Monitor through fork
watch -n 1 'curl -s localhost:5052/eth/v1/beacon/headers/head | \
  jq ".data.header.message.slot"'

groups:
  - name: validator_critical
    rules:
      # Node sync lag detection
      - alert: NodeSyncLag
        expr: (beacon_head_slot - beacon_node_slot) > 32
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Node {{ $labels.instance }} is {{ $value }} slots behind"
          runbook: "Check peer connectivity and disk I/O"

      # Missed attestations (validator income loss)
      - alert: MissedAttestations
        expr: increase(validator_missed_attestations_total[1h]) > 5
        labels:
          severity: warning
        annotations:
          summary: "Validator missed {{ $value }} attestations"

      # Missed block proposal (rare but costly)
      - alert: MissedProposal
        expr: increase(validator_proposals_missed_total[1h]) > 0
        labels:
          severity: critical
        annotations:
          summary: "MISSED BLOCK PROPOSAL - immediate investigation required"

      # Slashing detection (should NEVER fire)
      - alert: SlashingDetected
        expr: validator_slashed == 1
        labels:
          severity: page
        annotations:
          summary: "🚨 VALIDATOR SLASHED - IMMEDIATE ACTION REQUIRED"

      # VeChainThor specific: block production
      - alert: ThorNoNewBlocks
        expr: increase(thor_chain_head_number[10m]) == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "No new blocks on Thor node {{ $labels.instance }}"

      # Peer count monitoring
      - alert: LowPeerCount
        expr: eth_p2p_peers < 10 OR thor_p2p_peers < 5
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Low peer count: {{ $value }} peers"

#!/bin/bash
# health_check.sh - Validates RPC is synced, not just responding

LOCAL_SLOT=$(curl -s localhost:8899 -X POST \
  -H "Content-Type: application/json" \
  -d '{"jsonrpc":"2.0","id":1,"method":"getSlot"}' | jq .result)

CLUSTER_SLOT=$(curl -s https://api.mainnet-beta.solana.com -X POST \
  -H "Content-Type: application/json" \
  -d '{"jsonrpc":"2.0","id":1,"method":"getSlot"}' | jq .result)

DIFF=$((CLUSTER_SLOT - LOCAL_SLOT))

if [ $DIFF -gt 50 ]; then
  echo "UNHEALTHY: $DIFF slots behind cluster"
  exit 1
fi

echo "HEALTHY: $DIFF slots behind (within tolerance)"
exit 0

#!/bin/bash
# Emergency resync procedure
# Time estimates: Full node ~24h, Archive node ~2-3 weeks

# 1. Stop clients gracefully
docker-compose down

# 2. Backup current state (if potentially recoverable)
rsync -av /data/geth /backup/geth-$(date +%Y%m%d)

# 3. Clear corrupted state
rm -rf /data/geth/chaindata

# 4. Restart with checkpoint sync (if supported)
geth --syncmode snap \
  --checkpoint-url https://checkpoint.eth.io

# 5. Monitor sync progress
watch -n 10 'geth attach --exec "eth.syncing"'