diff --git a/.gitignore b/.gitignore index 0ab871c3..32bba8a1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ tmp/ +.idea/ test params.yml +*.DS_Store diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index 8e26f86b..d8ff00c3 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -128,7 +128,7 @@ Default: a single validator. | kind | string | validator | Role of the node in the network: `validator`, `rpc` or `archive` | | cl_type | string | heimdall-v2 | Consensus Layer (CL) client type | | cl_image | string | 0xpolygon/heimdall-v2:0.6.0 | Image for the CL client | -| cl_queue_image | string | rabbitmq:4.2.5 | Image for the CL queue | +| cl_queue_image | string | rabbitmq:4.2.4 | Image for the CL queue | | cl_log_level | string | info | Log level for the CL client | | cl_log_format | string | text | Log format for the CL client | | cl_min_retain_blocks | int | 0 | Minimal distance from current height to retain height | @@ -137,7 +137,7 @@ Default: a single validator. | cl_storage_pruning_interval | string | 10m0s | Interval between prune routines. | | cl_indexer_pruning_enabled | bool | false | Pruning enabling. | | el_type | string | bor | Execution Layer (EL) client type: `bor` or `erigon` | -| el_image | string | 0xpolygon/bor:2.7.1 | Image for the EL client (bor: `0xpolygon/bor:2.7.1`, erigon: `0xpolygon/erigon:v3.5.0`) | +| el_image | string | 0xpolygon/bor:2.7.0 | Image for the EL client (bor: `0xpolygon/bor:2.7.0`, erigon: `0xpolygon/erigon:v3.5.0`) | | el_log_level | string | info | Log level for the EL client | | el_log_format | string | text | Log format for the EL client | | el_bor_produce_witness | bool | false | Allow bor to start producing witnesses | @@ -147,9 +147,9 @@ Default: a single validator. ### `setup_images` -| Field | Type | Default | Description | +| Field | Type | Default | Description | | -------------------------- | ------ | ------------------------------------------------------ | ------------------------------------------- | -| contract_deployer | string | ghcr.io/0xpolygon/pos-contract-deployer:f1c2f5d1 | Image used to deploy MATIC contracts to L1 | +| contract_deployer | string | ghcr.io/0xpolygon/pos-contract-deployer:d96d592 | Image used to deploy MATIC contracts to L1 | | el_genesis_builder | string | ghcr.io/0xpolygon/pos-el-genesis-builder:96a19dd | Image used to create the L2 EL genesis file | | validator_config_generator | string | ghcr.io/0xpolygon/pos-validator-config-generator:0.6.0 | Image used to generate validator configs | @@ -182,13 +182,13 @@ You can check the admin private key and mnemonic default values at `src/config/i | napoli_fork_block | int | 0 | Block number for Napoli hard fork activation | | ahmedabad_fork_block | int | 0 | Block number for Ahmedabad hard fork activation | | bhilai_fork_block | int | 0 | Block number for Bhilai hard fork activation | -| rio_fork_block | int | 256 | Block number for Rio hard fork activation | -| madhugiri_fork_block | int | 256 | Block number for Madhugiri hard fork activation | -| madhugiri_pro_fork_block | int | 256 | Block number for Madhugiri Pro hard fork activation | -| dandeli_fork_block | int | 256 | Block number for Dandeli hard fork activation | -| lisovo_fork_block | int | 256 | Block number for Lisovo hard fork activation | -| lisovo_pro_fork_block | int | 256 | Block number for Lisovo Pro hard fork activation | -| giugliano_fork_block | int | 256 | Block number for Giugliano hard fork activation | +| rio_fork_block | int | 128 | Block number for Rio hard fork activation | +| madhugiri_fork_block | int | 128 | Block number for Madhugiri hard fork activation | +| madhugiri_pro_fork_block | int | 128 | Block number for Madhugiri Pro hard fork activation | +| dandeli_fork_block | int | 128 | Block number for Dandeli hard fork activation | +| lisovo_fork_block | int | 128 | Block number for Lisovo hard fork activation | +| lisovo_pro_fork_block | int | 128 | Block number for Lisovo Pro hard fork activation | +| giugliano_fork_block | int | 128 | Block number for Giugliano hard fork activation | ### `additional_services` diff --git a/scripts/snapshot/restore.sh b/scripts/snapshot/restore.sh index 75aadb78..49093aed 100755 --- a/scripts/snapshot/restore.sh +++ b/scripts/snapshot/restore.sh @@ -12,37 +12,37 @@ source "${SCRIPT_DIR}/log.sh" ############################################################################ restore_docker_volumes() { - local volume_folder_path="$1" + local volume_folder_path="$1" - # Restore from archives (.tar.gz) - for v in "$volume_folder_path"/*.tar.gz; do - [[ -f "$v" ]] || continue - ( - volume_name=$(basename "$v" .tar.gz | sed 's/_/--/g') - echo "$volume_name" - docker volume create "$volume_name" >/dev/null - docker run --rm \ - -v "$volume_name":/data \ - -v "$volume_folder_path":/backup \ - alpine tar xzf /backup/$(basename "$v") -C /data - ) & - done + # Restore from archives (.tar.gz) + for v in "$volume_folder_path"/*.tar.gz; do + [[ -f "$v" ]] || continue + ( + volume_name=$(basename "$v" .tar.gz | sed 's/_/--/g') + echo "$volume_name" + docker volume create "$volume_name" > /dev/null + docker run --rm \ + -v "$volume_name":/data \ + -v "$volume_folder_path":/backup \ + alpine tar xzf /backup/$(basename "$v") -C /data + ) & + done - # Restore from directories - for d in "$volume_folder_path"/*/; do - [[ -d "$d" ]] || continue - ( - volume_name=$(basename "$d" | sed 's/_/--/g') - echo "$volume_name" - docker volume create "$volume_name" >/dev/null - docker run --rm \ - -v "$volume_name":/data \ - -v "$(realpath "$d")":/backup \ - alpine sh -c "cd /backup && tar cf - . | tar xf - -C /data" - ) & - done + # Restore from directories + for d in "$volume_folder_path"/*/; do + [[ -d "$d" ]] || continue + ( + volume_name=$(basename "$d" | sed 's/_/--/g') + echo "$volume_name" + docker volume create "$volume_name" > /dev/null + docker run --rm \ + -v "$volume_name":/data \ + -v "$(realpath "$d")":/backup \ + alpine sh -c "cd /backup && tar cf - . | tar xf - -C /data" + ) & + done - wait + wait } ############################################################################## diff --git a/scripts/snapshot/snapshot.sh b/scripts/snapshot/snapshot.sh index 5eba85fc..d4eede97 100755 --- a/scripts/snapshot/snapshot.sh +++ b/scripts/snapshot/snapshot.sh @@ -15,45 +15,45 @@ source "${SCRIPT_DIR}/log.sh" # Polls all L2 EL RPC endpoints until every node has reached the target block. wait_for_rpcs_to_reach_block() { - local enclave_name="$1" - local target_block="$2" - - # Get all L2 EL service names (strip UUID suffix) - mapfile -t l2_el_services < <(docker ps --filter "network=kt-$enclave_name" --format "{{.Names}}" \ - | grep l2-el | sort | awk -F'--' '{print $1}') - - # Resolve RPC URLs once — they don't change between checks - declare -A rpc_urls + local enclave_name="$1" + local target_block="$2" + + # Get all L2 EL service names (strip UUID suffix) + mapfile -t l2_el_services < <(docker ps --filter "network=kt-$enclave_name" --format "{{.Names}}" | + grep l2-el | sort | awk -F'--' '{print $1}') + + # Resolve RPC URLs once — they don't change between checks + declare -A rpc_urls + for service in "${l2_el_services[@]}"; do + rpc_urls[$service]=$(kurtosis port print "$enclave_name" "$service" rpc) + done + + # Poll each service until all have reached the target block + local num_steps=50 + for step in $(seq 1 "$num_steps"); do + log_info "Check ${step}/${num_steps}" + local all_ready=true for service in "${l2_el_services[@]}"; do - rpc_urls[$service]=$(kurtosis port print "$enclave_name" "$service" rpc) + local block_number status + block_number=$(cast bn --rpc-url "${rpc_urls[$service]}") + if [[ "$block_number" -lt "$target_block" ]]; then + status="NOT READY" + all_ready=false + else + status="OK" + fi + log_info "- $service: $block_number/$target_block - $status" done - # Poll each service until all have reached the target block - local num_steps=50 - for step in $(seq 1 "$num_steps"); do - log_info "Check ${step}/${num_steps}" - local all_ready=true - for service in "${l2_el_services[@]}"; do - local block_number status - block_number=$(cast bn --rpc-url "${rpc_urls[$service]}") - if [[ "$block_number" -lt "$target_block" ]]; then - status="NOT READY" - all_ready=false - else - status="OK" - fi - log_info "- $service: $block_number/$target_block - $status" - done - - if $all_ready; then - log_info "All L2 RPCs reached block $target_block" - return 0 - fi - log_info "Not all L2 RPCs reached block $target_block, retrying in 10s..." - sleep 10 - done - log_error "Not all L2 RPCs reached block $target_block after $num_steps steps" - return 1 + if $all_ready; then + log_info "All L2 RPCs reached block $target_block" + return 0 + fi + log_info "Not all L2 RPCs reached block $target_block, retrying in 10s..." + sleep 10 + done + log_error "Not all L2 RPCs reached block $target_block after $num_steps steps" + return 1 } ############################################################################## @@ -61,77 +61,77 @@ wait_for_rpcs_to_reach_block() { ############################################################################## backup_docker_volumes() { - local enclave_name="$1" - local volume_folder_path="$2" - - enclave_uuid=$(kurtosis enclave inspect "$enclave_name" --full-uuids | awk '/^UUID:/ {print $2}') - mkdir -p "$volume_folder_path" - - # Build mapping: original_volume_name -> sanitized_volume_name - declare -A volume_mapping - local temp_mounts=$(mktemp) - trap "rm -f '$temp_mounts'" EXIT - - # Get all containers (including stopped ones) for this enclave - mapfile -t CONTAINERS < <(docker ps --all --filter "network=kt-$enclave_name" --format "{{.Names}}" \ - | grep -Ev "kurtosis|files-artifacts-expander|validator-key-generation|read|run|reader|deriver|deployer|generator|monitor") - - # Extract volume mounts from all containers - for container in "${CONTAINERS[@]}"; do - # Get container's clean service name (remove UUID suffix) - clean_service=$(echo "$container" | sed 's/--[a-f0-9]\{32\}$//') - - # Get all volume mounts for this container - docker inspect "$container" --format '{{json .Mounts}}' | jq -r '.[] | select(.Type == "volume") | "\(.Name)|\(.Destination)"' | while IFS='|' read -r volume_name mount_path; do - # Determine sanitized volume name based on volume type - if [[ "$volume_name" == data-* ]]; then - # L1 pattern: data-X-Y-Z--UUID -> enclave-X-Y-Z-data - sanitized_name="${enclave_name}-${clean_service}-data" - elif [[ "$volume_name" =~ -data--[a-f0-9]{32}$ ]]; then - # L2 pattern: X-Y-Z-data--UUID -> enclave-X-Y-Z-data - # Remove the UUID suffix to get the base name - base_name=$(echo "$volume_name" | sed 's/--[a-f0-9]\{32\}$//') - sanitized_name="${enclave_name}-${base_name}" - elif [[ "$volume_name" == files-artifact-expansion-* ]]; then - # files-artifact-expansion--UUID1--UUID2 -> enclave-service-mountpath - clean_mount=$(echo "$mount_path" | sed 's|^/||' | sed 's|/|-|g') - sanitized_name="${enclave_name}-${clean_service}-${clean_mount}" - else - # Fallback: remove UUID and prefix with enclave name - base_name=$(echo "$volume_name" | sed 's/--[a-f0-9]\{32\}$//') - sanitized_name="${enclave_name}-${base_name}" - fi - - # Store mapping - echo "${volume_name}|${sanitized_name}" >> "$temp_mounts" - done - done - - # Load mappings into associative array - while IFS='|' read -r original sanitized; do - volume_mapping["$original"]="$sanitized" - done < "$temp_mounts" - rm -f "$temp_mounts" - - # Backup volumes using sanitized names - for v in "${!volume_mapping[@]}"; do - ( - sanitized_v="${volume_mapping[$v]}" - echo "$sanitized_v" - backup_file="$volume_folder_path/$(echo "$sanitized_v" | sed 's/--/_/g').tar.gz" - docker run --rm \ - -v "$v":/data:ro \ - -v "$volume_folder_path":/backup \ - alpine tar czf /backup/$(basename "$backup_file") -C /data . - ) & + local enclave_name="$1" + local volume_folder_path="$2" + + enclave_uuid=$(kurtosis enclave inspect "$enclave_name" --full-uuids | awk '/^UUID:/ {print $2}') + mkdir -p "$volume_folder_path" + + # Build mapping: original_volume_name -> sanitized_volume_name + declare -A volume_mapping + local temp_mounts=$(mktemp) + trap "rm -f '$temp_mounts'" EXIT + + # Get all containers (including stopped ones) for this enclave + mapfile -t CONTAINERS < <(docker ps --all --filter "network=kt-$enclave_name" --format "{{.Names}}" | + grep -Ev "kurtosis|files-artifacts-expander|validator-key-generation|read|run|reader|deriver|deployer|generator|monitor") + + # Extract volume mounts from all containers + for container in "${CONTAINERS[@]}"; do + # Get container's clean service name (remove UUID suffix) + clean_service=$(echo "$container" | sed 's/--[a-f0-9]\{32\}$//') + + # Get all volume mounts for this container + docker inspect "$container" --format '{{json .Mounts}}' | jq -r '.[] | select(.Type == "volume") | "\(.Name)|\(.Destination)"' | while IFS='|' read -r volume_name mount_path; do + # Determine sanitized volume name based on volume type + if [[ "$volume_name" == data-* ]]; then + # L1 pattern: data-X-Y-Z--UUID -> enclave-X-Y-Z-data + sanitized_name="${enclave_name}-${clean_service}-data" + elif [[ "$volume_name" =~ -data--[a-f0-9]{32}$ ]]; then + # L2 pattern: X-Y-Z-data--UUID -> enclave-X-Y-Z-data + # Remove the UUID suffix to get the base name + base_name=$(echo "$volume_name" | sed 's/--[a-f0-9]\{32\}$//') + sanitized_name="${enclave_name}-${base_name}" + elif [[ "$volume_name" == files-artifact-expansion-* ]]; then + # files-artifact-expansion--UUID1--UUID2 -> enclave-service-mountpath + clean_mount=$(echo "$mount_path" | sed 's|^/||' | sed 's|/|-|g') + sanitized_name="${enclave_name}-${clean_service}-${clean_mount}" + else + # Fallback: remove UUID and prefix with enclave name + base_name=$(echo "$volume_name" | sed 's/--[a-f0-9]\{32\}$//') + sanitized_name="${enclave_name}-${base_name}" + fi + + # Store mapping + echo "${volume_name}|${sanitized_name}" >> "$temp_mounts" done - wait - - # The alpine container runs as root, so the tar.gz files are root-owned. - # Fix ownership so they can be cleaned up without sudo. - docker run --rm \ + done + + # Load mappings into associative array + while IFS='|' read -r original sanitized; do + volume_mapping["$original"]="$sanitized" + done < "$temp_mounts" + rm -f "$temp_mounts" + + # Backup volumes using sanitized names + for v in "${!volume_mapping[@]}"; do + ( + sanitized_v="${volume_mapping[$v]}" + echo "$sanitized_v" + backup_file="$volume_folder_path/$(echo "$sanitized_v" | sed 's/--/_/g').tar.gz" + docker run --rm \ + -v "$v":/data:ro \ -v "$volume_folder_path":/backup \ - alpine chown -R "$(id -u):$(id -g)" /backup + alpine tar czf /backup/$(basename "$backup_file") -C /data . + ) & + done + wait + + # The alpine container runs as root, so the tar.gz files are root-owned. + # Fix ownership so they can be cleaned up without sudo. + docker run --rm \ + -v "$volume_folder_path":/backup \ + alpine chown -R "$(id -u):$(id -g)" /backup } ############################################################################## @@ -139,47 +139,47 @@ backup_docker_volumes() { ############################################################################## generate_docker_compose() { - local enclave_name="$1" - local docker_compose_file="$2" - - # Store container names in an array - mapfile -t CONTAINERS < <(docker ps --all --filter "network=kt-$enclave_name" --format "{{.Names}}" \ - | grep -Ev "kurtosis|files-artifacts-expander|validator-key-generation|read|run|reader|deriver|deployer|generator|monitor") - - # Generate docker-compose using docker-autocompose. - # Use array expansion to pass each element as a separate argument - # Otherwise bash performs word splitting on the result, and since each container name is on its own line - # (separated by newlines), each becomes a separate argument - docker run --rm \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - ghcr.io/red5d/docker-autocompose@sha256:4d878dbda37b77ec6babea06d398095ff46054b06ee507abbb3f1e71eef2e868 \ - "${CONTAINERS[@]}" > "$docker_compose_file" + local enclave_name="$1" + local docker_compose_file="$2" + + # Store container names in an array + mapfile -t CONTAINERS < <(docker ps --all --filter "network=kt-$enclave_name" --format "{{.Names}}" | + grep -Ev "kurtosis|files-artifacts-expander|validator-key-generation|read|run|reader|deriver|deployer|generator|monitor") + + # Generate docker-compose using docker-autocompose. + # Use array expansion to pass each element as a separate argument + # Otherwise bash performs word splitting on the result, and since each container name is on its own line + # (separated by newlines), each becomes a separate argument + docker run --rm \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + ghcr.io/red5d/docker-autocompose@sha256:4d878dbda37b77ec6babea06d398095ff46054b06ee507abbb3f1e71eef2e868 \ + "${CONTAINERS[@]}" > "$docker_compose_file" } configure_networks() { - local docker_compose_file="$1" + local docker_compose_file="$1" - # Replace networks section with single network named after enclave - yq --in-place --yaml-output \ - --arg new_network "$enclave_name" \ - '.networks = {($new_network): {name: $new_network}}' "$docker_compose_file" + # Replace networks section with single network named after enclave + yq --in-place --yaml-output \ + --arg new_network "$enclave_name" \ + '.networks = {($new_network): {name: $new_network}}' "$docker_compose_file" - # Update all service network references to use enclave name - yq --in-place --yaml-output \ - --arg new_network "$enclave_name" \ - '.services[].networks = [$new_network]' "$docker_compose_file" + # Update all service network references to use enclave name + yq --in-place --yaml-output \ + --arg new_network "$enclave_name" \ + '.services[].networks = [$new_network]' "$docker_compose_file" - # Remove unnecessary fields from all services - # TODO: Check if we can remove hostnames? - yq --in-place --yaml-output 'del(.version, .services[].ports, .services[].labels, .services[].logging, .services[].stdin_open, .services[].ipc)' "$docker_compose_file" + # Remove unnecessary fields from all services + # TODO: Check if we can remove hostnames? + yq --in-place --yaml-output 'del(.version, .services[].ports, .services[].labels, .services[].logging, .services[].stdin_open, .services[].ipc)' "$docker_compose_file" } sanitize_service_names() { - local docker_compose_file="$1" + local docker_compose_file="$1" - # Remove UUID suffixes from service names and add enclave prefix - yq --in-place --yaml-output \ - --arg enclave_prefix "${enclave_name}-" ' + # Remove UUID suffixes from service names and add enclave prefix + yq --in-place --yaml-output \ + --arg enclave_prefix "${enclave_name}-" ' .services |= ( to_entries | map(.key |= ($enclave_prefix + (. | sub("--[a-f0-9]{32}$"; "")))) | @@ -187,19 +187,19 @@ sanitize_service_names() { ) ' "$docker_compose_file" - # Update container_name fields: remove UUID suffix and add enclave prefix - yq --in-place --yaml-output \ - --arg enclave_prefix "${enclave_name}-" ' + # Update container_name fields: remove UUID suffix and add enclave prefix + yq --in-place --yaml-output \ + --arg enclave_prefix "${enclave_name}-" ' .services[].container_name |= ($enclave_prefix + (. | sub("--[a-f0-9]{32}$"; ""))) ' "$docker_compose_file" } sanitize_volume_names() { - local docker_compose_file="$1" + local docker_compose_file="$1" - # Remove UUID suffixes from top-level volume names and add enclave prefix - yq --in-place --yaml-output \ - --arg enclave_prefix "${enclave_name}-" ' + # Remove UUID suffixes from top-level volume names and add enclave prefix + yq --in-place --yaml-output \ + --arg enclave_prefix "${enclave_name}-" ' .volumes |= ( to_entries | map(.key |= ($enclave_prefix + (. | sub("--[a-f0-9]{32}$"; "")))) | @@ -207,9 +207,9 @@ sanitize_volume_names() { ) ' "$docker_compose_file" - # Update volume references in service volume mounts - yq --in-place --yaml-output \ - --arg enclave_prefix "${enclave_name}-" ' + # Update volume references in service volume mounts + yq --in-place --yaml-output \ + --arg enclave_prefix "${enclave_name}-" ' .services[].volumes[]? |= ( if contains(":") then (split(":") | .[0] |= ($enclave_prefix + (. | sub("--[a-f0-9]{32}$"; "")))) | join(":") @@ -221,64 +221,64 @@ sanitize_volume_names() { ) ' "$docker_compose_file" - # Rename file-artifact volumes based on service name and mount path - rename_file_artifact_volumes "$docker_compose_file" + # Rename file-artifact volumes based on service name and mount path + rename_file_artifact_volumes "$docker_compose_file" - # Rename L1 data volumes to move -data suffix to end - rename_data_volumes "$docker_compose_file" + # Rename L1 data volumes to move -data suffix to end + rename_data_volumes "$docker_compose_file" - # Sort volumes alphabetically - yq --in-place --yaml-output '.volumes = (.volumes | to_entries | sort_by(.key) | from_entries)' "$docker_compose_file" + # Sort volumes alphabetically + yq --in-place --yaml-output '.volumes = (.volumes | to_entries | sort_by(.key) | from_entries)' "$docker_compose_file" } rename_file_artifact_volumes() { - local docker_compose_file="$1" - - mapping_file=$(mktemp) - yq '.services | to_entries[] | .key as $service | .value.volumes[]? | select(. | type == "string" and contains("files-artifact-expansion")) | split(":") | "\(.[0])|\($service)|\(.[1])"' "$docker_compose_file" | tr -d '"' > "$mapping_file" - - declare -A volume_mapping - while IFS='|' read -r old_volume service mount_path; do - if [[ -n "${volume_mapping[$old_volume]:-}" ]]; then - continue - fi - clean_path=$(echo "$mount_path" | sed 's|^/||' | sed 's|/|-|g') - new_volume="${service}-${clean_path}" - volume_mapping["$old_volume"]="$new_volume" - done < "$mapping_file" - - for old_vol in "${!volume_mapping[@]}"; do - new_vol="${volume_mapping[$old_vol]}" - yq --in-place --yaml-output \ - --arg old "$old_vol" \ - --arg new "$new_vol" ' + local docker_compose_file="$1" + + mapping_file=$(mktemp) + yq '.services | to_entries[] | .key as $service | .value.volumes[]? | select(. | type == "string" and contains("files-artifact-expansion")) | split(":") | "\(.[0])|\($service)|\(.[1])"' "$docker_compose_file" | tr -d '"' > "$mapping_file" + + declare -A volume_mapping + while IFS='|' read -r old_volume service mount_path; do + if [[ -n "${volume_mapping[$old_volume]:-}" ]]; then + continue + fi + clean_path=$(echo "$mount_path" | sed 's|^/||' | sed 's|/|-|g') + new_volume="${service}-${clean_path}" + volume_mapping["$old_volume"]="$new_volume" + done < "$mapping_file" + + for old_vol in "${!volume_mapping[@]}"; do + new_vol="${volume_mapping[$old_vol]}" + yq --in-place --yaml-output \ + --arg old "$old_vol" \ + --arg new "$new_vol" ' .volumes = (.volumes | to_entries | map(if .key == $old then {key: $new, value: .value} else . end) | from_entries) ' "$docker_compose_file" - sed -i "s|${old_vol}:|${new_vol}:|g" "$docker_compose_file" - done - rm -f "$mapping_file" + sed -i "s|${old_vol}:|${new_vol}:|g" "$docker_compose_file" + done + rm -f "$mapping_file" } rename_data_volumes() { - local docker_compose_file="$1" + local docker_compose_file="$1" - for old_vol in $(yq '.volumes | keys | .[] | select(startswith("'${enclave_name}'-data-"))' "$docker_compose_file" | tr -d '"'); do - new_vol=$(echo "$old_vol" | sed "s/^${enclave_name}-data-/${enclave_name}-/" | sed 's/$/-data/') - yq --in-place --yaml-output \ - --arg old "$old_vol" \ - --arg new "$new_vol" ' + for old_vol in $(yq '.volumes | keys | .[] | select(startswith("'${enclave_name}'-data-"))' "$docker_compose_file" | tr -d '"'); do + new_vol=$(echo "$old_vol" | sed "s/^${enclave_name}-data-/${enclave_name}-/" | sed 's/$/-data/') + yq --in-place --yaml-output \ + --arg old "$old_vol" \ + --arg new "$new_vol" ' .volumes = (.volumes | to_entries | map(if .key == $old then {key: $new, value: .value} else . end) | from_entries) ' "$docker_compose_file" - sed -i "s|${old_vol}:|${new_vol}:|g" "$docker_compose_file" - done + sed -i "s|${old_vol}:|${new_vol}:|g" "$docker_compose_file" + done } add_network_aliases() { - local docker_compose_file="$1" + local docker_compose_file="$1" - yq --in-place --yaml-output \ - --arg new_network "$enclave_name" \ - --arg enclave_prefix "${enclave_name}-" ' + yq --in-place --yaml-output \ + --arg new_network "$enclave_name" \ + --arg enclave_prefix "${enclave_name}-" ' .services |= with_entries( .value.networks = { ($new_network): { @@ -290,11 +290,11 @@ add_network_aliases() { } configure_service_dependencies() { - local docker_compose_file="$1" + local docker_compose_file="$1" - # L2: CL depends on RabbitMQ with matching index - yq --in-place --yaml-output \ - --arg enclave_name "$enclave_name" ' + # L2: CL depends on RabbitMQ with matching index + yq --in-place --yaml-output \ + --arg enclave_name "$enclave_name" ' .services |= with_entries( if (.key | test("^" + $enclave_name + "-l2-cl-[0-9]+-")) and (.key | test("rabbitmq") | not) then (.key | capture("^" + $enclave_name + "-l2-cl-(?[0-9]+)-")) as $match | @@ -303,9 +303,9 @@ configure_service_dependencies() { ) ' "$docker_compose_file" - # L2: EL depends on CL with matching index - yq --in-place --yaml-output \ - --arg enclave_name "$enclave_name" ' + # L2: EL depends on CL with matching index + yq --in-place --yaml-output \ + --arg enclave_name "$enclave_name" ' (.services | keys) as $all_services | .services |= with_entries( if .key | test("^" + $enclave_name + "-l2-el-[0-9]+-") then @@ -318,11 +318,11 @@ configure_service_dependencies() { } add_health_checks() { - local docker_compose_file="$1" + local docker_compose_file="$1" - # RabbitMQ health check - yq --in-place --yaml-output \ - --arg enclave_name "$enclave_name" ' + # RabbitMQ health check + yq --in-place --yaml-output \ + --arg enclave_name "$enclave_name" ' .services |= with_entries( if .key | test("^" + $enclave_name + "-l2-cl-[0-9]+-rabbitmq") then .value.healthcheck = { @@ -336,9 +336,9 @@ add_health_checks() { ) ' "$docker_compose_file" - # Heimdall health check - yq --in-place --yaml-output \ - --arg enclave_name "$enclave_name" ' + # Heimdall health check + yq --in-place --yaml-output \ + --arg enclave_name "$enclave_name" ' .services |= with_entries( if .key | test("^" + $enclave_name + "-l2-cl-[0-9]+-heimdall") then .value.healthcheck = { @@ -354,11 +354,11 @@ add_health_checks() { } configure_ports() { - local docker_compose_file="$1" + local docker_compose_file="$1" - # L1 CL (consensus layer) - beacon API on port 4000 + (index - 1) - yq --in-place --yaml-output \ - --arg enclave_name "$enclave_name" ' + # L1 CL (consensus layer) - beacon API on port 4000 + (index - 1) + yq --in-place --yaml-output \ + --arg enclave_name "$enclave_name" ' .services |= with_entries( if (.key | test("^" + $enclave_name + "-cl-[0-9]+-")) and (.key | test("-l2-") | not) then (.key | capture("^" + $enclave_name + "-cl-(?[0-9]+)-")) as $match | @@ -367,9 +367,9 @@ configure_ports() { ) ' "$docker_compose_file" - # L1 EL (execution layer) - JSON-RPC on port 8545 + (index - 1) - yq --in-place --yaml-output \ - --arg enclave_name "$enclave_name" ' + # L1 EL (execution layer) - JSON-RPC on port 8545 + (index - 1) + yq --in-place --yaml-output \ + --arg enclave_name "$enclave_name" ' .services |= with_entries( if (.key | test("^" + $enclave_name + "-el-[0-9]+-")) and (.key | test("-l2-") | not) then (.key | capture("^" + $enclave_name + "-el-(?[0-9]+)-")) as $match | @@ -378,9 +378,9 @@ configure_ports() { ) ' "$docker_compose_file" - # L2 CL (consensus layer) - REST API on port 1317 + (index - 1) - yq --in-place --yaml-output \ - --arg enclave_name "$enclave_name" ' + # L2 CL (consensus layer) - REST API on port 1317 + (index - 1) + yq --in-place --yaml-output \ + --arg enclave_name "$enclave_name" ' .services |= with_entries( if (.key | test("^" + $enclave_name + "-l2-cl-[0-9]+-")) and (.key | test("rabbitmq") | not) then (.key | capture("^" + $enclave_name + "-l2-cl-(?[0-9]+)-")) as $match | @@ -389,9 +389,9 @@ configure_ports() { ) ' "$docker_compose_file" - # L2 EL (execution layer) - JSON-RPC on port 9545 + (index - 1) - yq --in-place --yaml-output \ - --arg enclave_name "$enclave_name" ' + # L2 EL (execution layer) - JSON-RPC on port 9545 + (index - 1) + yq --in-place --yaml-output \ + --arg enclave_name "$enclave_name" ' .services |= with_entries( if .key | test("^" + $enclave_name + "-l2-el-[0-9]+-") then (.key | capture("^" + $enclave_name + "-l2-el-(?[0-9]+)-")) as $match | @@ -402,14 +402,14 @@ configure_ports() { } sanitize_docker_compose() { - local docker_compose_file="$1" - configure_networks "$docker_compose_file" - sanitize_service_names "$docker_compose_file" - sanitize_volume_names "$docker_compose_file" - add_network_aliases "$docker_compose_file" - configure_service_dependencies "$docker_compose_file" - add_health_checks "$docker_compose_file" - configure_ports "$docker_compose_file" + local docker_compose_file="$1" + configure_networks "$docker_compose_file" + sanitize_service_names "$docker_compose_file" + sanitize_volume_names "$docker_compose_file" + add_network_aliases "$docker_compose_file" + configure_service_dependencies "$docker_compose_file" + add_health_checks "$docker_compose_file" + configure_ports "$docker_compose_file" } ############################################################################## @@ -419,8 +419,8 @@ sanitize_docker_compose() { default_enclave_name="pos" enclave_name=${1:-"$default_enclave_name"} if [[ -z "$enclave_name" ]]; then - log_error "Enclave name cannot be empty" - exit 1 + log_error "Enclave name cannot be empty" + exit 1 fi log_info "Using enclave name: $enclave_name" @@ -451,7 +451,7 @@ log_info "Docker-compose generated and sanitized" log_info "Packaging snapshot into docker image" # Create a Dockerfile pushd "$tmp_dir" -cat > "Dockerfile" <<'EOF' +cat > "Dockerfile" << 'EOF' FROM scratch COPY volumes/*.tar.gz /volumes/ COPY docker-compose.yaml /docker-compose.yaml diff --git a/src/cl/heimdall_v2/launcher.star b/src/cl/heimdall_v2/launcher.star index 97483459..5c05e181 100644 --- a/src/cl/heimdall_v2/launcher.star +++ b/src/cl/heimdall_v2/launcher.star @@ -16,6 +16,7 @@ def launch( cl_node_ids, l1_rpc_url, el_rpc_url, + el_grpc_url, amqp_url, container_proc_manager_artifact, ): @@ -38,7 +39,9 @@ def launch( # URLs. "amqp_url": amqp_url, "el_rpc_url": el_rpc_url, + "el_grpc_url": el_grpc_url, "l1_rpc_url": l1_rpc_url, + "cl_bor_grpc_flag": participant.get("cl_bor_grpc_flag"), # Port numbers. "rest_api_port_number": shared.REST_API_PORT_NUMBER, "grpc_port_number": shared.GRPC_PORT_NUMBER, diff --git a/src/cl/launcher.star b/src/cl/launcher.star index 1c92864d..08d80d3e 100644 --- a/src/cl/launcher.star +++ b/src/cl/launcher.star @@ -40,6 +40,7 @@ def launch( el_node_name = el_launcher.generate_name(participant, id) el_rpc_url = "http://{}:{}".format(el_node_name, el_shared.RPC_PORT_NUMBER) + el_grpc_url = "http://{}:{}".format(el_node_name, el_shared.GRPC_PORT_NUMBER) service = launch_method( plan, @@ -51,6 +52,7 @@ def launch( node_ids, l1_rpc_url, el_rpc_url, + el_grpc_url, rabbitmq_url, container_proc_manager_artifact, ) diff --git a/src/config/constants.star b/src/config/constants.star index 79fa36ac..8160cfd8 100644 --- a/src/config/constants.star +++ b/src/config/constants.star @@ -55,20 +55,20 @@ IMAGES = { "l1_anvil_image": "ghcr.io/foundry-rs/foundry:v1.6.0-rc1", # layer 2 "l2_cl_heimdall_v2_image": "0xpolygon/heimdall-v2:0.6.0", - "l2_el_bor_image": "0xpolygon/bor:2.7.1", + "l2_el_bor_image": "0xpolygon/bor:2.7.0", "l2_el_erigon_image": "0xpolygon/erigon:v3.5.0", - "l2_cl_queue_image": "rabbitmq:4.2.5", + "l2_cl_queue_image": "rabbitmq:4.2.4", # utilities - "pos_contract_deployer_image": "ghcr.io/0xpolygon/pos-contract-deployer:bc28cfd8", + "pos_contract_deployer_image": "ghcr.io/0xpolygon/pos-contract-deployer:d96d592", "pos_el_genesis_builder_image": "ghcr.io/0xpolygon/pos-el-genesis-builder:96a19dd", "pos_validator_config_generator_image": "ghcr.io/0xpolygon/pos-validator-config-generator:0.6.0", "toolbox_image": "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/public/toolbox:0.0.12", # additional services "status_checker_image": "ghcr.io/0xpolygon/status-checker:v0.2.9", # observability - "prometheus_image": "prom/prometheus:v3.11.2", - "grafana_image": "grafana/grafana:12.4.2", - "panoptichain_image": "ghcr.io/0xpolygon/panoptichain:v5.2.0", + "prometheus_image": "prom/prometheus:v3.10.0", + "grafana_image": "grafana/grafana:12.3.4", + "panoptichain_image": "ghcr.io/0xpolygon/panoptichain:v5.1.2", "ethstats_server_image": "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/public/ethstats-server:9da2124", } @@ -106,14 +106,13 @@ EL_HARD_FORK_BLOCKS = { "napoli": 0, "ahmedabad": 0, "bhilai": 0, - # rio must be enabled at block 256 because it's hardcoded in heimdall-v2 codebase - # https://github.com/0xPolygon/heimdall-v2/blob/4ff4059d7d83bcadc81e88d513f178ca3ba15fd8/helper/config.go#L488 - "rio": 256, - # hardforks happening after rio should also be enabled at block 256 or later - "madhugiri": 256, - "madhugiriPro": 256, - "dandeli": 256, - "lisovo": 256, - "lisovoPro": 256, - "giugliano": 256, + # rio must be enabled at block 128 because it's hardcoded in heimdall-v2 codebase + "rio": 128, + # hardforks happening after rio should also be enabled at block 128 or later + "madhugiri": 128, + "madhugiriPro": 128, + "dandeli": 128, + "lisovo": 128, + "lisovoPro": 128, + "giugliano": 128, } diff --git a/src/config/input_parser.star b/src/config/input_parser.star index 6970faa7..0a02c87c 100644 --- a/src/config/input_parser.star +++ b/src/config/input_parser.star @@ -84,6 +84,7 @@ POLYGON_POS_PARTICIPANT = { "cl_indexer_pruning_enabled": constants.CL_INDEXER_PRUNING_ENABLED, "el_type": constants.EL_TYPE.bor, "el_image": constants.IMAGES.get("l2_el_bor_image"), + "cl_bor_grpc_flag": False, "count": 1, } diff --git a/src/config/sanity_check.star b/src/config/sanity_check.star index 045bf9c4..87b0a936 100644 --- a/src/config/sanity_check.star +++ b/src/config/sanity_check.star @@ -25,6 +25,7 @@ POLYGON_POS_PARAMS = { "el_bor_produce_witness", # Allow bor to start producing witnesses. "el_bor_sync_with_witness", # Enable bor to sync new blocks using witnesses. "el_bor_stateless_parallel_import", # Enable bor to use parallel import in stateless mode. + "cl_bor_grpc_flag", # Enable heimdall → bor gRPC transport (default HTTP). "count", ], "setup_images": [ diff --git a/src/el/bor/launcher.star b/src/el/bor/launcher.star index b847deb5..5c77c595 100644 --- a/src/el/bor/launcher.star +++ b/src/el/bor/launcher.star @@ -67,6 +67,7 @@ def launch( "discovery_port_number": shared.DISCOVERY_PORT_NUMBER, "metrics_port_number": shared.METRICS_PORT_NUMBER, "pprof_port_number": shared.PPROF_PORT_NUMBER, + "grpc_port_number": shared.GRPC_PORT_NUMBER, "ethstats_server_port_number": constants.ETHSTATS_SERVER_PORT_NUMBER, }, ), @@ -115,6 +116,10 @@ def launch( number=shared.PPROF_PORT_NUMBER, application_protocol="http", ), + shared.GRPC_PORT_ID: PortSpec( + number=shared.GRPC_PORT_NUMBER, + application_protocol="grpc", + ), }, files={ # app data diff --git a/src/el/shared.star b/src/el/shared.star index f43cbc39..14aceaf0 100644 --- a/src/el/shared.star +++ b/src/el/shared.star @@ -14,6 +14,9 @@ METRICS_PORT_NUMBER = 7071 PPROF_PORT_ID = "pprof" PPROF_PORT_NUMBER = 6060 +GRPC_PORT_ID = "grpc" +GRPC_PORT_NUMBER = 3131 + # CPU and memory limits. MAX_CPU = 4000 # in milicores (4 cores) MAX_MEM = 16384 # in megabytes (16 GB) diff --git a/static_files/cl/heimdall_v2/app.toml b/static_files/cl/heimdall_v2/app.toml index 144b56ec..1b7a496d 100644 --- a/static_files/cl/heimdall_v2/app.toml +++ b/static_files/cl/heimdall_v2/app.toml @@ -66,7 +66,7 @@ index-events = [] # IavlCacheSize set the size of the iavl tree cache (in number of nodes). iavl-cache-size = 781250 -# IAVLDisableFastNode enables or disables the fast node feature of IAVL. +# IAVLDisableFastNode enables or disables the fast node feature of IAVL. # Default is false. iavl-disable-fastnode = false @@ -247,10 +247,10 @@ eth_rpc_url = "{{.l1_rpc_url}}" bor_rpc_url = "{{.el_rpc_url}}" # GRPC flag for bor chain -bor_grpc_flag = "false" +bor_grpc_flag = "{{ if .cl_bor_grpc_flag }}true{{ else }}false{{ end }}" # GRPC endpoint for bor chain -bor_grpc_url = "localhost:3131" +bor_grpc_url = "{{.el_grpc_url}}" # RPC endpoint for cometBFT comet_bft_rpc_url = "http://0.0.0.0:{{.rpc_port_number}}" @@ -277,11 +277,9 @@ sh_stake_update_interval = "3h0m0s" sh_checkpoint_ack_interval = "30m0s" sh_max_depth_duration = "24h0m0s" -#### gas limits #### -main_chain_gas_limit = "5000000" - -#### gas price #### -main_chain_max_gas_price = "400000000000" +#### gas price configs (EIP-1559) #### +main_chain_gas_fee_cap = "500000000000" +main_chain_gas_tip_cap = "10000000000" ##### Timeout Config ##### no_ack_wait_time = "30m0s" diff --git a/static_files/el/bor/config.toml b/static_files/el/bor/config.toml index fafdb4ef..eaceae5a 100644 --- a/static_files/el/bor/config.toml +++ b/static_files/el/bor/config.toml @@ -129,6 +129,15 @@ ethstats = "{{.node_name}}:{{.ethstats_server_secret}}@ethstats-server:{{.ethsta # Enable expensive metrics collection and reporting (default: false) expensive = false +[grpc] + # gRPC server for heimdall↔bor communication. Bound to 0.0.0.0 inside the + # container so sibling containers (heimdall) can reach it. Bor's own + # production default is 127.0.0.1:3131; we override for this devnet topology. + # Token authentication is left empty — kurtosis runs over the internal + # docker network without TLS, and heimdall refuses to send a bearer token + # over plaintext to a non-loopback peer. + addr = "0.0.0.0:{{.grpc_port_number}}" + [pprof] # Enable the pprof HTTP server (default: false) pprof = true @@ -191,3 +200,11 @@ ethstats = "{{.node_name}}:{{.ethstats_server_secret}}@ethstats-server:{{.ethsta globalqueue = 131072 # Maximum amount of time non-executable transaction are queued lifetime = "1h30m0s" + # Enable periodic re-announcement of stuck pending transactions to peers (default: true) + rebroadcast = true + # How often to check for and rebroadcast stuck transactions + rebroadcast-interval = "10s" + # Transactions in pending pool longer than this are considered stuck + rebroadcast-max-age = "1m" + # Max transactions to rebroadcast per cycle to limit bandwidth usage (min: 1) + rebroadcast-batch-size = 200 diff --git a/static_files/scripts/container-proc-manager.sh b/static_files/scripts/container-proc-manager.sh index 1fc1c695..afad45d9 100755 --- a/static_files/scripts/container-proc-manager.sh +++ b/static_files/scripts/container-proc-manager.sh @@ -16,7 +16,6 @@ # - Exits with the child process exit code on natural termination. # - Runs indefinitely (via tail -f /dev/null) if SIGTRAP was received. - # Assign the command passed as arguments to a variable. # The "$*" captures all command-line arguments as a single string. # For example, running `./container-proc-manager.sh sleep 100` will store "sleep 100" in child_cmd.