From 0b0c053d12aa4517dd4cfef75c32c86da479f1c8 Mon Sep 17 00:00:00 2001 From: Michael Silberling <4006780+MSilb7@users.noreply.github.com> Date: Mon, 19 May 2025 16:19:10 -0400 Subject: [PATCH 1/5] add transaction_hash to grouping sets --- ...y_trto_txto_txhash_txmethod_v1__CREATE.sql | 100 ++++++++ ...y_trto_txto_txhash_txmethod_v1__INSERT.sql | 235 ++++++++++++++++++ 2 files changed, 335 insertions(+) create mode 100644 src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__CREATE.sql create mode 100644 src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql diff --git a/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__CREATE.sql b/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__CREATE.sql new file mode 100644 index 00000000000..c2672d52bc8 --- /dev/null +++ b/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__CREATE.sql @@ -0,0 +1,100 @@ +CREATE TABLE IF NOT EXISTS _placeholder_ +( + `dt` Date, + `chain` String, + `chain_id` Int32, + `network` String, + `trace_to_address` FixedString(42), + `tx_to_address` FixedString(42), + `tx_hash` String, + `tx_success` Bool, + `tx_method_id` String, + `count_distinct_trace_from_addresses` UInt64, + `count_distinct_tx_from_addresses` UInt64, + `sum_tx_l2_gas_used` UInt64, + `sum_tx_l2_gas_used_tx_success` UInt64, + `sum_tx_fee_native` Float64, + `sum_tx_fee_native_tx_success` Float64, + `sum_tx_l1_fee_native` Float64, + `sum_tx_l1_fee_native_tx_success` Float64, + `sum_tx_l2_fee_native` Float64, + `sum_tx_l2_fee_native_tx_success` Float64, + `sum_tx_l2_priority_fee_native` Float64, + `sum_tx_l2_priority_fee_native_tx_success` Float64, + `sum_tx_l2_base_fee_native` Float64, + `sum_tx_l2_base_fee_native_tx_success` Float64, + `sum_tx_l2_legacy_extra_fee_native` Float64, + `sum_tx_l2_legacy_extra_fee_native_tx_success` Float64, + + `weighted_sum_tx_l2_gas_price_gwei_called` Float64, + `weighted_sum_tx_l2_priority_gas_price_gwei_called` Float64, + `weighted_sum_tx_l2_legacy_extra_gas_price_gwei_called` Float64, + `weighted_sum_tx_l1_base_gas_price_gwei_called` Float64, + `weighted_sum_tx_l1_blob_base_gas_price_gwei_called` Float64, + + `sum_tx_input_byte_length_called` UInt64, + `sum_tx_input_zero_bytes_called` UInt64, + `sum_tx_input_nonzero_bytes_called` Int64, + + `sum_tx_l1_base_scaled_size_called` Float64, + `sum_tx_l1_blob_scaled_size_called` Float64, + `sum_tx_l1_gas_used_unified_called` UInt64, + `sum_tx_estimated_size_called` UInt64, + + `sum_trace_gas_used` UInt64, + `sum_trace_gas_used_tx_success` UInt64, + + `sum_trace_gas_used_minus_subtraces` Float64, + `sum_trace_gas_used_minus_subtraces_tx_success` Float64, + + `sum_tx_l2_gas_used_amortized_by_call` Float64, + `sum_tx_l2_gas_used_amortized_by_call_tx_success` Float64, + + `sum_tx_l1_gas_used_unified_amortized_by_call` Float64, + `sum_tx_l1_base_scaled_size_amortized_by_call` Float64, + `sum_tx_l1_blob_scaled_size_amortized_by_call` Float64, + + `sum_tx_estimated_size_amortized_by_call` Float64, + + `sum_tx_l2_fee_native_minus_subtraces` Float64, + `sum_tx_l2_fee_native_minus_subtraces_tx_success` Float64, + + `sum_tx_l2_base_fee_native_minus_subtraces` Float64, + `sum_tx_l2_priority_fee_native_minus_subtraces` Float64, + `sum_tx_l2_legacy_base_fee_native_minus_subtraces` Float64, + + `sum_tx_fee_native_amortized_by_call` Float64, + `sum_tx_fee_native_amortized_by_call_tx_success` Float64, + + `sum_tx_l2_fee_native_amortized_by_call` Float64, + `sum_tx_l2_fee_native_amortized_by_call_tx_success` Float64, + + `sum_tx_l1_fee_native_amortized_by_call` Float64, + `sum_tx_l1_fee_native_amortized_by_call_tx_success` Float64, + + `sum_tx_l2_base_fee_native_amortized_by_call` Float64, + `sum_tx_l2_priority_fee_native_amortized_by_call` Float64, + `sum_tx_fee_native_l1_amortized_l2_minus_subtraces` Float64, + + `count_top_level_calls` UInt64, + `count_internal_calls_all_types` UInt64, + `count_internal_calls_all_types_trace_success` UInt64, + `count_internal_calls_static_type` UInt64, + `count_internal_calls_static_type_trace_success` UInt64, + `count_internal_calls_delegate_type` UInt64, + `count_internal_calls_delegate_type_trace_success` UInt64, + `count_internal_calls_call_type` UInt64, + `count_internal_calls_call_type_trace_success` UInt64, + + `count_transactions_called_with_internal_type_call` UInt64, + `count_transactions_called_with_internal_type_call_or_delegate` UInt64, + + `count_traces_type_all_types` UInt64, + `count_traces_trace_type_call` UInt64, + `count_traces_trace_type_suicide` UInt64, + `count_traces_trace_type_create` UInt64, + `count_traces_trace_type_create2` UInt64, + `count_traces_trace_type_create_any` UInt64 +) +ENGINE = ReplacingMergeTree +ORDER BY (dt, chain, chain_id, network, trace_to_address, tx_to_address, tx_hash, tx_method_id) diff --git a/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql b/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql new file mode 100644 index 00000000000..45ff8972ca5 --- /dev/null +++ b/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql @@ -0,0 +1,235 @@ +/** +Aggregation grain is tr_to_address, tx_to_address, tx_method_id +*/ + +WITH + +refined_traces AS ( + SELECT + dt + , chain + , chain_id + , network + , accurateCast(trace_to_address, 'FixedString(42)') AS trace_to_address + , accurateCast(trace_from_address, 'FixedString(42)') AS trace_from_address + , accurateCast(transaction_hash, 'String') AS transaction_hash + , accurateCast(block_number, 'UInt64') AS block_number + , accurateCast(num_traces_in_txn, 'UInt32') AS num_traces_in_txn + , accurateCast(trace_gas_used, 'UInt64') AS trace_gas_used + , accurateCast(sum_subtraces_gas_used, 'Float64') AS sum_subtraces_gas_used + , accurateCast(gas_used_minus_subtraces, 'Float64') AS gas_used_minus_subtraces + , accurateCast(tx_l2_gas_used_amortized_by_call, 'Float64') AS tx_l2_gas_used_amortized_by_call + , accurateCast(coalesce(tx_l1_gas_used_unified_amortized_by_call, 0), 'Float64') AS tx_l1_gas_used_unified_amortized_by_call + , accurateCast(coalesce(tx_l1_base_scaled_size_amortized_by_call, 0), 'Float64') AS tx_l1_base_scaled_size_amortized_by_call + , accurateCast(coalesce(tx_l1_blob_scaled_size_amortized_by_call, 0), 'Float64') AS tx_l1_blob_scaled_size_amortized_by_call + , accurateCast(coalesce(tx_estimated_size_amortized_by_call, 0), 'Float64') AS tx_estimated_size_amortized_by_call + , accurateCast(tx_l2_fee_native_minus_subtraces, 'Float64') AS tx_l2_fee_native_minus_subtraces + , accurateCast(tx_l2_base_fee_native_minus_subtraces, 'Float64') AS tx_l2_base_fee_native_minus_subtraces + , accurateCast(tx_l2_priority_fee_native_minus_subtraces, 'Float64') AS tx_l2_priority_fee_native_minus_subtraces + , accurateCast(tx_l2_legacy_base_fee_native_minus_subtraces, 'Float64') AS tx_l2_legacy_base_fee_native_minus_subtraces + , accurateCast(tx_fee_native_amortized_by_call, 'Float64') AS tx_fee_native_amortized_by_call + , accurateCast(tx_l2_fee_native_amortized_by_call, 'Float64') AS tx_l2_fee_native_amortized_by_call + , accurateCast(tx_l1_fee_native_amortized_by_call, 'Float64') AS tx_l1_fee_native_amortized_by_call + , accurateCast(tx_l2_base_fee_native_amortized_by_call, 'Float64') AS tx_l2_base_fee_native_amortized_by_call + , accurateCast(tx_l2_priority_fee_native_amortized_by_call, 'Float64') AS tx_l2_priority_fee_native_amortized_by_call + + , if(trace_address = '' AND call_type != '', 1, 0) AS is_top_level_call + + -- non-null trace addresses, null is the transaction-level call. + , if(trace_address != '' AND call_type != '', 1, 0) AS is_internal_call_all_types + , if(trace_address != '' AND call_type != '' AND trace_success, 1, 0) AS is_internal_call_all_types_trace_success + + --static calls only read state, and can not write + , if(trace_address != '' AND call_type = 'staticcall', 1, 0) AS is_internal_call_static_type + , if(trace_address != '' AND call_type = 'staticcall' AND trace_success, 1, 0) AS is_internal_call_static_type_trace_success + + --delegate calls call a function on another contract + , if(trace_address != '' AND call_type = 'delegatecall', 1, 0) AS is_internal_call_delegate_type + , if(trace_address != '' AND call_type = 'delegatecall' AND trace_success, 1, 0) AS is_internal_call_delegate_type_trace_success + + --normal function calls + , if(trace_address != '' AND call_type = 'call', 1, 0) AS is_internal_call_call_type + , if(trace_address != '' AND call_type = 'call' AND trace_success, 1, 0) AS is_internal_call_call_type_trace_success + + , if(trace_type = 'call', 1, 0) AS is_type_call + , if(trace_type = 'suicide', 1, 0) AS is_type_suicide + , if(trace_type = 'create', 1, 0) AS is_type_create + , if(trace_type = 'create2', 1, 0) AS is_type_create2 + , if(trace_type LIKE 'create%', 1, 0) AS is_type_create_any + + -- Transaction Fields + + , accurateCast(tx_success, 'Bool') AS tx_success + , accurateCast(tx_from_address, 'FixedString(42)') AS tx_from_address + , accurateCast(tx_to_address, 'FixedString(42)') AS tx_to_address + , accurateCast(tx_method_id, 'String') AS tx_method_id + , accurateCast(tx_l2_gas_used, 'UInt64') AS tx_l2_gas_used + , accurateCast(tx_fee_native, 'Float64') AS tx_fee_native + + , accurateCast(tx_l1_fee_native, 'Float64') AS tx_l1_fee_native + , accurateCast(tx_l2_fee_native, 'Float64') AS tx_l2_fee_native + , accurateCast(tx_l2_priority_fee_native, 'Float64') AS tx_l2_priority_fee_native + , accurateCast(tx_l2_base_fee_native, 'Float64') AS tx_l2_base_fee_native + , accurateCast(tx_l2_legacy_extra_fee_native, 'Float64') AS tx_l2_legacy_extra_fee_native + + , accurateCast(tx_l2_gas_price_gwei, 'Float64') AS tx_l2_gas_price_gwei + , accurateCast(tx_l2_priority_gas_price_gwei, 'Float64') AS tx_l2_priority_gas_price_gwei + , accurateCast(tx_l2_legacy_extra_gas_price_gwei, 'Float64') AS tx_l2_legacy_extra_gas_price_gwei + , accurateCast(coalesce(tx_l1_base_gas_price_gwei, 0), 'Float64') AS tx_l1_base_gas_price_gwei + , accurateCast(coalesce(tx_l1_blob_base_gas_price_gwei, 0), 'Float64') AS tx_l1_blob_base_gas_price_gwei + + , accurateCast(coalesce(tx_l1_gas_used_unified, 0), 'Float64') AS tx_l1_gas_used_unified + , accurateCast(coalesce(tx_l1_base_scaled_size, 0), 'Float64') AS tx_l1_base_scaled_size + , accurateCast(coalesce(tx_l1_blob_scaled_size, 0), 'Float64') AS tx_l1_blob_scaled_size + , accurateCast(coalesce(tx_estimated_size, 0), 'Float64') AS tx_estimated_size + + , accurateCast(tx_input_byte_length, 'UInt32') AS tx_input_byte_length + , accurateCast(tx_input_zero_bytes, 'UInt32') AS tx_input_zero_bytes + , accurateCast(tx_input_nonzero_bytes, 'UInt32') AS tx_input_nonzero_bytes + + FROM INPUT_BLOCKBATCH('blockbatch/refined_traces/refined_traces_fees_v2') +) + + +SELECT + tr.dt + , tr.chain + , tr.chain_id + , tr.network + , tr.trace_to_address + , tr.tx_to_address AS tx_to_address + , tr.transaction_hash AS tx_hash + , tr.tx_success AS tx_success + , tr.tx_method_id AS tx_method_id + + , uniq(tr.trace_from_address) AS count_distinct_trace_from_addresses + , uniq(tr.tx_from_address) AS count_distinct_tx_from_addresses + + -- Aggregate transaction metrics, in transactions when a given contract was called + /* + Note: We can NOT sum these up at the project level, since a project may have + many of its contracts called in a single transaction. + + For these use cases (i.e. project impact metrics), we can aggregate `aggregated_traces_tr_to_hash` + at the project name level, only after all trace_to_address entries are mapped to their respective + project name (i.e. a transaction may call 10 contracts, but this may only represent 3 projects). + */ + + -- Transaction Level Gas and Fees + , sum(tr.tx_l2_gas_used) AS sum_tx_l2_gas_used + , sum(CASE WHEN tr.tx_success THEN tr.tx_l2_gas_used ELSE 0 END) AS sum_tx_l2_gas_used_tx_success + + , sum(tr.tx_fee_native) AS sum_tx_fee_native + , sum(CASE WHEN tr.tx_success THEN tr.tx_fee_native ELSE 0 END) AS sum_tx_fee_native_tx_success + + , sum(tr.tx_l1_fee_native) AS sum_tx_l1_fee_native + , sum(CASE WHEN tr.tx_success THEN tr.tx_l1_fee_native ELSE 0 END) AS sum_tx_l1_fee_native_tx_success + + , sum(tr.tx_l2_fee_native) AS sum_tx_l2_fee_native + , sum(CASE WHEN tr.tx_success THEN tr.tx_l2_fee_native ELSE 0 END) AS sum_tx_l2_fee_native_tx_success + + , sum(tr.tx_l2_priority_fee_native) AS sum_tx_l2_priority_fee_native + , sum(CASE WHEN tr.tx_success THEN tr.tx_l2_priority_fee_native ELSE 0 END) AS sum_tx_l2_priority_fee_native_tx_success + + , sum(tr.tx_l2_base_fee_native) AS sum_tx_l2_base_fee_native + , sum(CASE WHEN tr.tx_success THEN tr.tx_l2_base_fee_native ELSE 0 END) AS sum_tx_l2_base_fee_native_tx_success + + , sum(tr.tx_l2_legacy_extra_fee_native) AS sum_tx_l2_legacy_extra_fee_native + , sum(CASE WHEN tr.tx_success THEN tr.tx_l2_legacy_extra_fee_native ELSE 0 END) AS sum_tx_l2_legacy_extra_fee_native_tx_success + + -- Transaction-Level Gas Prices + , sum(tr.tx_l2_gas_price_gwei * tr.tx_l2_gas_used) AS weighted_sum_tx_l2_gas_price_gwei_called + , sum(tr.tx_l2_priority_gas_price_gwei * tr.tx_l2_gas_used) AS weighted_sum_tx_l2_priority_gas_price_gwei_called + , sum(tr.tx_l2_legacy_extra_gas_price_gwei * tr.tx_l2_gas_used) AS weighted_sum_tx_l2_legacy_extra_gas_price_gwei_called + , sum(tr.tx_l1_base_gas_price_gwei * tr.tx_l2_gas_used) AS weighted_sum_tx_l1_base_gas_price_gwei_called + , sum(tr.tx_l1_blob_base_gas_price_gwei * tr.tx_l2_gas_used) AS weighted_sum_tx_l1_blob_base_gas_price_gwei_called + + -- Transaction-Level transaction sizes + , sum(tr.tx_input_byte_length) AS sum_tx_input_byte_length_called + , sum(tr.tx_input_zero_bytes) AS sum_tx_input_zero_bytes_called + , sum(tr.tx_input_nonzero_bytes) AS sum_tx_input_nonzero_bytes_called + , sum(tr.tx_l1_base_scaled_size) AS sum_tx_l1_base_scaled_size_called + , sum(tr.tx_l1_blob_scaled_size) AS sum_tx_l1_blob_scaled_size_called + , sum(tr.tx_l1_gas_used_unified) AS sum_tx_l1_gas_used_unified_called + , sum(tr.tx_estimated_size) AS sum_tx_estimated_size_called + + -- Sum Internal Trace Gas + /* + Note: As opposed to transaction-level equivalent metrics, these CAN be summed up at a project level. + These methodologies were designed to distribute gas usage and fees across each call, so that they + still represent a "part of a whole," whereas transaction-level metrics do not. + */ + + , sum(tr.trace_gas_used) AS sum_trace_gas_used + , sum(CASE WHEN tr.tx_success THEN tr.trace_gas_used ELSE 0 END) AS sum_trace_gas_used_tx_success + + , sum(tr.gas_used_minus_subtraces) AS sum_trace_gas_used_minus_subtraces + , sum(CASE WHEN tr.tx_success THEN tr.gas_used_minus_subtraces ELSE 0 END) AS sum_trace_gas_used_minus_subtraces_tx_success + + , sum(tr.tx_l2_gas_used_amortized_by_call) AS sum_tx_l2_gas_used_amortized_by_call + , sum(CASE WHEN tr.tx_success THEN tr.tx_l2_gas_used_amortized_by_call ELSE 0 END) AS sum_tx_l2_gas_used_amortized_by_call_tx_success + + , sum(tr.tx_l1_gas_used_unified_amortized_by_call) AS sum_tx_l1_gas_used_unified_amortized_by_call + , sum(tr.tx_l1_base_scaled_size_amortized_by_call) AS sum_tx_l1_base_scaled_size_amortized_by_call + , sum(tr.tx_l1_blob_scaled_size_amortized_by_call) AS sum_tx_l1_blob_scaled_size_amortized_by_call + , sum(tr.tx_estimated_size_amortized_by_call) AS sum_tx_estimated_size_amortized_by_call + + , sum(tr.tx_l2_fee_native_minus_subtraces) AS sum_tx_l2_fee_native_minus_subtraces + , sum(CASE WHEN tr.tx_success THEN tr.tx_l2_fee_native_minus_subtraces ELSE 0 END) AS sum_tx_l2_fee_native_minus_subtraces_tx_success + + , sum(tr.tx_l2_base_fee_native_minus_subtraces) AS sum_tx_l2_base_fee_native_minus_subtraces + , sum(tr.tx_l2_priority_fee_native_minus_subtraces) AS sum_tx_l2_priority_fee_native_minus_subtraces + , sum(tr.tx_l2_legacy_base_fee_native_minus_subtraces) AS sum_tx_l2_legacy_base_fee_native_minus_subtraces + + , sum(tr.tx_fee_native_amortized_by_call) AS sum_tx_fee_native_amortized_by_call + , sum(CASE WHEN tr.tx_success THEN tr.tx_fee_native_amortized_by_call ELSE 0 END) AS sum_tx_fee_native_amortized_by_call_tx_success + + , sum(tr.tx_l2_fee_native_amortized_by_call) AS sum_tx_l2_fee_native_amortized_by_call + , sum(CASE WHEN tr.tx_success THEN tr.tx_l2_fee_native_amortized_by_call ELSE 0 END) AS sum_tx_l2_fee_native_amortized_by_call_tx_success + + , sum(tr.tx_l1_fee_native_amortized_by_call) AS sum_tx_l1_fee_native_amortized_by_call + , sum(CASE WHEN tr.tx_success THEN tr.tx_l1_fee_native_amortized_by_call ELSE 0 END) AS sum_tx_l1_fee_native_amortized_by_call_tx_success + + , sum(tr.tx_l2_base_fee_native_amortized_by_call) AS sum_tx_l2_base_fee_native_amortized_by_call + , sum(tr.tx_l2_priority_fee_native_amortized_by_call) AS sum_tx_l2_priority_fee_native_amortized_by_call + + , sum(tr.tx_l1_fee_native / tr.num_traces_in_txn + CAST(tr.trace_gas_used - coalesce(tr.sum_subtraces_gas_used, 0) AS Float64) * tr.tx_l2_gas_price_gwei * 1e-9) AS sum_tx_fee_native_l1_amortized_l2_minus_subtraces + + + -- Calls + , sum(tr.is_top_level_call) AS count_top_level_calls + --count non-null trace addresses, null is the transaction-level call. + , sum(tr.is_internal_call_all_types) AS count_internal_calls_all_types + , sum(tr.is_internal_call_all_types_trace_success) AS count_internal_calls_all_types_trace_success + --static calls only read state, and can not write + , sum(tr.is_internal_call_static_type) AS count_internal_calls_static_type + , sum(tr.is_internal_call_static_type_trace_success) AS count_internal_calls_static_type_trace_success + --delegate calls call a function on another contract + , sum(tr.is_internal_call_delegate_type) AS count_internal_calls_delegate_type + , sum(tr.is_internal_call_delegate_type_trace_success) AS count_internal_calls_delegate_type_trace_success + --normal function calls + , sum(tr.is_internal_call_call_type) AS count_internal_calls_call_type + , sum(tr.is_internal_call_call_type_trace_success) AS count_internal_calls_call_type_trace_success + + -- Experimental + , uniq(CASE WHEN tr.is_internal_call_all_types THEN tr.transaction_hash END) AS count_transactions_called_with_internal_type_call + , uniq(CASE WHEN tr.is_internal_call_all_types OR tr.is_internal_call_delegate_type THEN tr.transaction_hash END) AS count_transactions_called_with_internal_type_call_or_delegate + + + --count by trace type + , sum(1) AS count_traces_type_all_types + , sum(tr.is_type_call) AS count_traces_trace_type_call + , sum(tr.is_type_suicide) AS count_traces_trace_type_suicide + , sum(tr.is_type_create) AS count_traces_trace_type_create + , sum(tr.is_type_create2) AS count_traces_trace_type_create2 + , sum(tr.is_type_create_any) AS count_traces_trace_type_create_any + +FROM refined_traces AS tr +GROUP BY + tr.dt + , tr.chain + , tr.chain_id + , tr.network + , tr.trace_to_address + , tr.tx_to_address + , tr.tx_method_id From 9bfccb556811c394f380b3827092729427ca3d58 Mon Sep 17 00:00:00 2001 From: Michael Silberling <4006780+MSilb7@users.noreply.github.com> Date: Mon, 19 May 2025 16:22:18 -0400 Subject: [PATCH 2/5] comment --- .../aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql b/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql index 45ff8972ca5..3dad8475873 100644 --- a/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql +++ b/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql @@ -1,5 +1,5 @@ /** -Aggregation grain is tr_to_address, tx_to_address, tx_method_id +Aggregation grain is tr_to_address, tx_to_address, tx_method_id, tx_hash */ WITH From cd1fba354e0a1afbe248b24d963fae55d9925a32 Mon Sep 17 00:00:00 2001 From: Michael Silberling <4006780+MSilb7@users.noreply.github.com> Date: Mon, 19 May 2025 16:22:48 -0400 Subject: [PATCH 3/5] groupby fix --- .../aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql b/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql index 3dad8475873..82034c222f0 100644 --- a/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql +++ b/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__INSERT.sql @@ -232,4 +232,6 @@ GROUP BY , tr.network , tr.trace_to_address , tr.tx_to_address + , tr.transaction_hash + , tr.tx_success , tr.tx_method_id From aa5d83ebc544d072eb61c068cbf44348412af877 Mon Sep 17 00:00:00 2001 From: Michael Silberling <4006780+MSilb7@users.noreply.github.com> Date: Mon, 19 May 2025 16:26:16 -0400 Subject: [PATCH 4/5] add to dagster --- src/op_analytics/dagster/assets/blockbatchloaddaily.py | 2 ++ .../datapipeline/etl/blockbatchloaddaily/datasets.py | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/src/op_analytics/dagster/assets/blockbatchloaddaily.py b/src/op_analytics/dagster/assets/blockbatchloaddaily.py index 6ee8049b08e..a01a0170e2e 100644 --- a/src/op_analytics/dagster/assets/blockbatchloaddaily.py +++ b/src/op_analytics/dagster/assets/blockbatchloaddaily.py @@ -11,6 +11,7 @@ TRACES_AGG1, TRACES_AGG2, TRACES_AGG3, + TRACES_AGG4, DAILY_ADDRESS_SUMMARY, DAILY_TX_ACTIVITY_SEGMENTS, ) @@ -28,6 +29,7 @@ def aggreagated_traces(context: OpExecutionContext): TRACES_AGG1, TRACES_AGG2, TRACES_AGG3, + TRACES_AGG4, DAILY_ADDRESS_SUMMARY, DAILY_TX_ACTIVITY_SEGMENTS, ]: diff --git a/src/op_analytics/datapipeline/etl/blockbatchloaddaily/datasets.py b/src/op_analytics/datapipeline/etl/blockbatchloaddaily/datasets.py index 1e8a6052150..6a6fcc42c64 100644 --- a/src/op_analytics/datapipeline/etl/blockbatchloaddaily/datasets.py +++ b/src/op_analytics/datapipeline/etl/blockbatchloaddaily/datasets.py @@ -54,6 +54,15 @@ ignore_zero_rows_chain_dts=ALLOWED_EMPTY_DATES, ) +TRACES_AGG4 = ClickHouseDateChainETL( + output_root_path="blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1", + inputs_blockbatch=[ + "blockbatch/refined_traces/refined_traces_fees_v2", + ], + ignore_zero_rows_chains=ALLOWED_EMPTY_CHAINS, + ignore_zero_rows_chain_dts=ALLOWED_EMPTY_DATES, +) + DAILY_ADDRESS_SUMMARY = ClickHouseDateChainETL( output_root_path="blockbatch_daily/aggtxs/daily_address_summary_v1", From 1013f62af3c9ed948d24e75b6f7a60dfeb46b0f7 Mon Sep 17 00:00:00 2001 From: Michael Silberling <4006780+MSilb7@users.noreply.github.com> Date: Mon, 19 May 2025 17:33:44 -0400 Subject: [PATCH 5/5] backfills --- .../blockbatch_clickhouse/load_daily.ipynb | 28 ++++++++++++------- ...y_trto_txto_txhash_txmethod_v1__CREATE.sql | 2 +- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/notebooks/adhoc/blockbatch_clickhouse/load_daily.ipynb b/notebooks/adhoc/blockbatch_clickhouse/load_daily.ipynb index 5514ca47669..f0330481fef 100644 --- a/notebooks/adhoc/blockbatch_clickhouse/load_daily.ipynb +++ b/notebooks/adhoc/blockbatch_clickhouse/load_daily.ipynb @@ -16,17 +16,16 @@ " TRACES_AGG1,\n", " TRACES_AGG2,\n", " TRACES_AGG3,\n", + " TRACES_AGG4,\n", " DAILY_ADDRESS_SUMMARY,\n", ")\n", "\n", - "\n", - "for agg in [TRACES_AGG1, TRACES_AGG2, TRACES_AGG3, DAILY_ADDRESS_SUMMARY]:\n", - " result = daily_to_clickhouse(\n", - " dataset=agg,\n", - " range_spec=\"@20250501:20250508\",\n", - " num_workers=2,\n", - " )\n", - " print(result)\n" + "agg_list = [\n", + " TRACES_AGG4\n", + " # TRACES_AGG1, TRACES_AGG2, TRACES_AGG3, TRACES_AGG4, DAILY_ADDRESS_SUMMARY\n", + "]\n", + "chains_list = ['op','unichain']\n", + "# agg_list" ] }, { @@ -34,7 +33,16 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "for agg in agg_list:\n", + " result = daily_to_clickhouse(\n", + " dataset=agg,\n", + " range_spec=\"@20250501:20250518\",\n", + " num_workers=2,\n", + " chains = chains_list\n", + " )\n", + " print(result)" + ] } ], "metadata": { @@ -53,7 +61,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.6" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__CREATE.sql b/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__CREATE.sql index c2672d52bc8..a9710b4f2f7 100644 --- a/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__CREATE.sql +++ b/src/op_analytics/datapipeline/etl/blockbatchloaddaily/ddl/blockbatch_daily/aggtraces/daily_trto_txto_txhash_txmethod_v1__CREATE.sql @@ -97,4 +97,4 @@ CREATE TABLE IF NOT EXISTS _placeholder_ `count_traces_trace_type_create_any` UInt64 ) ENGINE = ReplacingMergeTree -ORDER BY (dt, chain, chain_id, network, trace_to_address, tx_to_address, tx_hash, tx_method_id) +ORDER BY (dt, chain, chain_id, network, trace_to_address, tx_to_address, tx_hash, tx_method_id) \ No newline at end of file