From 784e89725ffafa090427d2650b3bff57c866bdd2 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Fri, 2 Jan 2026 23:29:52 +0300 Subject: [PATCH 01/49] Initial version of the application. --- Apps/QueryLogsDuckDBApp/App.cs | 514 ++++++++++++++++++ .../QueryLogsDuckDBApp.csproj | 48 ++ Apps/QueryLogsDuckDBApp/dnsApp.config | 4 + DnsServer.sln | 8 + 4 files changed, 574 insertions(+) create mode 100644 Apps/QueryLogsDuckDBApp/App.cs create mode 100644 Apps/QueryLogsDuckDBApp/QueryLogsDuckDBApp.csproj create mode 100644 Apps/QueryLogsDuckDBApp/dnsApp.config diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs new file mode 100644 index 000000000..d5a8e4901 --- /dev/null +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -0,0 +1,514 @@ +/* +Technitium DNS Server +Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) +Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +using DnsServerCore.ApplicationCommon; +using DuckDB.NET.Data; +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using System.Net; +using System.Text.Json; +using System.Threading.Channels; +using System.Threading.Tasks; +using TechnitiumLibrary; +using TechnitiumLibrary.Net.Dns; +using TechnitiumLibrary.Net.Dns.ResourceRecords; + +namespace QueryLogsDuckDB +{ + public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs + { + #region variables + + private IDnsServer _dnsServer; + private DuckDBConnection _conn; + private string _parquetPath; + private string _dbPath; + + private bool _enableLogging; + private int _bufferedRows = 0; + private const int MAX_BATCH_SIZE = 10000; + private const string BUFFER_TABLE = "dns_buffer"; + private const string UNIFIED_VIEW = "dns_logs"; + + private Channel _channel; + private Task _consumerTask; + private bool _disposed; + + #endregion variables + + #region IDisposable + + public void Dispose() + { + if (_disposed) return; + _channel?.Writer.TryComplete(); + _consumerTask?.Wait(5000); + FlushToParquetAsync().GetAwaiter().GetResult(); + _conn?.Dispose(); + _disposed = true; + } + + #endregion IDisposable + + #region private + + private async Task RefreshViewAsync() + { + string parquetSource = File.Exists(_parquetPath) + ? $"read_parquet('{_parquetPath}')" + : $"(SELECT * FROM {BUFFER_TABLE} WHERE 1=0)"; + + using var cmd = _conn.CreateCommand(); + cmd.CommandText = $"CREATE OR REPLACE VIEW {UNIFIED_VIEW} AS SELECT * FROM {BUFFER_TABLE} UNION ALL SELECT * FROM {parquetSource};"; + await cmd.ExecuteNonQueryAsync(); + } + + private async Task ProcessLogsAsync() + { + var batch = new List(MAX_BATCH_SIZE); + while (await _channel.Reader.WaitToReadAsync()) + { + while (batch.Count < MAX_BATCH_SIZE && _channel.Reader.TryRead(out var log)) + { + batch.Add(log); + } + + if (batch.Count > 0) + { + await BulkInsertInternalAsync(batch); + batch.Clear(); + } + } + } + + private async Task BulkInsertInternalAsync(List logs) + { + try + { + using (var appender = _conn.CreateAppender(BUFFER_TABLE)) + { + foreach (var log in logs) + { + if (log.Request is null || log.Response is null) + continue; // skip corrupt entries defensively + + var question = log.Request.Question?[0]; + var row = appender.CreateRow(); + + // RTT is only meaningful when Metadata exists and Tag is null + double? rtt = + (log.Response.Tag is null && log.Response.Metadata is not null) + ? log.Response.Metadata.RoundTripTime + : null; + + // Nullable values prepared up front + byte? tag = + log.Response.Tag is null ? (byte?)null : ((byte)log.Response.Tag); + + byte rcode = (byte)log.Response.RCODE; + + string? qname = + question?.Name is null ? null : question.Name.ToLowerInvariant(); + + ushort? qtype = + question is null ? (ushort?)null : (ushort)question.Type; + + ushort? qclass = + question is null ? (ushort?)null : (ushort)question.Class; + + // Append values — emit NULLs where appropriate + row.AppendValue(log.Timestamp); + + row.AppendValue( + log.RemoteEP?.Address is null + ? null + : log.RemoteEP.Address.ToString()); + + row.AppendValue((byte)log.Protocol); + + if (tag is null) row.AppendNullValue(); else row.AppendValue(tag.Value); + + if (rtt is null) row.AppendNullValue(); else row.AppendValue(rtt.Value); + + row.AppendValue(rcode); + + if (qname is null) row.AppendNullValue(); else row.AppendValue(qname); + + if (qtype is null) row.AppendNullValue(); else row.AppendValue(qtype.Value); + + if (qclass is null) row.AppendNullValue(); else row.AppendValue(qclass.Value); + + var answer = FormatAnswer(log.Response); + if (answer is null) row.AppendNullValue(); else row.AppendValue(answer); + + row.EndRow(); + + _bufferedRows++; + } + + } + + if (_bufferedRows >= MAX_BATCH_SIZE) + { + await FlushToParquetAsync(); + } + } + catch (Exception ex) { _dnsServer.WriteLog(ex); } + } + + private string? FormatAnswer(DnsDatagram response) + { + if (response.Answer.Count == 0) return null; + return string.Join(", ", response.Answer.Select(r => $"{r.Type} {r.RDATA}")); + } + + private async Task FlushToParquetAsync() + { + if (_bufferedRows == 0) return; + string tempFile = _parquetPath + ".tmp"; + + using (var cmd = _conn.CreateCommand()) + { + cmd.CommandText = $"COPY (SELECT * FROM {UNIFIED_VIEW} ORDER BY timestamp ASC) TO '{tempFile}' (FORMAT PARQUET, COMPRESSION 'ZSTD');"; + await cmd.ExecuteNonQueryAsync(); + + cmd.CommandText = $"DELETE FROM {BUFFER_TABLE};"; + await cmd.ExecuteNonQueryAsync(); + } + + if (File.Exists(_parquetPath)) File.Delete(_parquetPath); + File.Move(tempFile, _parquetPath); + + _bufferedRows = 0; + await RefreshViewAsync(); + } + + #endregion private + + #region public + + public async Task InitializeAsync(IDnsServer dnsServer, string config) + { + _dnsServer = dnsServer; + + using JsonDocument jsonDocument = JsonDocument.Parse(config); + JsonElement jsonConfig = jsonDocument.RootElement; + + _enableLogging = jsonConfig.GetPropertyValue("enableLogging", true); + string dbFileName = jsonConfig.GetPropertyValue("dbPath", "querylogs.duckdb"); + _dbPath = Path.IsPathRooted(dbFileName) ? dbFileName : Path.Combine(_dnsServer.ApplicationFolder, dbFileName); + _parquetPath = Path.ChangeExtension(_dbPath, ".parquet"); + + // Initialize DuckDB Connection + _conn = new DuckDBConnection($"Data Source={_dbPath}"); + await _conn.OpenAsync(); + + // Load plugin + using (var cmd = _conn.CreateCommand()) + { + cmd.CommandText = "INSTALL parquet;LOAD parquet;"; + await cmd.ExecuteNonQueryAsync(); + } + + // Setup Schema + using (var cmd = _conn.CreateCommand()) + { + cmd.CommandText = $@" + CREATE TEMP TABLE IF NOT EXISTS {BUFFER_TABLE} ( + timestamp TIMESTAMP, + client_ip VARCHAR(39), + protocol UTINYINT, + response_type UTINYINT, + response_rtt DOUBLE, + rcode UTINYINT, + qname VARCHAR(255), + qtype USMALLINT, + qclass USMALLINT, + answer TEXT + );"; + await cmd.ExecuteNonQueryAsync(); + } + + await RefreshViewAsync(); + + // Start Producer-Consumer Channel + _channel = Channel.CreateBounded(new BoundedChannelOptions(200000) + { + SingleReader = true, + FullMode = BoundedChannelFullMode.DropWrite + }); + + _consumerTask = Task.Run(ProcessLogsAsync); + } + + public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response) + { + if (_enableLogging) + _channel.Writer.TryWrite(new LogEntry(timestamp, request, remoteEP, protocol, response)); + + return Task.CompletedTask; + } + + public async Task QueryLogsAsync( + long pageNumber, + int entriesPerPage, + bool descendingOrder, + DateTime? start, + DateTime? end, + IPAddress clientIpAddress, + DnsTransportProtocol? protocol, + DnsServerResponseType? responseType, + DnsResponseCode? rcode, + string qname, + DnsResourceRecordType? qtype, + DnsClass? qclass) + { + using var cmd = _conn.CreateCommand(); + var filters = new List(); + + // ----- filters ----- + + if (start.HasValue) + { + filters.Add("timestamp >= @start"); + cmd.Parameters.Add(new DuckDBParameter("@start", start.Value)); + } + + if (end.HasValue) + { + filters.Add("timestamp <= @end"); + cmd.Parameters.Add(new DuckDBParameter("@end", end.Value)); + } + + if (clientIpAddress is not null) + { + filters.Add("client_ip = @ip"); + cmd.Parameters.Add(new DuckDBParameter("@ip", clientIpAddress.ToString())); + } + + if (protocol.HasValue) + { + filters.Add("protocol = @p"); + cmd.Parameters.Add(new DuckDBParameter("@p", (byte)protocol.Value)); + } + + if (responseType.HasValue) + { + filters.Add("response_type = @rt"); + cmd.Parameters.Add(new DuckDBParameter("@rt", (byte)responseType.Value)); + } + + if (rcode.HasValue) + { + filters.Add("rcode = @rc"); + cmd.Parameters.Add(new DuckDBParameter("@rc", (byte)rcode.Value)); + } + + if (qtype.HasValue) + { + filters.Add("qtype = @qt"); + cmd.Parameters.Add(new DuckDBParameter("@qt", (ushort)qtype.Value)); + } + + if (qclass.HasValue) + { + filters.Add("qclass = @qc"); + cmd.Parameters.Add(new DuckDBParameter("@qc", (ushort)qclass.Value)); + } + + if (!string.IsNullOrWhiteSpace(qname)) + { + filters.Add("LOWER(qname) LIKE @qn"); + cmd.Parameters.Add(new DuckDBParameter("@qn", $"%{qname.ToLowerInvariant()}%")); + } + + string whereSql = filters.Count > 0 + ? " WHERE " + string.Join(" AND ", filters) + : string.Empty; + + // ----- count ----- + + cmd.CommandText = $"SELECT COUNT(*) FROM {UNIFIED_VIEW} {whereSql}"; + long totalEntries = Convert.ToInt64(await cmd.ExecuteScalarAsync()); + long totalPages = (long)Math.Ceiling((double)totalEntries / entriesPerPage); + pageNumber = Math.Clamp(pageNumber, 1, Math.Max(1, totalPages)); + + // ----- data query ----- + + cmd.CommandText = $@" + SELECT + timestamp, -- 0 + client_ip, -- 1 + protocol, -- 2 (UTINYINT, may be NULL) + response_type, -- 3 (UTINYINT, may be NULL) + response_rtt, -- 4 (DOUBLE, may be NULL) + rcode, -- 5 (UTINYINT, may be NULL) + qname, -- 6 (TEXT, may be NULL) + qtype, -- 7 (USMALLINT, may be NULL) + qclass, -- 8 (USMALLINT, may be NULL) + answer -- 9 (TEXT, may be NULL) + FROM {UNIFIED_VIEW} + {whereSql} + ORDER BY timestamp {(descendingOrder ? "DESC" : "ASC")} + LIMIT {entriesPerPage} + OFFSET {(pageNumber - 1) * entriesPerPage} + "; + + + var entries = new List(); + + using var reader = await cmd.ExecuteReaderAsync(); + + while (await reader.ReadAsync()) + { + DateTime timestamp = reader.GetDateTime(0); + + IPAddress clientIp = + reader.IsDBNull(1) + ? IPAddress.None + : IPAddress.Parse(reader.GetString(1)); + + // protocol (nullable in DB) + DnsTransportProtocol? proto = + reader.IsDBNull(2) + ? null + : SafeEnum(reader.GetByte(2)); + + // response_type (nullable in DB) + DnsServerResponseType? respType = + reader.IsDBNull(3) + ? null + : SafeEnum(reader.GetByte(3)); + + // rtt (nullable) + double? rtt = + reader.IsDBNull(4) ? null : reader.GetDouble(4); + + // rcode (nullable in DB) + DnsResponseCode? respCode = + reader.IsDBNull(5) + ? null + : SafeEnum(reader.GetByte(5)); + + string? qn = reader.IsDBNull(6) ? null : reader.GetString(6); + + DnsResourceRecordType? qt = + reader.IsDBNull(7) + ? null + : SafeEnum((ushort)reader.GetInt16(7)); + + DnsClass? qc = + reader.IsDBNull(8) + ? null + : SafeEnum((ushort)reader.GetInt16(8)); + + string? answer = + reader.IsDBNull(9) ? null : reader.GetString(9); + + DnsQuestionRecord? question = null; + + if (qn is not null && qt.HasValue && qc.HasValue) + { + question = new DnsQuestionRecord( + qn, + qt.Value, + qc.Value, + false + ); + } + + // DnsLogEntry takes NON-nullable enums → we must provide defaults + entries.Add( + new DnsLogEntry( + 0, + timestamp, + clientIp, + proto ?? default, // safe fallback + respType ?? default, // ← avoids InvalidCastException + rtt ?? 0d, + respCode ?? default, + question, + answer + ) + ); + } + + return new DnsLogPage(pageNumber, totalPages, totalEntries, entries); + } + + private static TEnum? SafeEnum(TRaw? value) + where TEnum : struct, Enum + where TRaw : struct, IConvertible + { + if (value is null) return null; + + // normalize value to UInt64 for comparison + ulong v = Convert.ToUInt64(value); + + // compare against enum values (normalized) + foreach (var ev in Enum.GetValues(typeof(TEnum))) + { + if (Convert.ToUInt64(ev) == v) + return (TEnum)Enum.ToObject(typeof(TEnum), v); + } + + // not a valid enum member + return null; + } + + + #endregion public + + #region properties + + public string Description + { get { return "Logs DNS requests to DuckDB using Parquet for high-performance analytical storage."; } } + + #endregion properties + + private readonly struct LogEntry + { + #region variables + + public readonly DateTime Timestamp; + public readonly DnsDatagram Request; + public readonly IPEndPoint RemoteEP; + public readonly DnsTransportProtocol Protocol; + public readonly DnsDatagram Response; + + #endregion variables + + #region constructor + + public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response) + { + Timestamp = timestamp; + Request = request; + RemoteEP = remoteEP; + Protocol = protocol; + Response = response; + } + + #endregion constructor + } + } +} \ No newline at end of file diff --git a/Apps/QueryLogsDuckDBApp/QueryLogsDuckDBApp.csproj b/Apps/QueryLogsDuckDBApp/QueryLogsDuckDBApp.csproj new file mode 100644 index 000000000..f706684fc --- /dev/null +++ b/Apps/QueryLogsDuckDBApp/QueryLogsDuckDBApp.csproj @@ -0,0 +1,48 @@ + + + + net9.0 + false + true + 1.0 + false + Technitium + Technitium DNS Server + Zafer Balkan + QueryLogsDuckDBApp + QueryLogsDuckDB + https://technitium.com/dns/ + https://github.com/TechnitiumSoftware/DnsServer + Logs all incoming DNS requests and their responses in a DuckDB database that can be queried from the DNS Server web console. + false + Library + enable + + + + + + + + + false + + + + + + ..\..\..\TechnitiumLibrary\bin\TechnitiumLibrary.Net.dll + false + + + ..\..\..\TechnitiumLibrary\bin\TechnitiumLibrary.dll + false + + + + + + PreserveNewest + + + diff --git a/Apps/QueryLogsDuckDBApp/dnsApp.config b/Apps/QueryLogsDuckDBApp/dnsApp.config new file mode 100644 index 000000000..1efee9043 --- /dev/null +++ b/Apps/QueryLogsDuckDBApp/dnsApp.config @@ -0,0 +1,4 @@ +{ + "enableLogging": true, + "dbPath": "querylogs.db" +} diff --git a/DnsServer.sln b/DnsServer.sln index 0a2a6756d..976abfe59 100644 --- a/DnsServer.sln +++ b/DnsServer.sln @@ -68,8 +68,11 @@ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QueryLogsMySqlApp", "Apps\QueryLogsMySqlApp\QueryLogsMySqlApp.csproj", "{699E2A1D-D917-4825-939E-65CDB2B16A96}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MispConnectorApp", "Apps\MispConnectorApp\MispConnectorApp.csproj", "{83C8180A-0F86-F9A0-8F41-6FD61FAC41CB}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DnsServerCore.HttpApi", "DnsServerCore.HttpApi\DnsServerCore.HttpApi.csproj", "{1A49D371-D08C-475E-B7A2-6E8ECD181FD6}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QueryLogsDuckDBApp", "Apps\QueryLogsDuckDBApp\QueryLogsDuckDBApp.csproj", "{B4F714DB-B90F-467A-9DD1-4F944A84B4F2}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -208,6 +211,10 @@ Global {1A49D371-D08C-475E-B7A2-6E8ECD181FD6}.Debug|Any CPU.Build.0 = Debug|Any CPU {1A49D371-D08C-475E-B7A2-6E8ECD181FD6}.Release|Any CPU.ActiveCfg = Release|Any CPU {1A49D371-D08C-475E-B7A2-6E8ECD181FD6}.Release|Any CPU.Build.0 = Release|Any CPU + {B4F714DB-B90F-467A-9DD1-4F944A84B4F2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B4F714DB-B90F-467A-9DD1-4F944A84B4F2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B4F714DB-B90F-467A-9DD1-4F944A84B4F2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B4F714DB-B90F-467A-9DD1-4F944A84B4F2}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -240,6 +247,7 @@ Global {6F655C97-FD43-4FE1-B15A-6C783D2D91C9} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} {699E2A1D-D917-4825-939E-65CDB2B16A96} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} {83C8180A-0F86-F9A0-8F41-6FD61FAC41CB} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} + {B4F714DB-B90F-467A-9DD1-4F944A84B4F2} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {6747BB6D-2826-4356-A213-805FBCCF9201} From ff4b82eefdff65698da679b3f59ace68abc1e751 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Mon, 5 Jan 2026 21:40:16 +0300 Subject: [PATCH 02/49] Simplified the database access --- Apps/QueryLogsDuckDBApp/App.cs | 533 +++++++++++++-------------------- 1 file changed, 206 insertions(+), 327 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index d5a8e4901..73873363f 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -21,8 +21,6 @@ You should have received a copy of the GNU General Public License using DuckDB.NET.Data; using System; using System.Collections.Generic; -using System.Data; -using System.IO; using System.Linq; using System.Net; using System.Text.Json; @@ -38,20 +36,14 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs { #region variables - private IDnsServer _dnsServer; - private DuckDBConnection _conn; - private string _parquetPath; - private string _dbPath; - - private bool _enableLogging; - private int _bufferedRows = 0; - private const int MAX_BATCH_SIZE = 10000; - private const string BUFFER_TABLE = "dns_buffer"; - private const string UNIFIED_VIEW = "dns_logs"; - + private const int CHANNEL_CAPACITY = 200_000; + private const int MAX_BATCH_SIZE = 10_000; private Channel _channel; + private DuckDBConnection _conn; private Task _consumerTask; private bool _disposed; + private IDnsServer _dnsServer; + private bool _enableLogging; #endregion variables @@ -59,11 +51,13 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs public void Dispose() { - if (_disposed) return; - _channel?.Writer.TryComplete(); - _consumerTask?.Wait(5000); - FlushToParquetAsync().GetAwaiter().GetResult(); - _conn?.Dispose(); + if (_disposed) + return; + + try { _channel?.Writer.TryComplete(); } catch { } + try { _consumerTask?.Wait(5000); } catch { } + try { _conn?.Dispose(); } catch { } + _disposed = true; } @@ -71,135 +65,147 @@ public void Dispose() #region private - private async Task RefreshViewAsync() + private static string? FormatAnswer(DnsDatagram resp) { - string parquetSource = File.Exists(_parquetPath) - ? $"read_parquet('{_parquetPath}')" - : $"(SELECT * FROM {BUFFER_TABLE} WHERE 1=0)"; - - using var cmd = _conn.CreateCommand(); - cmd.CommandText = $"CREATE OR REPLACE VIEW {UNIFIED_VIEW} AS SELECT * FROM {BUFFER_TABLE} UNION ALL SELECT * FROM {parquetSource};"; - await cmd.ExecuteNonQueryAsync(); - } + if (resp.Answer.Count == 0) + return null; - private async Task ProcessLogsAsync() - { - var batch = new List(MAX_BATCH_SIZE); - while (await _channel.Reader.WaitToReadAsync()) - { - while (batch.Count < MAX_BATCH_SIZE && _channel.Reader.TryRead(out var log)) - { - batch.Add(log); - } + if (resp.Answer.Count > 2 && resp.IsZoneTransfer) + return "[ZONE TRANSFER]"; - if (batch.Count > 0) - { - await BulkInsertInternalAsync(batch); - batch.Clear(); - } - } + return string.Join(", ", + resp.Answer.Select(r => $"{r.Type} {r.RDATA}")); } - private async Task BulkInsertInternalAsync(List logs) + private async Task BulkInsertAsync(List logs) { try { - using (var appender = _conn.CreateAppender(BUFFER_TABLE)) - { - foreach (var log in logs) - { - if (log.Request is null || log.Response is null) - continue; // skip corrupt entries defensively + using var appender = _conn.CreateAppender("dns_logs"); - var question = log.Request.Question?[0]; - var row = appender.CreateRow(); + foreach (var log in logs) + { + if (log.Request is null || log.Response is null) + continue; - // RTT is only meaningful when Metadata exists and Tag is null - double? rtt = - (log.Response.Tag is null && log.Response.Metadata is not null) - ? log.Response.Metadata.RoundTripTime + var question = + log.Request.Question.Count > 0 + ? log.Request.Question[0] : null; - // Nullable values prepared up front - byte? tag = - log.Response.Tag is null ? (byte?)null : ((byte)log.Response.Tag); - - byte rcode = (byte)log.Response.RCODE; - - string? qname = - question?.Name is null ? null : question.Name.ToLowerInvariant(); - - ushort? qtype = - question is null ? (ushort?)null : (ushort)question.Type; + double? rtt = + (log.Response.Tag is null && log.Response.Metadata is not null) + ? log.Response.Metadata.RoundTripTime + : null; - ushort? qclass = - question is null ? (ushort?)null : (ushort)question.Class; + var row = appender.CreateRow(); - // Append values — emit NULLs where appropriate - row.AppendValue(log.Timestamp); + row.AppendValue(_dnsServer.ServerDomain); + row.AppendValue(log.Timestamp); + row.AppendValue(log.RemoteEP.Address.ToString()); + row.AppendValue((byte)log.Protocol); - row.AppendValue( - log.RemoteEP?.Address is null - ? null - : log.RemoteEP.Address.ToString()); + if (log.Response.Tag is null) + row.AppendNullValue(); + else + row.AppendValue((byte)log.Response.Tag); - row.AppendValue((byte)log.Protocol); + if (rtt is null) + row.AppendNullValue(); + else + row.AppendValue(rtt.Value); - if (tag is null) row.AppendNullValue(); else row.AppendValue(tag.Value); + row.AppendValue((byte)log.Response.RCODE); - if (rtt is null) row.AppendNullValue(); else row.AppendValue(rtt.Value); - - row.AppendValue(rcode); - - if (qname is null) row.AppendNullValue(); else row.AppendValue(qname); - - if (qtype is null) row.AppendNullValue(); else row.AppendValue(qtype.Value); - - if (qclass is null) row.AppendNullValue(); else row.AppendValue(qclass.Value); - - var answer = FormatAnswer(log.Response); - if (answer is null) row.AppendNullValue(); else row.AppendValue(answer); - - row.EndRow(); - - _bufferedRows++; + if (question is null) + { + row.AppendNullValue(); + row.AppendNullValue(); + row.AppendNullValue(); + } + else + { + row.AppendValue(question.Name.ToLowerInvariant()); + row.AppendValue((ushort)question.Type); + row.AppendValue((ushort)question.Class); } - } + var answer = FormatAnswer(log.Response); + if (answer is null) + row.AppendNullValue(); + else + row.AppendValue(answer); - if (_bufferedRows >= MAX_BATCH_SIZE) - { - await FlushToParquetAsync(); + row.EndRow(); } } - catch (Exception ex) { _dnsServer.WriteLog(ex); } + catch (Exception ex) + { + _dnsServer?.WriteLog(ex); + } } - private string? FormatAnswer(DnsDatagram response) + private async Task CreateSchemaAsync() { - if (response.Answer.Count == 0) return null; - return string.Join(", ", response.Answer.Select(r => $"{r.Type} {r.RDATA}")); + using DuckDBCommand cmd = _conn.CreateCommand(); + + cmd.CommandText = @" +CREATE TABLE IF NOT EXISTS dns_logs ( + server VARCHAR(255) NOT NULL, + timestamp TIMESTAMP NOT NULL, + client_ip VARCHAR(39) NOT NULL, + protocol UTINYINT NOT NULL, + response_type UTINYINT, + response_rtt DOUBLE, + rcode UTINYINT NOT NULL, + qname VARCHAR(255), + qtype USMALLINT, + qclass USMALLINT, + answer TEXT +);"; + await cmd.ExecuteNonQueryAsync(); + + string[] indexes = + [ + "CREATE INDEX IF NOT EXISTS idx_srv ON dns_logs(server);", + "CREATE INDEX IF NOT EXISTS idx_ts ON dns_logs(timestamp);", + "CREATE INDEX IF NOT EXISTS idx_ip ON dns_logs(client_ip);", + "CREATE INDEX IF NOT EXISTS idx_proto ON dns_logs(protocol);", + "CREATE INDEX IF NOT EXISTS idx_resp ON dns_logs(response_type);", + "CREATE INDEX IF NOT EXISTS idx_rcode ON dns_logs(rcode);", + "CREATE INDEX IF NOT EXISTS idx_qname ON dns_logs(qname);", + "CREATE INDEX IF NOT EXISTS idx_qtype ON dns_logs(qtype);", + "CREATE INDEX IF NOT EXISTS idx_qclass ON dns_logs(qclass);" + ]; + + foreach (string sql in indexes) + { + cmd.CommandText = sql; + await cmd.ExecuteNonQueryAsync(); + } } - private async Task FlushToParquetAsync() + private async Task ProcessLogsAsync() { - if (_bufferedRows == 0) return; - string tempFile = _parquetPath + ".tmp"; + var batch = new List(MAX_BATCH_SIZE); - using (var cmd = _conn.CreateCommand()) + while (await _channel.Reader.WaitToReadAsync()) { - cmd.CommandText = $"COPY (SELECT * FROM {UNIFIED_VIEW} ORDER BY timestamp ASC) TO '{tempFile}' (FORMAT PARQUET, COMPRESSION 'ZSTD');"; - await cmd.ExecuteNonQueryAsync(); + while (batch.Count < MAX_BATCH_SIZE && + _channel.Reader.TryRead(out var log)) + { + batch.Add(log); + } - cmd.CommandText = $"DELETE FROM {BUFFER_TABLE};"; - await cmd.ExecuteNonQueryAsync(); + if (batch.Count > 0) + { + await BulkInsertAsync(batch); + batch.Clear(); + } } - if (File.Exists(_parquetPath)) File.Delete(_parquetPath); - File.Move(tempFile, _parquetPath); - - _bufferedRows = 0; - await RefreshViewAsync(); + if (batch.Count > 0) + await BulkInsertAsync(batch); } #endregion private @@ -210,272 +216,145 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) { _dnsServer = dnsServer; - using JsonDocument jsonDocument = JsonDocument.Parse(config); - JsonElement jsonConfig = jsonDocument.RootElement; - - _enableLogging = jsonConfig.GetPropertyValue("enableLogging", true); - string dbFileName = jsonConfig.GetPropertyValue("dbPath", "querylogs.duckdb"); - _dbPath = Path.IsPathRooted(dbFileName) ? dbFileName : Path.Combine(_dnsServer.ApplicationFolder, dbFileName); - _parquetPath = Path.ChangeExtension(_dbPath, ".parquet"); + using JsonDocument json = JsonDocument.Parse(config); + JsonElement cfg = json.RootElement; - // Initialize DuckDB Connection - _conn = new DuckDBConnection($"Data Source={_dbPath}"); - await _conn.OpenAsync(); + _enableLogging = cfg.GetPropertyValue("enableLogging", true); - // Load plugin - using (var cmd = _conn.CreateCommand()) - { - cmd.CommandText = "INSTALL parquet;LOAD parquet;"; - await cmd.ExecuteNonQueryAsync(); - } + string dbPath = cfg.GetPropertyValue("dbPath", "querylogs.db"); - // Setup Schema - using (var cmd = _conn.CreateCommand()) - { - cmd.CommandText = $@" - CREATE TEMP TABLE IF NOT EXISTS {BUFFER_TABLE} ( - timestamp TIMESTAMP, - client_ip VARCHAR(39), - protocol UTINYINT, - response_type UTINYINT, - response_rtt DOUBLE, - rcode UTINYINT, - qname VARCHAR(255), - qtype USMALLINT, - qclass USMALLINT, - answer TEXT - );"; - await cmd.ExecuteNonQueryAsync(); - } + if (!System.IO.Path.IsPathRooted(dbPath)) + dbPath = System.IO.Path.Combine(dnsServer.ApplicationFolder, dbPath); - await RefreshViewAsync(); + _channel = Channel.CreateBounded( + new BoundedChannelOptions(CHANNEL_CAPACITY) + { + SingleReader = true, + FullMode = BoundedChannelFullMode.DropWrite + }); - // Start Producer-Consumer Channel - _channel = Channel.CreateBounded(new BoundedChannelOptions(200000) - { - SingleReader = true, - FullMode = BoundedChannelFullMode.DropWrite - }); + _conn = new DuckDBConnection($"Data Source={dbPath}"); + await _conn.OpenAsync(); + await CreateSchemaAsync(); _consumerTask = Task.Run(ProcessLogsAsync); } - public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response) + public Task InsertLogAsync( + DateTime timestamp, + DnsDatagram request, + IPEndPoint remoteEP, + DnsTransportProtocol protocol, + DnsDatagram response) { if (_enableLogging) - _channel.Writer.TryWrite(new LogEntry(timestamp, request, remoteEP, protocol, response)); + _channel.Writer.TryWrite( + new LogEntry(timestamp, request, remoteEP, protocol, response)); return Task.CompletedTask; } public async Task QueryLogsAsync( - long pageNumber, - int entriesPerPage, - bool descendingOrder, - DateTime? start, - DateTime? end, - IPAddress clientIpAddress, - DnsTransportProtocol? protocol, - DnsServerResponseType? responseType, - DnsResponseCode? rcode, - string qname, - DnsResourceRecordType? qtype, - DnsClass? qclass) + long pageNumber, + int entriesPerPage, + bool descendingOrder, + DateTime? start, + DateTime? end, + IPAddress clientIpAddress, + DnsTransportProtocol? protocol, + DnsServerResponseType? responseType, + DnsResponseCode? rcode, + string qname, + DnsResourceRecordType? qtype, + DnsClass? qclass) { - using var cmd = _conn.CreateCommand(); - var filters = new List(); - - // ----- filters ----- - - if (start.HasValue) - { - filters.Add("timestamp >= @start"); - cmd.Parameters.Add(new DuckDBParameter("@start", start.Value)); - } - - if (end.HasValue) - { - filters.Add("timestamp <= @end"); - cmd.Parameters.Add(new DuckDBParameter("@end", end.Value)); - } - - if (clientIpAddress is not null) - { - filters.Add("client_ip = @ip"); - cmd.Parameters.Add(new DuckDBParameter("@ip", clientIpAddress.ToString())); - } - - if (protocol.HasValue) - { - filters.Add("protocol = @p"); - cmd.Parameters.Add(new DuckDBParameter("@p", (byte)protocol.Value)); - } + using DuckDBCommand cmd = _conn.CreateCommand(); - if (responseType.HasValue) - { - filters.Add("response_type = @rt"); - cmd.Parameters.Add(new DuckDBParameter("@rt", (byte)responseType.Value)); - } - - if (rcode.HasValue) - { - filters.Add("rcode = @rc"); - cmd.Parameters.Add(new DuckDBParameter("@rc", (byte)rcode.Value)); - } - - if (qtype.HasValue) - { - filters.Add("qtype = @qt"); - cmd.Parameters.Add(new DuckDBParameter("@qt", (ushort)qtype.Value)); - } + List filters = new List(); - if (qclass.HasValue) + if (start is not null) { - filters.Add("qclass = @qc"); - cmd.Parameters.Add(new DuckDBParameter("@qc", (ushort)qclass.Value)); + filters.Add("timestamp >= @s"); + cmd.Parameters.Add(new DuckDBParameter("@s", start)); } - if (!string.IsNullOrWhiteSpace(qname)) + if (end is not null) { - filters.Add("LOWER(qname) LIKE @qn"); - cmd.Parameters.Add(new DuckDBParameter("@qn", $"%{qname.ToLowerInvariant()}%")); + filters.Add("timestamp <= @e"); + cmd.Parameters.Add(new DuckDBParameter("@e", end)); } string whereSql = filters.Count > 0 - ? " WHERE " + string.Join(" AND ", filters) + ? " WHERE " + string.Join(" AND ", filters) : string.Empty; - // ----- count ----- - - cmd.CommandText = $"SELECT COUNT(*) FROM {UNIFIED_VIEW} {whereSql}"; + cmd.CommandText = $"SELECT count() FROM dns_logs {whereSql}"; long totalEntries = Convert.ToInt64(await cmd.ExecuteScalarAsync()); - long totalPages = (long)Math.Ceiling((double)totalEntries / entriesPerPage); - pageNumber = Math.Clamp(pageNumber, 1, Math.Max(1, totalPages)); - // ----- data query ----- + long totalPages = + (long)Math.Ceiling((double)totalEntries / entriesPerPage); + + pageNumber = Math.Clamp(pageNumber, 1, Math.Max(1, totalPages)); cmd.CommandText = $@" - SELECT - timestamp, -- 0 - client_ip, -- 1 - protocol, -- 2 (UTINYINT, may be NULL) - response_type, -- 3 (UTINYINT, may be NULL) - response_rtt, -- 4 (DOUBLE, may be NULL) - rcode, -- 5 (UTINYINT, may be NULL) - qname, -- 6 (TEXT, may be NULL) - qtype, -- 7 (USMALLINT, may be NULL) - qclass, -- 8 (USMALLINT, may be NULL) - answer -- 9 (TEXT, may be NULL) - FROM {UNIFIED_VIEW} - {whereSql} - ORDER BY timestamp {(descendingOrder ? "DESC" : "ASC")} - LIMIT {entriesPerPage} - OFFSET {(pageNumber - 1) * entriesPerPage} - "; - - - var entries = new List(); - - using var reader = await cmd.ExecuteReaderAsync(); +SELECT server, timestamp, client_ip, protocol, response_type, + response_rtt, rcode, qname, qtype, qclass, answer +FROM dns_logs +{whereSql} +ORDER BY timestamp {(descendingOrder ? "DESC" : "ASC")} +LIMIT {entriesPerPage} +OFFSET {(pageNumber - 1) * entriesPerPage}"; - while (await reader.ReadAsync()) - { - DateTime timestamp = reader.GetDateTime(0); + List list = new List(); - IPAddress clientIp = - reader.IsDBNull(1) - ? IPAddress.None - : IPAddress.Parse(reader.GetString(1)); + using System.Data.Common.DbDataReader reader = await cmd.ExecuteReaderAsync(); - // protocol (nullable in DB) - DnsTransportProtocol? proto = - reader.IsDBNull(2) - ? null - : SafeEnum(reader.GetByte(2)); + while (await reader.ReadAsync()) + { + var server = reader.GetString(0); + DateTime ts = reader.GetDateTime(1); + IPAddress ip = IPAddress.Parse(reader.GetString(2)); + DnsTransportProtocol proto = (DnsTransportProtocol)reader.GetByte(3); - // response_type (nullable in DB) - DnsServerResponseType? respType = - reader.IsDBNull(3) - ? null - : SafeEnum(reader.GetByte(3)); + DnsServerResponseType respType = + reader.IsDBNull(4) + ? default + : (DnsServerResponseType)reader.GetByte(4); - // rtt (nullable) double? rtt = - reader.IsDBNull(4) ? null : reader.GetDouble(4); - - // rcode (nullable in DB) - DnsResponseCode? respCode = reader.IsDBNull(5) ? null - : SafeEnum(reader.GetByte(5)); - - string? qn = reader.IsDBNull(6) ? null : reader.GetString(6); - - DnsResourceRecordType? qt = - reader.IsDBNull(7) - ? null - : SafeEnum((ushort)reader.GetInt16(7)); + : reader.GetDouble(5); - DnsClass? qc = - reader.IsDBNull(8) - ? null - : SafeEnum((ushort)reader.GetInt16(8)); + DnsResponseCode rc = (DnsResponseCode)reader.GetByte(6); - string? answer = - reader.IsDBNull(9) ? null : reader.GetString(9); + string? qn = + reader.IsDBNull(7) ? null : reader.GetString(7); DnsQuestionRecord? question = null; - if (qn is not null && qt.HasValue && qc.HasValue) + if (!reader.IsDBNull(8) && + !reader.IsDBNull(9) && + qn is not null) { question = new DnsQuestionRecord( qn, - qt.Value, - qc.Value, - false - ); + (DnsResourceRecordType)reader.GetInt16(8), + (DnsClass)reader.GetInt16(9), + false); } - // DnsLogEntry takes NON-nullable enums → we must provide defaults - entries.Add( - new DnsLogEntry( - 0, - timestamp, - clientIp, - proto ?? default, // safe fallback - respType ?? default, // ← avoids InvalidCastException - rtt ?? 0d, - respCode ?? default, - question, - answer - ) - ); - } - - return new DnsLogPage(pageNumber, totalPages, totalEntries, entries); - } - - private static TEnum? SafeEnum(TRaw? value) - where TEnum : struct, Enum - where TRaw : struct, IConvertible - { - if (value is null) return null; - - // normalize value to UInt64 for comparison - ulong v = Convert.ToUInt64(value); + string? ans = + reader.IsDBNull(10) ? null : reader.GetString(10); - // compare against enum values (normalized) - foreach (var ev in Enum.GetValues(typeof(TEnum))) - { - if (Convert.ToUInt64(ev) == v) - return (TEnum)Enum.ToObject(typeof(TEnum), v); + list.Add( + new DnsLogEntry( + 0, ts, ip, proto, respType, rtt, rc, question, ans)); } - // not a valid enum member - return null; + return new DnsLogPage(pageNumber, totalPages, totalEntries, list); } - #endregion public #region properties @@ -489,11 +368,11 @@ private readonly struct LogEntry { #region variables - public readonly DateTime Timestamp; - public readonly DnsDatagram Request; - public readonly IPEndPoint RemoteEP; public readonly DnsTransportProtocol Protocol; + public readonly IPEndPoint RemoteEP; + public readonly DnsDatagram Request; public readonly DnsDatagram Response; + public readonly DateTime Timestamp; #endregion variables From ce2a3df96ba95fdbe15d439e5163bb7b6f214652 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Mon, 5 Jan 2026 22:13:11 +0300 Subject: [PATCH 03/49] Fine-tuned default values --- Apps/QueryLogsDuckDBApp/App.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 73873363f..97cda23f8 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -36,8 +36,8 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs { #region variables - private const int CHANNEL_CAPACITY = 200_000; - private const int MAX_BATCH_SIZE = 10_000; + private const int CHANNEL_CAPACITY = 20_000; + private const int MAX_BATCH_SIZE = 1000; private Channel _channel; private DuckDBConnection _conn; private Task _consumerTask; @@ -77,7 +77,7 @@ public void Dispose() resp.Answer.Select(r => $"{r.Type} {r.RDATA}")); } - private async Task BulkInsertAsync(List logs) + private void BulkInsert(List logs) { try { @@ -199,13 +199,13 @@ private async Task ProcessLogsAsync() if (batch.Count > 0) { - await BulkInsertAsync(batch); + BulkInsert(batch); batch.Clear(); } } if (batch.Count > 0) - await BulkInsertAsync(batch); + BulkInsert(batch); } #endregion private From 35097420e11e48e9d0c4e5819fa15813cd7142e1 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Tue, 27 Jan 2026 17:34:00 +0200 Subject: [PATCH 04/49] Used a single index command for simpicity --- Apps/QueryLogsDuckDBApp/App.cs | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 97cda23f8..b499023c6 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -165,24 +165,9 @@ answer TEXT );"; await cmd.ExecuteNonQueryAsync(); - string[] indexes = - [ - "CREATE INDEX IF NOT EXISTS idx_srv ON dns_logs(server);", - "CREATE INDEX IF NOT EXISTS idx_ts ON dns_logs(timestamp);", - "CREATE INDEX IF NOT EXISTS idx_ip ON dns_logs(client_ip);", - "CREATE INDEX IF NOT EXISTS idx_proto ON dns_logs(protocol);", - "CREATE INDEX IF NOT EXISTS idx_resp ON dns_logs(response_type);", - "CREATE INDEX IF NOT EXISTS idx_rcode ON dns_logs(rcode);", - "CREATE INDEX IF NOT EXISTS idx_qname ON dns_logs(qname);", - "CREATE INDEX IF NOT EXISTS idx_qtype ON dns_logs(qtype);", - "CREATE INDEX IF NOT EXISTS idx_qclass ON dns_logs(qclass);" - ]; - - foreach (string sql in indexes) - { - cmd.CommandText = sql; - await cmd.ExecuteNonQueryAsync(); - } + string index = "CREATE INDEX IF NOT EXISTS idx_ts_srv_ip ON dns_logs(timestamp, server, client_ip);"; + cmd.CommandText = index; + await cmd.ExecuteNonQueryAsync(); } private async Task ProcessLogsAsync() From 02271d03fa05bc9b922b4b2b9f3fd07110ce93b5 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Tue, 27 Jan 2026 17:38:10 +0200 Subject: [PATCH 05/49] Used Dispose pattern for disposing the app --- Apps/QueryLogsDuckDBApp/App.cs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index b499023c6..61f5e42be 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -44,23 +44,31 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs private bool _disposed; private IDnsServer _dnsServer; private bool _enableLogging; - #endregion variables #region IDisposable public void Dispose() { - if (_disposed) - return; + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } - try { _channel?.Writer.TryComplete(); } catch { } - try { _consumerTask?.Wait(5000); } catch { } - try { _conn?.Dispose(); } catch { } + private void Dispose(bool disposing) + { + if (!_disposed) + { + if (disposing) + { + try { _channel?.Writer.TryComplete(); } catch { } + try { _consumerTask?.Wait(5000); } catch { } + try { _conn?.Dispose(); } catch { } + } - _disposed = true; + _disposed = true; + } } - #endregion IDisposable #region private From 616cac05466bf6e24dc0dff37127d48509a38bfd Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Tue, 27 Jan 2026 21:38:57 +0200 Subject: [PATCH 06/49] Added SingleWriter --- Apps/QueryLogsDuckDBApp/App.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 61f5e42be..03a7232fb 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -223,6 +223,7 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) new BoundedChannelOptions(CHANNEL_CAPACITY) { SingleReader = true, + SingleWriter = true, FullMode = BoundedChannelFullMode.DropWrite }); From f4f8f9401d51031e1fb8eee9c6a1cfb4e187ec29 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Tue, 27 Jan 2026 21:40:30 +0200 Subject: [PATCH 07/49] Updated Description --- Apps/QueryLogsDuckDBApp/App.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 03a7232fb..d78463e98 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -354,7 +354,7 @@ FROM dns_logs #region properties public string Description - { get { return "Logs DNS requests to DuckDB using Parquet for high-performance analytical storage."; } } + { get { return "Logs all incoming DNS requests and their responses in a DuckDB database that can be queried from the DNS Server web console."; } } #endregion properties From 72cb9a1bb5991de6177fce2dc7aa08d697f76209 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Tue, 27 Jan 2026 21:46:12 +0200 Subject: [PATCH 08/49] Made maxQueueSize configurable --- Apps/QueryLogsDuckDBApp/App.cs | 6 ++++-- Apps/QueryLogsDuckDBApp/dnsApp.config | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index d78463e98..9825c1d03 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -36,7 +36,6 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs { #region variables - private const int CHANNEL_CAPACITY = 20_000; private const int MAX_BATCH_SIZE = 1000; private Channel _channel; private DuckDBConnection _conn; @@ -44,6 +43,7 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs private bool _disposed; private IDnsServer _dnsServer; private bool _enableLogging; + private int _maxQueueSize; #endregion variables #region IDisposable @@ -216,11 +216,13 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) string dbPath = cfg.GetPropertyValue("dbPath", "querylogs.db"); + _maxQueueSize = cfg.GetPropertyValue("maxQueueSize", 20000); + if (!System.IO.Path.IsPathRooted(dbPath)) dbPath = System.IO.Path.Combine(dnsServer.ApplicationFolder, dbPath); _channel = Channel.CreateBounded( - new BoundedChannelOptions(CHANNEL_CAPACITY) + new BoundedChannelOptions(_maxQueueSize) { SingleReader = true, SingleWriter = true, diff --git a/Apps/QueryLogsDuckDBApp/dnsApp.config b/Apps/QueryLogsDuckDBApp/dnsApp.config index 1efee9043..2ff1b3162 100644 --- a/Apps/QueryLogsDuckDBApp/dnsApp.config +++ b/Apps/QueryLogsDuckDBApp/dnsApp.config @@ -1,4 +1,5 @@ { "enableLogging": true, - "dbPath": "querylogs.db" + "dbPath": "querylogs.db", + "maxQueueSize": 20000 } From 0678d59c557cd85b169abbb0ffb465bb0c5ec56c Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Tue, 27 Jan 2026 21:49:58 +0200 Subject: [PATCH 09/49] Added connection close on Dispose --- Apps/QueryLogsDuckDBApp/App.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 9825c1d03..c0354babf 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -63,7 +63,7 @@ private void Dispose(bool disposing) { try { _channel?.Writer.TryComplete(); } catch { } try { _consumerTask?.Wait(5000); } catch { } - try { _conn?.Dispose(); } catch { } + try { _conn?.Close(); _conn?.Dispose(); } catch { } } _disposed = true; From ec85d6c2b5cdfb47680e6bfb27396f73431d6109 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 13:31:39 +0200 Subject: [PATCH 10/49] Fixed missing query parameters --- Apps/QueryLogsDuckDBApp/App.cs | 90 ++++++++++++++++++++++++++++++---- 1 file changed, 80 insertions(+), 10 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index c0354babf..afd2b860a 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -270,20 +270,71 @@ public async Task QueryLogsAsync( if (start is not null) { - filters.Add("timestamp >= @s"); - cmd.Parameters.Add(new DuckDBParameter("@s", start)); + filters.Add("timestamp >= $s"); + cmd.Parameters.Add(new DuckDBParameter("s", start)); } if (end is not null) { - filters.Add("timestamp <= @e"); - cmd.Parameters.Add(new DuckDBParameter("@e", end)); + filters.Add("timestamp <= $e"); + cmd.Parameters.Add(new DuckDBParameter("e", end)); } - string whereSql = filters.Count > 0 + if (clientIpAddress is not null) + { + filters.Add("client_ip = $cip"); + cmd.Parameters.Add( + new DuckDBParameter("cip", clientIpAddress.ToString())); + } + + if (protocol is not null) + { + filters.Add("protocol = $proto"); + cmd.Parameters.Add( + new DuckDBParameter("proto", (byte)protocol.Value)); + } + + if (responseType is not null) + { + filters.Add("response_type = $rtype"); + cmd.Parameters.Add( + new DuckDBParameter("rtype", (byte)responseType.Value)); + } + + if (rcode is not null) + { + filters.Add("rcode = $rcode"); + cmd.Parameters.Add( + new DuckDBParameter("rcode", (byte)rcode.Value)); + } + + if (!string.IsNullOrWhiteSpace(qname)) + { + filters.Add("qname = $qname"); + cmd.Parameters.Add( + new DuckDBParameter("qname", qname)); + } + + if (qtype is not null) + { + filters.Add("qtype = $qtype"); + cmd.Parameters.Add( + new DuckDBParameter("qtype", (short)qtype.Value)); + } + + if (qclass is not null) + { + filters.Add("qclass = $qclass"); + cmd.Parameters.Add( + new DuckDBParameter("qclass", (short)qclass.Value)); + } + + string whereSql = + filters.Count > 0 ? " WHERE " + string.Join(" AND ", filters) - : string.Empty; + : string.Empty; + // Count query cmd.CommandText = $"SELECT count() FROM dns_logs {whereSql}"; long totalEntries = Convert.ToInt64(await cmd.ExecuteScalarAsync()); @@ -292,14 +343,33 @@ public async Task QueryLogsAsync( pageNumber = Math.Clamp(pageNumber, 1, Math.Max(1, totalPages)); + long offset = (pageNumber - 1) * entriesPerPage; + + //Pagination parameters + cmd.Parameters.Add( + new DuckDBParameter("limit", entriesPerPage)); + + cmd.Parameters.Add( + new DuckDBParameter("offset", offset)); + + // Main query cmd.CommandText = $@" -SELECT server, timestamp, client_ip, protocol, response_type, - response_rtt, rcode, qname, qtype, qclass, answer +SELECT server, + timestamp, + client_ip, + protocol, + response_type, + response_rtt, + rcode, + qname, + qtype, + qclass, + answer FROM dns_logs {whereSql} ORDER BY timestamp {(descendingOrder ? "DESC" : "ASC")} -LIMIT {entriesPerPage} -OFFSET {(pageNumber - 1) * entriesPerPage}"; +LIMIT $limit +OFFSET $offset"; List list = new List(); From abca7bea417c152f8a242f7aad6520af2e28e88a Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 14:26:25 +0200 Subject: [PATCH 11/49] Aligned bulk insert part with SQLite logs --- Apps/QueryLogsDuckDBApp/App.cs | 67 +++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index afd2b860a..85fa1e380 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -73,18 +73,6 @@ private void Dispose(bool disposing) #region private - private static string? FormatAnswer(DnsDatagram resp) - { - if (resp.Answer.Count == 0) - return null; - - if (resp.Answer.Count > 2 && resp.IsZoneTransfer) - return "[ZONE TRANSFER]"; - - return string.Join(", ", - resp.Answer.Select(r => $"{r.Type} {r.RDATA}")); - } - private void BulkInsert(List logs) { try @@ -101,11 +89,47 @@ private void BulkInsert(List logs) ? log.Request.Question[0] : null; - double? rtt = - (log.Response.Tag is null && log.Response.Metadata is not null) - ? log.Response.Metadata.RoundTripTime - : null; + //Response Type(Aligned) + DnsServerResponseType responseType; + + if (log.Response.Tag is null) + responseType = DnsServerResponseType.Recursive; + else + responseType = (DnsServerResponseType)log.Response.Tag; + + //RTT + double? rtt = null; + + if (responseType == DnsServerResponseType.Recursive && + log.Response.Metadata is not null) + { + rtt = log.Response.Metadata.RoundTripTime; + } + + //Answer (Aligned with SQLite) + string? answer = null; + + if (log.Response.Answer.Count == 0) + { + answer = null; + } + else if (log.Response.Answer.Count > 2 && + log.Response.IsZoneTransfer) + { + answer = "[ZONE TRANSFER]"; + } + else + { + foreach (var record in log.Response.Answer) + { + if (answer is null) + answer = record.Type + " " + record.RDATA; + else + answer += ", " + record.Type + " " + record.RDATA; + } + } + //Insert Row var row = appender.CreateRow(); row.AppendValue(_dnsServer.ServerDomain); @@ -113,10 +137,7 @@ private void BulkInsert(List logs) row.AppendValue(log.RemoteEP.Address.ToString()); row.AppendValue((byte)log.Protocol); - if (log.Response.Tag is null) - row.AppendNullValue(); - else - row.AppendValue((byte)log.Response.Tag); + row.AppendValue((byte)responseType); if (rtt is null) row.AppendNullValue(); @@ -138,7 +159,6 @@ private void BulkInsert(List logs) row.AppendValue((ushort)question.Class); } - var answer = FormatAnswer(log.Response); if (answer is null) row.AppendNullValue(); else @@ -382,10 +402,7 @@ FROM dns_logs IPAddress ip = IPAddress.Parse(reader.GetString(2)); DnsTransportProtocol proto = (DnsTransportProtocol)reader.GetByte(3); - DnsServerResponseType respType = - reader.IsDBNull(4) - ? default - : (DnsServerResponseType)reader.GetByte(4); + DnsServerResponseType respType = (DnsServerResponseType)reader.GetByte(4); double? rtt = reader.IsDBNull(5) From e0a853333e47798ce97e0b23a1beb1bad3429d6c Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 15:24:16 +0200 Subject: [PATCH 12/49] Added defensive controls --- Apps/QueryLogsDuckDBApp/App.cs | 135 +++++++++++++++++++++++++++------ 1 file changed, 111 insertions(+), 24 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 85fa1e380..193f8ddbc 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -21,7 +21,6 @@ You should have received a copy of the GNU General Public License using DuckDB.NET.Data; using System; using System.Collections.Generic; -using System.Linq; using System.Net; using System.Text.Json; using System.Threading.Channels; @@ -284,9 +283,35 @@ public async Task QueryLogsAsync( DnsResourceRecordType? qtype, DnsClass? qclass) { + if (entriesPerPage <= 0) + throw new ArgumentOutOfRangeException( + nameof(entriesPerPage), + "entriesPerPage must be greater than zero."); + + // Prevent pathological page sizes (DoS / memory abuse) + const int MaxPageSize = 10_000; + + if (entriesPerPage > MaxPageSize) + entriesPerPage = MaxPageSize; + + if (pageNumber < 1) + pageNumber = 1; + + // Normalize inverted time ranges + if (start is not null && + end is not null && + start > end) + { + (start, end) = (end, start); + } + using DuckDBCommand cmd = _conn.CreateCommand(); - List filters = new List(); + List filters = new(); + + /* --------------------------------- + Filters + --------------------------------- */ if (start is not null) { @@ -330,6 +355,8 @@ public async Task QueryLogsAsync( if (!string.IsNullOrWhiteSpace(qname)) { + qname = qname.Trim(); + filters.Add("qname = $qname"); cmd.Parameters.Add( new DuckDBParameter("qname", qname)); @@ -354,25 +381,59 @@ public async Task QueryLogsAsync( ? " WHERE " + string.Join(" AND ", filters) : string.Empty; - // Count query - cmd.CommandText = $"SELECT count() FROM dns_logs {whereSql}"; - long totalEntries = Convert.ToInt64(await cmd.ExecuteScalarAsync()); + /* --------------------------------- + Count + --------------------------------- */ + + cmd.CommandText = + $"SELECT count() FROM dns_logs {whereSql}"; + + long totalEntries = + Convert.ToInt64(await cmd.ExecuteScalarAsync()); long totalPages = - (long)Math.Ceiling((double)totalEntries / entriesPerPage); + totalEntries == 0 + ? 1 + : (long)Math.Ceiling( + (double)totalEntries / entriesPerPage); - pageNumber = Math.Clamp(pageNumber, 1, Math.Max(1, totalPages)); + pageNumber = + Math.Clamp(pageNumber, 1, totalPages); - long offset = (pageNumber - 1) * entriesPerPage; + /* --------------------------------- + Offset (overflow-safe) + --------------------------------- */ + + long offset; + + try + { + checked + { + offset = + (pageNumber - 1) * entriesPerPage; + } + } + catch (OverflowException) + { + offset = 0; + pageNumber = 1; + } + + /* --------------------------------- + Pagination parameters + --------------------------------- */ - //Pagination parameters cmd.Parameters.Add( new DuckDBParameter("limit", entriesPerPage)); cmd.Parameters.Add( new DuckDBParameter("offset", offset)); - // Main query + /* --------------------------------- + Main query + --------------------------------- */ + cmd.CommandText = $@" SELECT server, timestamp, @@ -391,34 +452,46 @@ FROM dns_logs LIMIT $limit OFFSET $offset"; - List list = new List(); + List list = new List(entriesPerPage); + + /* --------------------------------- + Read + --------------------------------- */ - using System.Data.Common.DbDataReader reader = await cmd.ExecuteReaderAsync(); + using var reader = + await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) { - var server = reader.GetString(0); DateTime ts = reader.GetDateTime(1); - IPAddress ip = IPAddress.Parse(reader.GetString(2)); - DnsTransportProtocol proto = (DnsTransportProtocol)reader.GetByte(3); - DnsServerResponseType respType = (DnsServerResponseType)reader.GetByte(4); + IPAddress ip = + IPAddress.Parse(reader.GetString(2)); + + DnsTransportProtocol proto = + (DnsTransportProtocol)reader.GetByte(3); + + DnsServerResponseType respType = + (DnsServerResponseType)reader.GetByte(4); double? rtt = reader.IsDBNull(5) ? null : reader.GetDouble(5); - DnsResponseCode rc = (DnsResponseCode)reader.GetByte(6); + DnsResponseCode rc = + (DnsResponseCode)reader.GetByte(6); string? qn = - reader.IsDBNull(7) ? null : reader.GetString(7); + reader.IsDBNull(7) + ? null + : reader.GetString(7); DnsQuestionRecord? question = null; - if (!reader.IsDBNull(8) && - !reader.IsDBNull(9) && - qn is not null) + if (qn is not null && + !reader.IsDBNull(8) && + !reader.IsDBNull(9)) { question = new DnsQuestionRecord( qn, @@ -428,14 +501,28 @@ FROM dns_logs } string? ans = - reader.IsDBNull(10) ? null : reader.GetString(10); + reader.IsDBNull(10) + ? null + : reader.GetString(10); list.Add( new DnsLogEntry( - 0, ts, ip, proto, respType, rtt, rc, question, ans)); + 0, + ts, + ip, + proto, + respType, + rtt, + rc, + question, + ans)); } - return new DnsLogPage(pageNumber, totalPages, totalEntries, list); + return new DnsLogPage( + pageNumber, + totalPages, + totalEntries, + list); } #endregion public From c874e6d450741e682860470d1ea4c9ba75306921 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 15:42:41 +0200 Subject: [PATCH 13/49] Added documentation for the appender. --- Apps/QueryLogsDuckDBApp/App.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 193f8ddbc..308178dec 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -76,8 +76,12 @@ private void BulkInsert(List logs) { try { - using var appender = _conn.CreateAppender("dns_logs"); - + // We create a new appender for each batch to avoid issues with concurrent usage + // By default, the appender performs commits every 204,800 rows. + // Since we are using smaller batches, we are forcing appender to close after each batch. + // It makes the flush to disk more frequent, but ensures data integrity in case of crashes. + // Each batch flush is atomic. + using DuckDBAppender appender = _conn.CreateAppender("dns_logs"); foreach (var log in logs) { if (log.Request is null || log.Response is null) From 253d42f6db8693989b3993800bb50d9be6477b73 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 15:57:39 +0200 Subject: [PATCH 14/49] Used explicit `Where` for filtering in foreach --- Apps/QueryLogsDuckDBApp/App.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 308178dec..939a448da 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -21,6 +21,7 @@ You should have received a copy of the GNU General Public License using DuckDB.NET.Data; using System; using System.Collections.Generic; +using System.Linq; using System.Net; using System.Text.Json; using System.Threading.Channels; @@ -82,11 +83,9 @@ private void BulkInsert(List logs) // It makes the flush to disk more frequent, but ensures data integrity in case of crashes. // Each batch flush is atomic. using DuckDBAppender appender = _conn.CreateAppender("dns_logs"); - foreach (var log in logs) + foreach (var log in logs + .Where(log => log.Request is not null && log.Response is not null)) { - if (log.Request is null || log.Response is null) - continue; - var question = log.Request.Question.Count > 0 ? log.Request.Question[0] From 996373bc019dab48ee0b1a5d687eac7fec0ac75f Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 15:59:27 +0200 Subject: [PATCH 15/49] Used StringBuilder for answer to minimize string allocations --- Apps/QueryLogsDuckDBApp/App.cs | 44 ++++++++++++++-------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 939a448da..06aa9d6ab 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -23,6 +23,7 @@ You should have received a copy of the GNU General Public License using System.Collections.Generic; using System.Linq; using System.Net; +using System.Text; using System.Text.Json; using System.Threading.Channels; using System.Threading.Tasks; @@ -83,10 +84,10 @@ private void BulkInsert(List logs) // It makes the flush to disk more frequent, but ensures data integrity in case of crashes. // Each batch flush is atomic. using DuckDBAppender appender = _conn.CreateAppender("dns_logs"); - foreach (var log in logs + foreach (LogEntry log in logs .Where(log => log.Request is not null && log.Response is not null)) { - var question = + DnsQuestionRecord? question = log.Request.Question.Count > 0 ? log.Request.Question[0] : null; @@ -108,31 +109,22 @@ private void BulkInsert(List logs) rtt = log.Response.Metadata.RoundTripTime; } - //Answer (Aligned with SQLite) - string? answer = null; - - if (log.Response.Answer.Count == 0) - { - answer = null; - } - else if (log.Response.Answer.Count > 2 && - log.Response.IsZoneTransfer) - { - answer = "[ZONE TRANSFER]"; - } - else + //Answer + StringBuilder answerBuilder = new StringBuilder(); + bool first = true; + foreach (DnsResourceRecord? record in log.Response.Answer) { - foreach (var record in log.Response.Answer) - { - if (answer is null) - answer = record.Type + " " + record.RDATA; - else - answer += ", " + record.Type + " " + record.RDATA; - } + if (!first) + answerBuilder.Append(", "); + answerBuilder.Append(record.Type); + answerBuilder.Append(' '); + answerBuilder.Append(record.RDATA); + first = false; } + string? answer = answerBuilder.ToString(); //Insert Row - var row = appender.CreateRow(); + IDuckDBAppenderRow row = appender.CreateRow(); row.AppendValue(_dnsServer.ServerDomain); row.AppendValue(log.Timestamp); @@ -202,12 +194,12 @@ answer TEXT private async Task ProcessLogsAsync() { - var batch = new List(MAX_BATCH_SIZE); + List batch = new List(MAX_BATCH_SIZE); while (await _channel.Reader.WaitToReadAsync()) { while (batch.Count < MAX_BATCH_SIZE && - _channel.Reader.TryRead(out var log)) + _channel.Reader.TryRead(out LogEntry log)) { batch.Add(log); } @@ -461,7 +453,7 @@ FROM dns_logs Read --------------------------------- */ - using var reader = + using System.Data.Common.DbDataReader reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) From 32b9ca311cc8e2839b84b71bcefc15deadafb53c Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 16:00:23 +0200 Subject: [PATCH 16/49] Simplified syntax --- Apps/QueryLogsDuckDBApp/App.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 06aa9d6ab..fd41ba17f 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -93,12 +93,9 @@ private void BulkInsert(List logs) : null; //Response Type(Aligned) - DnsServerResponseType responseType; - - if (log.Response.Tag is null) - responseType = DnsServerResponseType.Recursive; - else - responseType = (DnsServerResponseType)log.Response.Tag; + DnsServerResponseType responseType = log.Response.Tag is null + ? DnsServerResponseType.Recursive + : (DnsServerResponseType)log.Response.Tag; //RTT double? rtt = null; From 6c3c5784cdfa10201179bd30569297eed7700053 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 16:03:35 +0200 Subject: [PATCH 17/49] Used ushort to align with SQLite app; no functional advantage --- Apps/QueryLogsDuckDBApp/App.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index fd41ba17f..d44ba6fee 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -358,14 +358,14 @@ end is not null && { filters.Add("qtype = $qtype"); cmd.Parameters.Add( - new DuckDBParameter("qtype", (short)qtype.Value)); + new DuckDBParameter("qtype", (ushort)qtype.Value)); } if (qclass is not null) { filters.Add("qclass = $qclass"); cmd.Parameters.Add( - new DuckDBParameter("qclass", (short)qclass.Value)); + new DuckDBParameter("qclass", (ushort)qclass.Value)); } string whereSql = From dc34ae007b2de2f7f8528734c9e1b6b7250a6bf8 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 16:04:24 +0200 Subject: [PATCH 18/49] Extended buffer size to 200.000 --- Apps/QueryLogsDuckDBApp/App.cs | 2 +- Apps/QueryLogsDuckDBApp/dnsApp.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index d44ba6fee..a3a288c36 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -227,7 +227,7 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) string dbPath = cfg.GetPropertyValue("dbPath", "querylogs.db"); - _maxQueueSize = cfg.GetPropertyValue("maxQueueSize", 20000); + _maxQueueSize = cfg.GetPropertyValue("maxQueueSize", 200_000); if (!System.IO.Path.IsPathRooted(dbPath)) dbPath = System.IO.Path.Combine(dnsServer.ApplicationFolder, dbPath); diff --git a/Apps/QueryLogsDuckDBApp/dnsApp.config b/Apps/QueryLogsDuckDBApp/dnsApp.config index 2ff1b3162..7c0ad3efa 100644 --- a/Apps/QueryLogsDuckDBApp/dnsApp.config +++ b/Apps/QueryLogsDuckDBApp/dnsApp.config @@ -1,5 +1,5 @@ { "enableLogging": true, "dbPath": "querylogs.db", - "maxQueueSize": 20000 + "maxQueueSize": 200000 } From 727adfd13c55fe997bf044a161a6af7769e9f60d Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 16:19:16 +0200 Subject: [PATCH 19/49] Resused Stringbuilder for minimizing allocations --- Apps/QueryLogsDuckDBApp/App.cs | 40 +++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index a3a288c36..71c59416c 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -37,6 +37,11 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs { #region variables + [ThreadStatic] + private static StringBuilder? _sb; + + private const int DEFAULT_ANSWER_SIZE = 256; + private const int MAX_ANSWER_SIZE = 4000; private const int MAX_BATCH_SIZE = 1000; private Channel _channel; private DuckDBConnection _conn; @@ -74,6 +79,17 @@ private void Dispose(bool disposing) #region private + private static StringBuilder GetStringBuilder() + { + StringBuilder? sb = _sb; + + if (sb is null) + return _sb = new StringBuilder(DEFAULT_ANSWER_SIZE, MAX_ANSWER_SIZE); + + sb.Clear(); + return sb; + } + private void BulkInsert(List logs) { try @@ -107,18 +123,22 @@ private void BulkInsert(List logs) } //Answer - StringBuilder answerBuilder = new StringBuilder(); - bool first = true; - foreach (DnsResourceRecord? record in log.Response.Answer) + string? answer = string.Empty; + if (log.Response.Answer.Count != 0) { - if (!first) - answerBuilder.Append(", "); - answerBuilder.Append(record.Type); - answerBuilder.Append(' '); - answerBuilder.Append(record.RDATA); - first = false; + StringBuilder answerBuilder = GetStringBuilder(); + bool first = true; + foreach (DnsResourceRecord? record in log.Response.Answer) + { + if (!first) + answerBuilder.Append(", "); + answerBuilder.Append(record.Type); + answerBuilder.Append(' '); + answerBuilder.Append(record.RDATA); + first = false; + } + answer = answerBuilder.ToString(); } - string? answer = answerBuilder.ToString(); //Insert Row IDuckDBAppenderRow row = appender.CreateRow(); From 49e7ddf911aa9c4ad5dbcd1301754dacc55680b0 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 16:21:36 +0200 Subject: [PATCH 20/49] Added comments --- Apps/QueryLogsDuckDBApp/App.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 71c59416c..21f5b2042 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -40,16 +40,16 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs [ThreadStatic] private static StringBuilder? _sb; - private const int DEFAULT_ANSWER_SIZE = 256; - private const int MAX_ANSWER_SIZE = 4000; - private const int MAX_BATCH_SIZE = 1000; + private const int DEFAULT_ANSWER_SIZE = 256; // Initial capacity for answer StringBuilder + private const int MAX_ANSWER_SIZE = 4000; // To prevent excessive memory usage, answers larger than this will be truncated. Used the value of MySQL app. + private const int MAX_BATCH_SIZE = 1000; // Maximum number of log entries to process in a single batch private Channel _channel; private DuckDBConnection _conn; private Task _consumerTask; private bool _disposed; private IDnsServer _dnsServer; private bool _enableLogging; - private int _maxQueueSize; + private int _maxQueueSize; // Maximum number of log entries in the queue, default 200,000 #endregion variables #region IDisposable @@ -181,6 +181,9 @@ private void BulkInsert(List logs) catch (Exception ex) { _dnsServer?.WriteLog(ex); + // No need for a wait as this is running synchronously in the background, + // we just log and continue. + // No risk of concurrency issues and waiting here would block the logging thread. } } From d619721c5556e3acd10d99df1099d37be98520d6 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 16:26:52 +0200 Subject: [PATCH 21/49] Sanitized qname --- Apps/QueryLogsDuckDBApp/App.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 21f5b2042..8837c943f 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -370,8 +370,11 @@ end is not null && if (!string.IsNullOrWhiteSpace(qname)) { - qname = qname.Trim(); - + qname = qname.Trim().ToLowerInvariant(); + if (qname.Contains('*')) + { + qname = qname.Replace('*', '%'); + } filters.Add("qname = $qname"); cmd.Parameters.Add( new DuckDBParameter("qname", qname)); From 50c3cdc1580633c5db8e4f6bb497247ddd3cffe6 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 16:33:08 +0200 Subject: [PATCH 22/49] Added more comments --- Apps/QueryLogsDuckDBApp/App.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 8837c943f..4d03de65c 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -67,6 +67,9 @@ private void Dispose(bool disposing) { if (disposing) { + // To prevent blocking shutdown, we attempt to complete the channel and wait for the consumer task to finish. + // We leave catch blocks empty explicitly to ignore any exceptions during disposal. + // This ensures that the application can shut down gracefully without being hindered by logging operations. try { _channel?.Writer.TryComplete(); } catch { } try { _consumerTask?.Wait(5000); } catch { } try { _conn?.Close(); _conn?.Dispose(); } catch { } From 5f3da225a2b083d5e8bd8347822f662490415135 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 16:43:35 +0200 Subject: [PATCH 23/49] Formatting --- Apps/QueryLogsDuckDBApp/App.cs | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 4d03de65c..2b779344b 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -36,18 +36,23 @@ namespace QueryLogsDuckDB public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs { #region variables + // Initial capacity for answer StringBuilder + private const int DEFAULT_ANSWER_SIZE = 256; + + // To prevent excessive memory usage, answers larger than this will be truncated. Used the value of MySQL app. + private const int MAX_ANSWER_SIZE = 4000; + + // Maximum number of log entries to process in a single batch + private const int MAX_BATCH_SIZE = 1000; [ThreadStatic] private static StringBuilder? _sb; - private const int DEFAULT_ANSWER_SIZE = 256; // Initial capacity for answer StringBuilder - private const int MAX_ANSWER_SIZE = 4000; // To prevent excessive memory usage, answers larger than this will be truncated. Used the value of MySQL app. - private const int MAX_BATCH_SIZE = 1000; // Maximum number of log entries to process in a single batch - private Channel _channel; - private DuckDBConnection _conn; - private Task _consumerTask; + private Channel? _channel; + private DuckDBConnection? _conn; + private Task? _consumerTask; private bool _disposed; - private IDnsServer _dnsServer; + private IDnsServer? _dnsServer; private bool _enableLogging; private int _maxQueueSize; // Maximum number of log entries in the queue, default 200,000 #endregion variables @@ -102,7 +107,7 @@ private void BulkInsert(List logs) // Since we are using smaller batches, we are forcing appender to close after each batch. // It makes the flush to disk more frequent, but ensures data integrity in case of crashes. // Each batch flush is atomic. - using DuckDBAppender appender = _conn.CreateAppender("dns_logs"); + using DuckDBAppender appender = _conn!.CreateAppender("dns_logs"); foreach (LogEntry log in logs .Where(log => log.Request is not null && log.Response is not null)) { @@ -146,7 +151,7 @@ private void BulkInsert(List logs) //Insert Row IDuckDBAppenderRow row = appender.CreateRow(); - row.AppendValue(_dnsServer.ServerDomain); + row.AppendValue(_dnsServer!.ServerDomain); row.AppendValue(log.Timestamp); row.AppendValue(log.RemoteEP.Address.ToString()); row.AppendValue((byte)log.Protocol); @@ -192,7 +197,7 @@ private void BulkInsert(List logs) private async Task CreateSchemaAsync() { - using DuckDBCommand cmd = _conn.CreateCommand(); + using DuckDBCommand cmd = _conn!.CreateCommand(); cmd.CommandText = @" CREATE TABLE IF NOT EXISTS dns_logs ( @@ -219,7 +224,7 @@ private async Task ProcessLogsAsync() { List batch = new List(MAX_BATCH_SIZE); - while (await _channel.Reader.WaitToReadAsync()) + while (await _channel!.Reader.WaitToReadAsync()) { while (batch.Count < MAX_BATCH_SIZE && _channel.Reader.TryRead(out LogEntry log)) @@ -281,7 +286,7 @@ public Task InsertLogAsync( DnsDatagram response) { if (_enableLogging) - _channel.Writer.TryWrite( + _channel!.Writer.TryWrite( new LogEntry(timestamp, request, remoteEP, protocol, response)); return Task.CompletedTask; @@ -323,9 +328,9 @@ end is not null && (start, end) = (end, start); } - using DuckDBCommand cmd = _conn.CreateCommand(); + using DuckDBCommand cmd = _conn!.CreateCommand(); - List filters = new(); + List filters = new List(); /* --------------------------------- Filters From dc05c2e52eb7c8877a280f691db72f4e56128771 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 16:51:33 +0200 Subject: [PATCH 24/49] Fixed the ushort problem --- Apps/QueryLogsDuckDBApp/App.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 2b779344b..182eab285 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -521,8 +521,8 @@ FROM dns_logs { question = new DnsQuestionRecord( qn, - (DnsResourceRecordType)reader.GetInt16(8), - (DnsClass)reader.GetInt16(9), + (DnsResourceRecordType)reader.GetFieldValue(8), + (DnsClass)reader.GetFieldValue(9), false); } From 0de9e8314c428b05c4fc06e561e76510ec159203 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 16:54:36 +0200 Subject: [PATCH 25/49] Added dispose check to async task --- Apps/QueryLogsDuckDBApp/App.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 182eab285..bf73d7a3c 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -222,6 +222,8 @@ answer TEXT private async Task ProcessLogsAsync() { + if (_disposed) return; + List batch = new List(MAX_BATCH_SIZE); while (await _channel!.Reader.WaitToReadAsync()) From fe7e3b0939be712603d56aae9d6ac1d0be2689c9 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 17:02:30 +0200 Subject: [PATCH 26/49] Used a Config class for configuration. --- Apps/QueryLogsDuckDBApp/App.cs | 36 ++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index bf73d7a3c..bae1d231a 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -21,10 +21,12 @@ You should have received a copy of the GNU General Public License using DuckDB.NET.Data; using System; using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; using System.Linq; using System.Net; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using System.Threading.Channels; using System.Threading.Tasks; using TechnitiumLibrary; @@ -47,6 +49,7 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs [ThreadStatic] private static StringBuilder? _sb; + Config _config; private Channel? _channel; private DuckDBConnection? _conn; @@ -253,27 +256,22 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) { _dnsServer = dnsServer; - using JsonDocument json = JsonDocument.Parse(config); - JsonElement cfg = json.RootElement; + JsonSerializerOptions options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; + _config = JsonSerializer.Deserialize(config, options); + Validator.ValidateObject(_config, new ValidationContext(_config), validateAllProperties: true); - _enableLogging = cfg.GetPropertyValue("enableLogging", true); - - string dbPath = cfg.GetPropertyValue("dbPath", "querylogs.db"); - - _maxQueueSize = cfg.GetPropertyValue("maxQueueSize", 200_000); - - if (!System.IO.Path.IsPathRooted(dbPath)) - dbPath = System.IO.Path.Combine(dnsServer.ApplicationFolder, dbPath); + if (!System.IO.Path.IsPathRooted(_config.DbPath)) + _config.DbPath = System.IO.Path.Combine(dnsServer.ApplicationFolder, _config.DbPath); _channel = Channel.CreateBounded( - new BoundedChannelOptions(_maxQueueSize) + new BoundedChannelOptions(_config.MaxQueueSize) { SingleReader = true, SingleWriter = true, FullMode = BoundedChannelFullMode.DropWrite }); - _conn = new DuckDBConnection($"Data Source={dbPath}"); + _conn = new DuckDBConnection($"Data Source={_config.DbPath}"); await _conn.OpenAsync(); await CreateSchemaAsync(); @@ -287,7 +285,7 @@ public Task InsertLogAsync( DnsTransportProtocol protocol, DnsDatagram response) { - if (_enableLogging) + if (_config.EnableLogging) _channel!.Writer.TryWrite( new LogEntry(timestamp, request, remoteEP, protocol, response)); @@ -587,5 +585,17 @@ public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, Dn #endregion constructor } + + private class Config + { + [JsonPropertyName("enableLogging")] + public bool EnableLogging { get; set; } = true; + + [JsonPropertyName("dbPath")] + public string DbPath { get; set; } = "querylogs.db"; + + [JsonPropertyName("maxQueueSize")] + public int MaxQueueSize { get; set; } = 200_000; + } } } \ No newline at end of file From b1efc4f49df7a46b24899b439dec0d06991d7f90 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 17:12:07 +0200 Subject: [PATCH 27/49] Added Retention --- Apps/QueryLogsDuckDBApp/App.cs | 95 +++++++++++++++++++++++++-- Apps/QueryLogsDuckDBApp/dnsApp.config | 8 ++- 2 files changed, 93 insertions(+), 10 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index bae1d231a..15a330abe 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -49,15 +49,12 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs [ThreadStatic] private static StringBuilder? _sb; - Config _config; - private Channel? _channel; + Config? _config; private DuckDBConnection? _conn; private Task? _consumerTask; private bool _disposed; private IDnsServer? _dnsServer; - private bool _enableLogging; - private int _maxQueueSize; // Maximum number of log entries in the queue, default 200,000 #endregion variables #region IDisposable @@ -248,6 +245,83 @@ private async Task ProcessLogsAsync() BulkInsert(batch); } + private async Task RetentionLoopAsync() + { + // Initial delay + await Task.Delay(TimeSpan.FromMinutes(1)); + + while (!_disposed) + { + try + { + await RunRetentionAsync(); + } + catch (Exception ex) + { + _dnsServer?.WriteLog(ex); + } + + await Task.Delay(TimeSpan.FromMinutes(15)); + } + } + + private async Task RunRetentionAsync() + { + if (_conn is null || _config is null) + return; + + using DuckDBCommand cmd = _conn.CreateCommand(); + + long deleted = 0; + + /* --------------------------------- + Max records + --------------------------------- */ + + if (_config.MaxLogRecords > 0) + { + cmd.CommandText = @" +DELETE FROM dns_logs +WHERE timestamp NOT IN ( + SELECT timestamp + FROM dns_logs + ORDER BY timestamp DESC + LIMIT $limit +);"; + + cmd.Parameters.Clear(); + cmd.Parameters.Add( + new DuckDBParameter("limit", _config.MaxLogRecords)); + + deleted += await cmd.ExecuteNonQueryAsync(); + } + + /* --------------------------------- + Max days + --------------------------------- */ + + if (_config.MaxLogDays > 0) + { + DateTime cutoff = + DateTime.UtcNow.AddDays(-_config.MaxLogDays); + + cmd.CommandText = + "DELETE FROM dns_logs WHERE timestamp < $cutoff;"; + + cmd.Parameters.Clear(); + cmd.Parameters.Add( + new DuckDBParameter("cutoff", cutoff)); + + deleted += await cmd.ExecuteNonQueryAsync(); + } + + if (deleted > 0) + { + cmd.Parameters.Clear(); + cmd.CommandText = "CHECKPOINT;"; + await cmd.ExecuteNonQueryAsync(); + } + } #endregion private #region public @@ -258,6 +332,7 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) JsonSerializerOptions options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; _config = JsonSerializer.Deserialize(config, options); + _config ??= new Config(); Validator.ValidateObject(_config, new ValidationContext(_config), validateAllProperties: true); if (!System.IO.Path.IsPathRooted(_config.DbPath)) @@ -276,6 +351,7 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) await CreateSchemaAsync(); _consumerTask = Task.Run(ProcessLogsAsync); + _ = Task.Run(RetentionLoopAsync); } public Task InsertLogAsync( @@ -285,7 +361,7 @@ public Task InsertLogAsync( DnsTransportProtocol protocol, DnsDatagram response) { - if (_config.EnableLogging) + if (_config!.EnableLogging) _channel!.Writer.TryWrite( new LogEntry(timestamp, request, remoteEP, protocol, response)); @@ -588,11 +664,16 @@ public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, Dn private class Config { + [JsonPropertyName("dbPath")] + public string DbPath { get; set; } = "querylogs.db"; + [JsonPropertyName("enableLogging")] public bool EnableLogging { get; set; } = true; + [JsonPropertyName("maxLogDays")] + public int MaxLogDays { get; set; } = 30; - [JsonPropertyName("dbPath")] - public string DbPath { get; set; } = "querylogs.db"; + [JsonPropertyName("maxLogRecords")] + public long MaxLogRecords { get; set; } = 1_000_000; [JsonPropertyName("maxQueueSize")] public int MaxQueueSize { get; set; } = 200_000; diff --git a/Apps/QueryLogsDuckDBApp/dnsApp.config b/Apps/QueryLogsDuckDBApp/dnsApp.config index 7c0ad3efa..c141ede87 100644 --- a/Apps/QueryLogsDuckDBApp/dnsApp.config +++ b/Apps/QueryLogsDuckDBApp/dnsApp.config @@ -1,5 +1,7 @@ { - "enableLogging": true, - "dbPath": "querylogs.db", - "maxQueueSize": 200000 + "enableLogging": true, + "dbPath": "querylogs.db", + "maxQueueSize": 200000, + "maxLogDays:" 30, + "maxLogRecords": 1000000 } From 024a0063b813c53511415e4222c89c41a9079058 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 17:23:43 +0200 Subject: [PATCH 28/49] Fixed answer logic --- Apps/QueryLogsDuckDBApp/App.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 15a330abe..57baf5c81 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -131,7 +131,7 @@ private void BulkInsert(List logs) } //Answer - string? answer = string.Empty; + string? answer = null; if (log.Response.Answer.Count != 0) { StringBuilder answerBuilder = GetStringBuilder(); From 3e59a7e505e80ac98cf43273a76c744edf6efdfd Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 17:25:59 +0200 Subject: [PATCH 29/49] Used Cutoff Timestamp --- Apps/QueryLogsDuckDBApp/App.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 57baf5c81..f320ff5c5 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -281,13 +281,16 @@ Max records if (_config.MaxLogRecords > 0) { cmd.CommandText = @" -DELETE FROM dns_logs -WHERE timestamp NOT IN ( +WITH cutoff AS ( SELECT timestamp FROM dns_logs ORDER BY timestamp DESC - LIMIT $limit -);"; + OFFSET $limit + LIMIT 1 +) +DELETE FROM dns_logs +WHERE timestamp < (SELECT timestamp FROM cutoff); +"; cmd.Parameters.Clear(); cmd.Parameters.Add( From dafeb3582bf749438ec7a40ce7452748d0d821d7 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 18:33:33 +0200 Subject: [PATCH 30/49] Fixed broken config file --- Apps/QueryLogsDuckDBApp/dnsApp.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Apps/QueryLogsDuckDBApp/dnsApp.config b/Apps/QueryLogsDuckDBApp/dnsApp.config index c141ede87..081b6ccdc 100644 --- a/Apps/QueryLogsDuckDBApp/dnsApp.config +++ b/Apps/QueryLogsDuckDBApp/dnsApp.config @@ -2,6 +2,6 @@ "enableLogging": true, "dbPath": "querylogs.db", "maxQueueSize": 200000, - "maxLogDays:" 30, + "maxLogDays": 30, "maxLogRecords": 1000000 } From 8cb40d7b27a95742a73e07e21eeb22a4cf54b36e Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 18:34:52 +0200 Subject: [PATCH 31/49] Added semaphore for bulk inserts --- Apps/QueryLogsDuckDBApp/App.cs | 95 ++++++++++++++++++++-------------- 1 file changed, 56 insertions(+), 39 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index f320ff5c5..031ba0e1f 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -27,9 +27,9 @@ You should have received a copy of the GNU General Public License using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; -using TechnitiumLibrary; using TechnitiumLibrary.Net.Dns; using TechnitiumLibrary.Net.Dns.ResourceRecords; @@ -55,6 +55,8 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs private Task? _consumerTask; private bool _disposed; private IDnsServer? _dnsServer; + private readonly SemaphoreSlim _dbGate = new(1, 1); + #endregion variables #region IDisposable @@ -98,8 +100,9 @@ private static StringBuilder GetStringBuilder() return sb; } - private void BulkInsert(List logs) + private async Task BulkInsertAsync(List logs) { + await _dbGate.WaitAsync(); try { // We create a new appender for each batch to avoid issues with concurrent usage @@ -193,6 +196,10 @@ private void BulkInsert(List logs) // we just log and continue. // No risk of concurrency issues and waiting here would block the logging thread. } + finally + { + _dbGate.Release(); + } } private async Task CreateSchemaAsync() @@ -222,11 +229,9 @@ answer TEXT private async Task ProcessLogsAsync() { - if (_disposed) return; - List batch = new List(MAX_BATCH_SIZE); - while (await _channel!.Reader.WaitToReadAsync()) + while (!_disposed && await _channel!.Reader.WaitToReadAsync()) { while (batch.Count < MAX_BATCH_SIZE && _channel.Reader.TryRead(out LogEntry log)) @@ -236,13 +241,13 @@ private async Task ProcessLogsAsync() if (batch.Count > 0) { - BulkInsert(batch); + await BulkInsertAsync(batch); batch.Clear(); } } if (batch.Count > 0) - BulkInsert(batch); + await BulkInsertAsync(batch); } private async Task RetentionLoopAsync() @@ -270,17 +275,20 @@ private async Task RunRetentionAsync() if (_conn is null || _config is null) return; - using DuckDBCommand cmd = _conn.CreateCommand(); + await _dbGate.WaitAsync(); + try + { + using DuckDBCommand cmd = _conn.CreateCommand(); - long deleted = 0; + long deleted = 0; - /* --------------------------------- - Max records - --------------------------------- */ + /* --------------------------------- + Max records + --------------------------------- */ - if (_config.MaxLogRecords > 0) - { - cmd.CommandText = @" + if (_config.MaxLogRecords > 0) + { + cmd.CommandText = @" WITH cutoff AS ( SELECT timestamp FROM dns_logs @@ -292,37 +300,42 @@ DELETE FROM dns_logs WHERE timestamp < (SELECT timestamp FROM cutoff); "; - cmd.Parameters.Clear(); - cmd.Parameters.Add( - new DuckDBParameter("limit", _config.MaxLogRecords)); + cmd.Parameters.Clear(); + cmd.Parameters.Add( + new DuckDBParameter("limit", _config.MaxLogRecords)); - deleted += await cmd.ExecuteNonQueryAsync(); - } + deleted += await cmd.ExecuteNonQueryAsync(); + } - /* --------------------------------- - Max days - --------------------------------- */ + /* --------------------------------- + Max days + --------------------------------- */ - if (_config.MaxLogDays > 0) - { - DateTime cutoff = - DateTime.UtcNow.AddDays(-_config.MaxLogDays); + if (_config.MaxLogDays > 0) + { + DateTime cutoff = + DateTime.UtcNow.AddDays(-_config.MaxLogDays); - cmd.CommandText = - "DELETE FROM dns_logs WHERE timestamp < $cutoff;"; + cmd.CommandText = + "DELETE FROM dns_logs WHERE timestamp < $cutoff;"; - cmd.Parameters.Clear(); - cmd.Parameters.Add( - new DuckDBParameter("cutoff", cutoff)); + cmd.Parameters.Clear(); + cmd.Parameters.Add( + new DuckDBParameter("cutoff", cutoff)); - deleted += await cmd.ExecuteNonQueryAsync(); - } + deleted += await cmd.ExecuteNonQueryAsync(); + } - if (deleted > 0) + if (deleted > 0) + { + cmd.Parameters.Clear(); + cmd.CommandText = "CHECKPOINT;"; + await cmd.ExecuteNonQueryAsync(); + } + } + finally { - cmd.Parameters.Clear(); - cmd.CommandText = "CHECKPOINT;"; - await cmd.ExecuteNonQueryAsync(); + _dbGate.Release(); } } #endregion private @@ -364,7 +377,11 @@ public Task InsertLogAsync( DnsTransportProtocol protocol, DnsDatagram response) { - if (_config!.EnableLogging) + if (_disposed) return Task.CompletedTask; + if(_config is null) return Task.CompletedTask; + if(_conn is null) return Task.CompletedTask; + + if (_config.EnableLogging) _channel!.Writer.TryWrite( new LogEntry(timestamp, request, remoteEP, protocol, response)); From 57a5aeb97303333097ba34d55c2a092cb6bdb131 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 18:38:25 +0200 Subject: [PATCH 32/49] Added dispose check --- Apps/QueryLogsDuckDBApp/App.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 031ba0e1f..a69fc2321 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -252,21 +252,20 @@ private async Task ProcessLogsAsync() private async Task RetentionLoopAsync() { - // Initial delay await Task.Delay(TimeSpan.FromMinutes(1)); while (!_disposed) { try { + await Task.Delay(TimeSpan.FromMinutes(15)); + if (_disposed) break; await RunRetentionAsync(); } catch (Exception ex) { _dnsServer?.WriteLog(ex); } - - await Task.Delay(TimeSpan.FromMinutes(15)); } } From 2f3ebd66c2543652fa9612baf1b7900b6bfaee73 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 18:40:12 +0200 Subject: [PATCH 33/49] Added ranges for config --- Apps/QueryLogsDuckDBApp/App.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index a69fc2321..ce7f215be 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -688,13 +688,17 @@ private class Config [JsonPropertyName("enableLogging")] public bool EnableLogging { get; set; } = true; + [JsonPropertyName("maxLogDays")] + [Range(1, 365)] public int MaxLogDays { get; set; } = 30; [JsonPropertyName("maxLogRecords")] + [Range(1, 5_000_000_000)] public long MaxLogRecords { get; set; } = 1_000_000; [JsonPropertyName("maxQueueSize")] + [Range(1_000, 1_000_000)] public int MaxQueueSize { get; set; } = 200_000; } } From 246df48540a5eba841f983aef3dc033ddb44447a Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 18:41:14 +0200 Subject: [PATCH 34/49] Added logging for retention ops --- Apps/QueryLogsDuckDBApp/App.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index ce7f215be..f8d065ab7 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -330,6 +330,7 @@ Max days cmd.Parameters.Clear(); cmd.CommandText = "CHECKPOINT;"; await cmd.ExecuteNonQueryAsync(); + _dnsServer?.WriteLog($"DuckDB retention removed {deleted} records."); } } finally From 682f02121ec4f8706e9560aea3000c58eedd6335 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 18:42:39 +0200 Subject: [PATCH 35/49] USed LIKE instead of = for qname --- Apps/QueryLogsDuckDBApp/App.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index f8d065ab7..792363f37 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -479,7 +479,7 @@ end is not null && { qname = qname.Replace('*', '%'); } - filters.Add("qname = $qname"); + filters.Add("qname LIKE $qname"); cmd.Parameters.Add( new DuckDBParameter("qname", qname)); } From 31c24350cd007c4c7212fd29fd3e47d4ff11257a Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 18:45:35 +0200 Subject: [PATCH 36/49] Improved task management --- Apps/QueryLogsDuckDBApp/App.cs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 792363f37..a11ed7082 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -53,6 +53,7 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs Config? _config; private DuckDBConnection? _conn; private Task? _consumerTask; + private Task? _retentionTask; private bool _disposed; private IDnsServer? _dnsServer; private readonly SemaphoreSlim _dbGate = new(1, 1); @@ -78,7 +79,8 @@ private void Dispose(bool disposing) // We leave catch blocks empty explicitly to ignore any exceptions during disposal. // This ensures that the application can shut down gracefully without being hindered by logging operations. try { _channel?.Writer.TryComplete(); } catch { } - try { _consumerTask?.Wait(5000); } catch { } + try { _consumerTask?.Wait(5000); _consumerTask?.Dispose(); } catch { } + try { _retentionTask?.Wait(5000); _retentionTask?.Dispose(); } catch { } try { _conn?.Close(); _conn?.Dispose(); } catch { } } @@ -366,8 +368,12 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) await _conn.OpenAsync(); await CreateSchemaAsync(); - _consumerTask = Task.Run(ProcessLogsAsync); - _ = Task.Run(RetentionLoopAsync); + _consumerTask = Task.Run(ProcessLogsAsync).ContinueWith( + t => { var _ = t.Exception; }, + TaskContinuationOptions.OnlyOnFaulted); + _retentionTask = Task.Run(RetentionLoopAsync).ContinueWith( + t => { var _ = t.Exception; }, + TaskContinuationOptions.OnlyOnFaulted); } public Task InsertLogAsync( @@ -378,8 +384,8 @@ public Task InsertLogAsync( DnsDatagram response) { if (_disposed) return Task.CompletedTask; - if(_config is null) return Task.CompletedTask; - if(_conn is null) return Task.CompletedTask; + if (_config is null) return Task.CompletedTask; + if (_conn is null) return Task.CompletedTask; if (_config.EnableLogging) _channel!.Writer.TryWrite( @@ -689,7 +695,7 @@ private class Config [JsonPropertyName("enableLogging")] public bool EnableLogging { get; set; } = true; - + [JsonPropertyName("maxLogDays")] [Range(1, 365)] public int MaxLogDays { get; set; } = 30; From 25507acd3b8d1482e01af97387528a783c3ce047 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 18:47:15 +0200 Subject: [PATCH 37/49] Added error logging to dispose block --- Apps/QueryLogsDuckDBApp/App.cs | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index a11ed7082..db4c69831 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -76,12 +76,27 @@ private void Dispose(bool disposing) if (disposing) { // To prevent blocking shutdown, we attempt to complete the channel and wait for the consumer task to finish. - // We leave catch blocks empty explicitly to ignore any exceptions during disposal. // This ensures that the application can shut down gracefully without being hindered by logging operations. - try { _channel?.Writer.TryComplete(); } catch { } - try { _consumerTask?.Wait(5000); _consumerTask?.Dispose(); } catch { } - try { _retentionTask?.Wait(5000); _retentionTask?.Dispose(); } catch { } - try { _conn?.Close(); _conn?.Dispose(); } catch { } + try { _channel?.Writer.TryComplete(); } + catch (Exception ex) + { + Console.Error.WriteLine("QueryLogsDuckDB.App: Error while completing log channel during Dispose: " + ex); + } + try { _consumerTask?.Wait(5000); _consumerTask?.Dispose(); } + catch (Exception ex) + { + Console.Error.WriteLine("QueryLogsDuckDB.App: Error while waiting for consumer task during Dispose: " + ex); + } + try { _retentionTask?.Wait(5000); _retentionTask?.Dispose(); } + catch (Exception ex) + { + Console.Error.WriteLine("QueryLogsDuckDB.App: Error while waiting for retention task during Dispose: " + ex); + } + try { _conn?.Close(); _conn?.Dispose(); } + catch (Exception ex) + { + Console.Error.WriteLine("QueryLogsDuckDB.App: Error while closing/disposing DuckDB connection during Dispose: " + ex); + } } _disposed = true; From 07b2a33442561d3576beff2ff49abf66102b13e2 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 18:54:51 +0200 Subject: [PATCH 38/49] Fixed task continuations --- Apps/QueryLogsDuckDBApp/App.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index db4c69831..7ab1d949d 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -383,10 +383,11 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) await _conn.OpenAsync(); await CreateSchemaAsync(); - _consumerTask = Task.Run(ProcessLogsAsync).ContinueWith( - t => { var _ = t.Exception; }, + _consumerTask = Task.Run(ProcessLogsAsync); + await _consumerTask.ContinueWith(t => { var _ = t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); - _retentionTask = Task.Run(RetentionLoopAsync).ContinueWith( + _retentionTask = Task.Run(RetentionLoopAsync); + await _retentionTask.ContinueWith( t => { var _ = t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); } From 5f59631013c2bc7c156e27a206d573ab4eb15b40 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 18:59:40 +0200 Subject: [PATCH 39/49] Added guards for retention --- Apps/QueryLogsDuckDBApp/App.cs | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 7ab1d949d..3b10f7872 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -304,23 +304,35 @@ Max records if (_config.MaxLogRecords > 0) { - cmd.CommandText = @" + // Count first to avoid running a DELETE when there is nothing to prune. + cmd.Parameters.Clear(); + cmd.CommandText = "SELECT count() FROM dns_logs;"; + long totalRows = Convert.ToInt64(await cmd.ExecuteScalarAsync()); + if (totalRows > _config.MaxLogRecords) + { + // Keep newest N rows (by timestamp). The cutoff is the Nth newest row. + // NOTE: Timestamp collisions can still cause >N rows to be retained; strict N + // requires a stable tiebreaker column (handled in a separate PR if needed). + cmd.Parameters.Clear(); + + + cmd.CommandText = @" WITH cutoff AS ( SELECT timestamp FROM dns_logs ORDER BY timestamp DESC - OFFSET $limit + OFFSET ($limit - 1) LIMIT 1 ) DELETE FROM dns_logs WHERE timestamp < (SELECT timestamp FROM cutoff); "; + cmd.Parameters.Clear(); + cmd.Parameters.Add( + new DuckDBParameter("limit", _config.MaxLogRecords)); - cmd.Parameters.Clear(); - cmd.Parameters.Add( - new DuckDBParameter("limit", _config.MaxLogRecords)); - - deleted += await cmd.ExecuteNonQueryAsync(); + deleted += await cmd.ExecuteNonQueryAsync(); + } } /* --------------------------------- From 41e8f743a59563a7a8b33ec4d3b133e95bf18982 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 19:01:51 +0200 Subject: [PATCH 40/49] Revert "Fixed task continuations" This reverts commit 07b2a33442561d3576beff2ff49abf66102b13e2. --- Apps/QueryLogsDuckDBApp/App.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 3b10f7872..9250de990 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -395,11 +395,10 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) await _conn.OpenAsync(); await CreateSchemaAsync(); - _consumerTask = Task.Run(ProcessLogsAsync); - await _consumerTask.ContinueWith(t => { var _ = t.Exception; }, + _consumerTask = Task.Run(ProcessLogsAsync).ContinueWith( + t => { var _ = t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); - _retentionTask = Task.Run(RetentionLoopAsync); - await _retentionTask.ContinueWith( + _retentionTask = Task.Run(RetentionLoopAsync).ContinueWith( t => { var _ = t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); } From 248fa3e520d475cb1312dfbe35637946c5f19d7c Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 19:11:13 +0200 Subject: [PATCH 41/49] Guarded Answer block --- Apps/QueryLogsDuckDBApp/App.cs | 52 ++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 9250de990..86c02cc34 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -150,22 +150,50 @@ private async Task BulkInsertAsync(List logs) rtt = log.Response.Metadata.RoundTripTime; } - //Answer + // Answer (bounded, safe) string? answer = null; - if (log.Response.Answer.Count != 0) + + if (log.Response.Answer.Count > 0) { - StringBuilder answerBuilder = GetStringBuilder(); - bool first = true; - foreach (DnsResourceRecord? record in log.Response.Answer) + if (log.Response.IsZoneTransfer && log.Response.Answer.Count > 2) + { + answer = "[ZONE TRANSFER]"; + } + else { - if (!first) - answerBuilder.Append(", "); - answerBuilder.Append(record.Type); - answerBuilder.Append(' '); - answerBuilder.Append(record.RDATA); - first = false; + StringBuilder sb = GetStringBuilder(); + bool first = true; + + foreach (DnsResourceRecord record in log.Response.Answer) + { + if (!first) + { + if (sb.Length + 2 >= MAX_ANSWER_SIZE) + break; + + sb.Append(", "); + } + + string part = $"{record.Type} {record.RDATA}"; + + int remaining = + MAX_ANSWER_SIZE - sb.Length; + + if (remaining <= 0) + break; + + if (part.Length > remaining) + { + sb.Append(part.AsSpan(0, remaining)); + break; + } + + sb.Append(part); + first = false; + } + + answer = sb.Length == 0 ? null : sb.ToString(); } - answer = answerBuilder.ToString(); } //Insert Row From 9c3eedcb79c6d3c66d4ac513f6b542c7ba698113 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 19:13:14 +0200 Subject: [PATCH 42/49] Fixed max log records limit --- Apps/QueryLogsDuckDBApp/App.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 86c02cc34..50de988bd 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -756,7 +756,7 @@ private class Config public int MaxLogDays { get; set; } = 30; [JsonPropertyName("maxLogRecords")] - [Range(1, 5_000_000_000)] + [Range(1, 5_000_000)] public long MaxLogRecords { get; set; } = 1_000_000; [JsonPropertyName("maxQueueSize")] From 2e60d0ea8557714f73ae5cb4713b109c8dd1e6fb Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 21:20:44 +0200 Subject: [PATCH 43/49] Added aggregated exception logging --- Apps/QueryLogsDuckDBApp/App.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 50de988bd..76030258d 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -424,10 +424,18 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) await CreateSchemaAsync(); _consumerTask = Task.Run(ProcessLogsAsync).ContinueWith( - t => { var _ = t.Exception; }, + t => + { + if (t.Exception != null) + _dnsServer?.WriteLog(t.Exception); + }, TaskContinuationOptions.OnlyOnFaulted); _retentionTask = Task.Run(RetentionLoopAsync).ContinueWith( - t => { var _ = t.Exception; }, + t => + { + if (t.Exception != null) + _dnsServer?.WriteLog(t.Exception); + }, TaskContinuationOptions.OnlyOnFaulted); } From 00d1bfded9d20498b9c49419810a2e3c99450fd9 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 21:23:25 +0200 Subject: [PATCH 44/49] Improved Dispose --- Apps/QueryLogsDuckDBApp/App.cs | 42 +++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 76030258d..7fa925bec 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -80,22 +80,52 @@ private void Dispose(bool disposing) try { _channel?.Writer.TryComplete(); } catch (Exception ex) { - Console.Error.WriteLine("QueryLogsDuckDB.App: Error while completing log channel during Dispose: " + ex); + _dnsServer?.WriteLog("QueryLogsDuckDB.App: Error while completing log channel during Dispose: " + ex); + } + try + { + if (_consumerTask != null) + { + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(5000)); + try + { + _consumerTask.WaitAsync(cts.Token).GetAwaiter().GetResult(); + } + catch (OperationCanceledException) + { + // Ignore timeout/cancellation during dispose to avoid blocking shutdown indefinitely. + } + _consumerTask.Dispose(); + } } - try { _consumerTask?.Wait(5000); _consumerTask?.Dispose(); } catch (Exception ex) { - Console.Error.WriteLine("QueryLogsDuckDB.App: Error while waiting for consumer task during Dispose: " + ex); + _dnsServer?.WriteLog("QueryLogsDuckDB.App: Error while waiting for consumer task during Dispose: " + ex); + } + try + { + if (_retentionTask != null) + { + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(5000)); + try + { + _retentionTask.WaitAsync(cts.Token).GetAwaiter().GetResult(); + } + catch (OperationCanceledException) + { + // Ignore timeout/cancellation during dispose to avoid blocking shutdown indefinitely. + } + _retentionTask.Dispose(); + } } - try { _retentionTask?.Wait(5000); _retentionTask?.Dispose(); } catch (Exception ex) { - Console.Error.WriteLine("QueryLogsDuckDB.App: Error while waiting for retention task during Dispose: " + ex); + _dnsServer?.WriteLog("QueryLogsDuckDB.App: Error while waiting for retention task during Dispose: " + ex); } try { _conn?.Close(); _conn?.Dispose(); } catch (Exception ex) { - Console.Error.WriteLine("QueryLogsDuckDB.App: Error while closing/disposing DuckDB connection during Dispose: " + ex); + _dnsServer?.WriteLog("QueryLogsDuckDB.App: Error while closing/disposing DuckDB connection during Dispose: " + ex); } } From 05b589fb6f3693b4f3fe5bd0b4ddfd91780042f8 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 21:24:08 +0200 Subject: [PATCH 45/49] Disposed semaphore --- Apps/QueryLogsDuckDBApp/App.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 7fa925bec..2e11b7b74 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -127,6 +127,8 @@ private void Dispose(bool disposing) { _dnsServer?.WriteLog("QueryLogsDuckDB.App: Error while closing/disposing DuckDB connection during Dispose: " + ex); } + + _dbGate.Dispose(); } _disposed = true; From 7fb2ba0f9ec7e4e4f2b5b4993df7446d22fe3363 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 21:27:33 +0200 Subject: [PATCH 46/49] Added lock for query --- Apps/QueryLogsDuckDBApp/App.cs | 112 +++++++++++++++++---------------- 1 file changed, 59 insertions(+), 53 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 2e11b7b74..4df75a6e0 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -680,72 +680,78 @@ FROM dns_logs /* --------------------------------- Read --------------------------------- */ + await _dbGate.WaitAsync(); + try + { + using System.Data.Common.DbDataReader reader = await cmd.ExecuteReaderAsync(); - using System.Data.Common.DbDataReader reader = - await cmd.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + DateTime ts = reader.GetDateTime(1); - while (await reader.ReadAsync()) - { - DateTime ts = reader.GetDateTime(1); + IPAddress ip = + IPAddress.Parse(reader.GetString(2)); - IPAddress ip = - IPAddress.Parse(reader.GetString(2)); + DnsTransportProtocol proto = + (DnsTransportProtocol)reader.GetByte(3); - DnsTransportProtocol proto = - (DnsTransportProtocol)reader.GetByte(3); + DnsServerResponseType respType = + (DnsServerResponseType)reader.GetByte(4); - DnsServerResponseType respType = - (DnsServerResponseType)reader.GetByte(4); + double? rtt = + reader.IsDBNull(5) + ? null + : reader.GetDouble(5); - double? rtt = - reader.IsDBNull(5) - ? null - : reader.GetDouble(5); + DnsResponseCode rc = + (DnsResponseCode)reader.GetByte(6); - DnsResponseCode rc = - (DnsResponseCode)reader.GetByte(6); + string? qn = + reader.IsDBNull(7) + ? null + : reader.GetString(7); - string? qn = - reader.IsDBNull(7) - ? null - : reader.GetString(7); + DnsQuestionRecord? question = null; - DnsQuestionRecord? question = null; + if (qn is not null && + !reader.IsDBNull(8) && + !reader.IsDBNull(9)) + { + question = new DnsQuestionRecord( + qn, + (DnsResourceRecordType)reader.GetFieldValue(8), + (DnsClass)reader.GetFieldValue(9), + false); + } - if (qn is not null && - !reader.IsDBNull(8) && - !reader.IsDBNull(9)) - { - question = new DnsQuestionRecord( - qn, - (DnsResourceRecordType)reader.GetFieldValue(8), - (DnsClass)reader.GetFieldValue(9), - false); + string? ans = + reader.IsDBNull(10) + ? null + : reader.GetString(10); + + list.Add( + new DnsLogEntry( + 0, + ts, + ip, + proto, + respType, + rtt, + rc, + question, + ans)); } - string? ans = - reader.IsDBNull(10) - ? null - : reader.GetString(10); - - list.Add( - new DnsLogEntry( - 0, - ts, - ip, - proto, - respType, - rtt, - rc, - question, - ans)); + return new DnsLogPage( + pageNumber, + totalPages, + totalEntries, + list); + } + finally + { + _dbGate.Release(); } - - return new DnsLogPage( - pageNumber, - totalPages, - totalEntries, - list); } #endregion public From 91f5c2c44b0024c2e06aed8d231e10a791d33840 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 21:28:38 +0200 Subject: [PATCH 47/49] Removed duplicate --- Apps/QueryLogsDuckDBApp/App.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index 4df75a6e0..aee35c752 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -387,7 +387,6 @@ LIMIT 1 DELETE FROM dns_logs WHERE timestamp < (SELECT timestamp FROM cutoff); "; - cmd.Parameters.Clear(); cmd.Parameters.Add( new DuckDBParameter("limit", _config.MaxLogRecords)); From ac3fa71c6bd7c6195960d66611745c60c8b5a3dc Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 21:29:34 +0200 Subject: [PATCH 48/49] Updated ranges to match SQLite and MySQL Apps' behavior --- Apps/QueryLogsDuckDBApp/App.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index aee35c752..c84103e7c 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -797,11 +797,11 @@ private class Config public bool EnableLogging { get; set; } = true; [JsonPropertyName("maxLogDays")] - [Range(1, 365)] + [Range(0, 365)] public int MaxLogDays { get; set; } = 30; [JsonPropertyName("maxLogRecords")] - [Range(1, 5_000_000)] + [Range(0, 5_000_000)] public long MaxLogRecords { get; set; } = 1_000_000; [JsonPropertyName("maxQueueSize")] From 4a56cf05decceaf6d621bf01dfa077c750f0d1a6 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 28 Jan 2026 21:32:19 +0200 Subject: [PATCH 49/49] Promoted JsonSerializerOptions --- Apps/QueryLogsDuckDBApp/App.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Apps/QueryLogsDuckDBApp/App.cs b/Apps/QueryLogsDuckDBApp/App.cs index c84103e7c..976fac43a 100644 --- a/Apps/QueryLogsDuckDBApp/App.cs +++ b/Apps/QueryLogsDuckDBApp/App.cs @@ -49,15 +49,15 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs [ThreadStatic] private static StringBuilder? _sb; + private readonly SemaphoreSlim _dbGate = new(1, 1); + private readonly JsonSerializerOptions _options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; private Channel? _channel; Config? _config; private DuckDBConnection? _conn; private Task? _consumerTask; - private Task? _retentionTask; private bool _disposed; private IDnsServer? _dnsServer; - private readonly SemaphoreSlim _dbGate = new(1, 1); - + private Task? _retentionTask; #endregion variables #region IDisposable @@ -434,8 +434,7 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config) { _dnsServer = dnsServer; - JsonSerializerOptions options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; - _config = JsonSerializer.Deserialize(config, options); + _config = JsonSerializer.Deserialize(config, _options); _config ??= new Config(); Validator.ValidateObject(_config, new ValidationContext(_config), validateAllProperties: true);