Skip to content

Added extended logging capability with metadata in addition to using … #1827

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zbalkan wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Added extended logging capability with metadata in addition to using … #1827

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions Apps/AdvancedBlockingApp/App.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -605,14 +605,36 @@ string GetBlockingReport()
return blockingReport;
}

DnsQueryLogMetadata GetLogMetadata()
{
Dictionary<string, string> values = new Dictionary<string, string>(6, StringComparer.OrdinalIgnoreCase)
{
["source"] = "advanced-blocking-app",
["group"] = group.Name
};

if (blockListUrl?.Uri is not null)
values["blockListUrl"] = blockListUrl.Uri.AbsoluteUri;

Comment thread
zbalkan marked this conversation as resolved.
if (blockedRegex is null)
values["domain"] = blockedDomain ?? question.Name;
else
values["regex"] = blockedRegex;

return new DnsQueryLogMetadata(values);
}

DnsQueryLogMetadata logMetadata = GetLogMetadata();
DnsServerResponseMetadata responseMetadata = new DnsServerResponseMetadata(DnsServerResponseType.Blocked, logMetadata);

if (group.AllowTxtBlockingReport && (question.Type == DnsResourceRecordType.TXT))
{
//return meta data
string blockingReport = GetBlockingReport();

DnsResourceRecord[] answer = [new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, _blockingAnswerTtl, new DnsTXTRecordData(blockingReport))];

return Task.FromResult<DnsDatagram?>(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer));
return Task.FromResult<DnsDatagram?>(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer) { Tag = responseMetadata });
}
else
{
Expand Down Expand Up @@ -697,7 +719,7 @@ string GetBlockingReport()
}
}

return Task.FromResult<DnsDatagram?>(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, rcode, request.Question, answer, authority, null, request.EDNS is null ? ushort.MinValue : _dnsServer!.UdpPayloadSize, EDnsHeaderFlags.None, options));
return Task.FromResult<DnsDatagram?>(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, rcode, request.Question, answer, authority, null, request.EDNS is null ? ushort.MinValue : _dnsServer!.UdpPayloadSize, EDnsHeaderFlags.None, options) { Tag = responseMetadata });
}
}

Expand Down
9 changes: 7 additions & 2 deletions Apps/LogExporterApp/App.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ You should have received a copy of the GNU General Public License

namespace LogExporter
{
public sealed class App : IDnsApplication, IDnsQueryLogger
public sealed class App : IDnsApplication, IDnsQueryLoggerEx
{
#region variables

Expand Down Expand Up @@ -138,11 +138,16 @@ public Task InitializeAsync(IDnsServer dnsServer, string config)
}

public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response)
{
return InsertLogAsync(timestamp, request, remoteEP, protocol, response, null);
}

public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata)
{
if (_enableLogging)
{
if (_queuedLogs.Count < _config!.MaxQueueSize)
_queuedLogs.Enqueue(new LogEntry(timestamp, remoteEP, protocol, request, response, _config.EnableEdnsLogging));
_queuedLogs.Enqueue(new LogEntry(timestamp, remoteEP, protocol, request, response, _config.EnableEdnsLogging, metadata));
}

return Task.CompletedTask;
Expand Down
23 changes: 17 additions & 6 deletions Apps/LogExporterApp/LogEntry.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,17 @@ namespace LogExporter
{
public class LogEntry
{
public LogEntry(DateTime timestamp, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram request, DnsDatagram response, bool ednsLogging = false)
public LogEntry(DateTime timestamp, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram request, DnsDatagram response, bool ednsLogging = false, DnsQueryLogMetadata? metadata = null)
{
// Assign timestamp and ensure it's in UTC
Timestamp = timestamp.Kind == DateTimeKind.Utc ? timestamp : timestamp.ToUniversalTime();

// Extract client information
ClientIp = remoteEP.Address.ToString();
Protocol = protocol;
ResponseType = response.Tag == null ? DnsServerResponseType.Recursive : (DnsServerResponseType)response.Tag;
ResponseType = DnsServerResponseTag.GetResponseType(response.Tag);
DnsQueryLogMetadata? logMetadata = metadata ?? DnsServerResponseTag.GetLogMetadata(response.Tag);
BlockingMetadata = logMetadata?.Values;

if ((ResponseType == DnsServerResponseType.Recursive) && (response.Metadata is not null))
ResponseRtt = response.Metadata.RoundTripTime;
Expand Down Expand Up @@ -84,18 +86,27 @@ public LogEntry(DateTime timestamp, IPEndPoint remoteEP, DnsTransportProtocol pr

foreach (EDnsOption extendedErrorLog in response.EDNS.Options.Where(o => o.Code == EDnsOptionCode.EXTENDED_DNS_ERROR))
{
string[] extractedData = extendedErrorLog.Data.ToString().Replace("[", string.Empty).Replace("]", string.Empty).Split(":", StringSplitOptions.TrimEntries);
string[] extractedData = extendedErrorLog.Data.ToString().Replace("[", string.Empty).Replace("]", string.Empty).Split(":", 2, StringSplitOptions.TrimEntries);
string? errType = null;
string? message = null;

if (extractedData.Length > 0)
errType = extractedData[0];

if (extractedData.Length > 1)
message = extractedData[1];

EDNS.Add(new EDNSLog
{
ErrType = extractedData[0],
Message = extractedData[1]
ErrType = errType,
Message = message
});
}
}

public List<DnsResourceRecord> Answers { get; private set; }
public string ClientIp { get; private set; }
public IReadOnlyDictionary<string, string>? BlockingMetadata { get; private set; }
public List<EDNSLog> EDNS { get; private set; }
public DnsTransportProtocol Protocol { get; private set; }
public DnsQuestion? Question { get; private set; }
Expand Down Expand Up @@ -158,4 +169,4 @@ public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializer
}
}
}
}
}
9 changes: 9 additions & 0 deletions Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,15 @@ private static LogEvent Convert(LogEntry log)
}
}

// Add blocking metadata
if ((log.BlockingMetadata is not null) && (log.BlockingMetadata.Count > 0))
{
foreach (KeyValuePair<string, string> item in log.BlockingMetadata)
properties.Add(new LogEventProperty("blocking_" + item.Key, new ScalarValue(item.Value)));

properties.Add(new LogEventProperty("blockingMetadataSummary", new ScalarValue(string.Join(", ", log.BlockingMetadata.Select(kv => kv.Key + "=" + kv.Value)))));
}

// Define the message template to match the original summary format
const string templateText = "{questionsSummary}; RCODE: {rCode}; ANSWER: [{answersSummary}]";

Expand Down
16 changes: 11 additions & 5 deletions Apps/MispConnectorApp/App.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,16 +150,22 @@ public Task<DnsDatagram> ProcessRequestAsync(DnsDatagram request, IPEndPoint rem
}

string blockingReport = $"source=misp-connector;domain={blockedDomain}";
DnsQueryLogMetadata logMetadata = new DnsQueryLogMetadata(new Dictionary<string, string>(2, StringComparer.OrdinalIgnoreCase)
{
["source"] = "misp-connector",
["domain"] = blockedDomain
});
DnsServerResponseMetadata responseMetadata = new DnsServerResponseMetadata(DnsServerResponseType.Blocked, logMetadata);

EDnsOption[] options = null;
if (_config.AddExtendedDnsError && request.EDNS is not null)
{
options = new EDnsOption[] { new EDnsOption(EDnsOptionCode.EXTENDED_DNS_ERROR, new EDnsExtendedDnsErrorOptionData(EDnsExtendedDnsErrorCode.Blocked, string.Empty)) };
options = new EDnsOption[] { new EDnsOption(EDnsOptionCode.EXTENDED_DNS_ERROR, new EDnsExtendedDnsErrorOptionData(EDnsExtendedDnsErrorCode.Blocked, blockingReport)) };
}

if (_config.AllowTxtBlockingReport && question.Type == DnsResourceRecordType.TXT)
{
DnsResourceRecord[] answer = new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData(string.Empty)) };
DnsResourceRecord[] answer = new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData(blockingReport)) };
return Task.FromResult(new DnsDatagram(
ID: request.Identifier,
isResponse: true,
Expand All @@ -178,7 +184,7 @@ public Task<DnsDatagram> ProcessRequestAsync(DnsDatagram request, IPEndPoint rem
udpPayloadSize: request.EDNS is null ? ushort.MinValue : _dnsServer.UdpPayloadSize,
ednsFlags: EDnsHeaderFlags.None,
options: options
));
) { Tag = responseMetadata });
}

DnsResourceRecord[] authority = { new DnsResourceRecord(question.Name, DnsResourceRecordType.SOA, question.Class, 60, _soaRecord) };
Expand All @@ -200,7 +206,7 @@ public Task<DnsDatagram> ProcessRequestAsync(DnsDatagram request, IPEndPoint rem
udpPayloadSize: request.EDNS is null ? ushort.MinValue : _dnsServer.UdpPayloadSize,
ednsFlags: EDnsHeaderFlags.None,
options: options
));
) { Tag = responseMetadata });
}

#endregion public
Expand Down Expand Up @@ -590,4 +596,4 @@ private class MispResponseData
public List<MispAttribute> Attribute { get; set; }
}
}
}
}
47 changes: 34 additions & 13 deletions Apps/QueryLogsMySqlApp/App.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ You should have received a copy of the GNU General Public License

namespace QueryLogsMySql
{
public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs
public sealed class App : IDnsApplication, IDnsQueryLoggerEx, IDnsQueryLogs
{
#region variables

Expand Down Expand Up @@ -216,14 +216,14 @@ private async Task BulkInsertLogsAsync(List<LogEntry> logs, StringBuilder sb)
await using (MySqlCommand command = connection.CreateCommand())
{
sb.Length = 0;
sb.Append("INSERT INTO dns_logs (server, timestamp, client_ip, protocol, response_type, response_rtt, rcode, qname, qtype, qclass, answer) VALUES ");
sb.Append("INSERT INTO dns_logs (server, timestamp, client_ip, protocol, response_type, response_rtt, rcode, qname, qtype, qclass, answer, blocking_metadata) VALUES ");

for (int i = 0; i < logs.Count; i++)
{
if (i == 0)
sb.Append($"(@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i})");
sb.Append($"(@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i}, @blocking_metadata{i})");
else
sb.Append($", (@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i})");
sb.Append($", (@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i}, @blocking_metadata{i})");
}
command.CommandText = sb.ToString();

Expand All @@ -242,18 +242,14 @@ private async Task BulkInsertLogsAsync(List<LogEntry> logs, StringBuilder sb)
MySqlParameter paramQtype = command.Parameters.Add("@qtype" + i, MySqlDbType.Int16);
MySqlParameter paramQclass = command.Parameters.Add("@qclass" + i, MySqlDbType.Int16);
MySqlParameter paramAnswer = command.Parameters.Add("@answer" + i, MySqlDbType.VarChar);
MySqlParameter paramBlockingMetadata = command.Parameters.Add("@blocking_metadata" + i, MySqlDbType.Text);

paramServer.Value = _dnsServer?.ServerDomain;
paramTimestamp.Value = log.Timestamp;
paramClientIp.Value = log.RemoteEP.Address.ToString();
paramProtocol.Value = (byte)log.Protocol;

DnsServerResponseType responseType;

if (log.Response.Tag == null)
responseType = DnsServerResponseType.Recursive;
else
responseType = (DnsServerResponseType)log.Response.Tag;
DnsServerResponseType responseType = DnsServerResponseTag.GetResponseType(log.Response.Tag);

paramResponseType.Value = (byte)responseType;

Expand Down Expand Up @@ -304,6 +300,11 @@ private async Task BulkInsertLogsAsync(List<LogEntry> logs, StringBuilder sb)

paramAnswer.Value = answer;
}

if (log.Metadata is null)
paramBlockingMetadata.Value = DBNull.Value;
else
paramBlockingMetadata.Value = JsonSerializer.Serialize(log.Metadata.Values);
}

await command.ExecuteNonQueryAsync();
Expand Down Expand Up @@ -375,7 +376,8 @@ client_ip VARCHAR(39) NOT NULL,
qname VARCHAR(255),
qtype SMALLINT,
qclass SMALLINT,
answer VARCHAR(4000)
answer VARCHAR(4000),
blocking_metadata TEXT
);
";

Expand Down Expand Up @@ -406,6 +408,18 @@ answer VARCHAR(4000)
{ }
}

await using (MySqlCommand command = connection.CreateCommand())
{
command.CommandText = "ALTER TABLE dns_logs ADD blocking_metadata TEXT;";

try
{
await command.ExecuteNonQueryAsync();
}
catch
{ }
}

await using (MySqlCommand command = connection.CreateCommand())
{
command.CommandText = "CREATE INDEX index_server ON dns_logs (server);";
Expand Down Expand Up @@ -666,9 +680,14 @@ answer VARCHAR(4000)
}

public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response)
{
return InsertLogAsync(timestamp, request, remoteEP, protocol, response, null);
}

public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata)
{
if (_enableLogging)
_channelWriter?.TryWrite(new LogEntry(timestamp, request, remoteEP, protocol, response));
_channelWriter?.TryWrite(new LogEntry(timestamp, request, remoteEP, protocol, response, metadata));

return Task.CompletedTask;
}
Expand Down Expand Up @@ -892,18 +911,20 @@ readonly struct LogEntry
public readonly IPEndPoint RemoteEP;
public readonly DnsTransportProtocol Protocol;
public readonly DnsDatagram Response;
public readonly DnsQueryLogMetadata? Metadata;

#endregion

#region constructor

public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response)
public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata = null)
{
Timestamp = timestamp;
Request = request;
RemoteEP = remoteEP;
Protocol = protocol;
Response = response;
Metadata = metadata ?? DnsServerResponseTag.GetLogMetadata(response.Tag);
}

#endregion
Expand Down
Loading
Loading