From 14dfc62ac2e8eb1178bfa19ed903db195a4e61db Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Sun, 27 Feb 2022 14:51:13 -0800 Subject: [PATCH 1/8] sort sidecar, this makes diffing easier --- src/clogutils/CLogDecodedTraceLine.cs | 4 ++-- src/clogutils/CLogEncodingCLogTypeSearch.cs | 2 +- src/clogutils/CLogSidecar.cs | 8 +++++--- src/clogutils/ConfigFile/CLogModuleUsageInformation.cs | 5 +++++ 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/clogutils/CLogDecodedTraceLine.cs b/src/clogutils/CLogDecodedTraceLine.cs index 72288bf..14ed12a 100644 --- a/src/clogutils/CLogDecodedTraceLine.cs +++ b/src/clogutils/CLogDecodedTraceLine.cs @@ -20,7 +20,7 @@ namespace clogutils [JsonObject(MemberSerialization.OptIn)] public class CLogDecodedTraceLine { - [JsonProperty] public Dictionary> ModuleProperites = new Dictionary>(); + [JsonProperty] public SortedDictionary> ModuleProperites = new SortedDictionary>(); public CLogDecodedTraceLine(string uniqueId, string sourceFile, string userString, string userStringNoPrefix, CLogLineMatch m, CLogConfigurationFile c, CLogTraceMacroDefination mac, CLogFileProcessor.CLogVariableBundle[] args, CLogFileProcessor.DecomposedString decompString) @@ -142,7 +142,7 @@ public void AddConfigFileProperty(string module, string key, string value) if (!ModuleProperites.ContainsKey(module)) { - ModuleProperites[module] = new Dictionary(); + ModuleProperites[module] = new SortedDictionary(); } ModuleProperites[module][key] = value; diff --git a/src/clogutils/CLogEncodingCLogTypeSearch.cs b/src/clogutils/CLogEncodingCLogTypeSearch.cs index ecca173..b9a68c7 100644 --- a/src/clogutils/CLogEncodingCLogTypeSearch.cs +++ b/src/clogutils/CLogEncodingCLogTypeSearch.cs @@ -42,7 +42,7 @@ public bool IsEncodableArg { get { - return !Synthesized && + return !Synthesized && (EncodingType != CLogEncodingType.UniqueAndDurableIdentifier && EncodingType != CLogEncodingType.UserEncodingString); } diff --git a/src/clogutils/CLogSidecar.cs b/src/clogutils/CLogSidecar.cs index d07fec0..f4e5f0b 100644 --- a/src/clogutils/CLogSidecar.cs +++ b/src/clogutils/CLogSidecar.cs @@ -25,7 +25,7 @@ public class ClogSidecar_V1 [JsonProperty] public int Version { get; set; } [JsonProperty] - public Dictionary EventBundlesV2 { get; set; } = new Dictionary(); + public SortedDictionary EventBundlesV2 { get; set; } = new SortedDictionary(); [JsonProperty] public CLogConfigurationFile ConfigFile { get; set; } @@ -67,7 +67,7 @@ public class ClogSidecar_V2 [JsonProperty] public int Version { get; set; } [JsonProperty] - public Dictionary EventBundlesV2 { get; set; } = new Dictionary(); + public SortedDictionary EventBundlesV2 { get; set; } = new SortedDictionary(); [JsonProperty] public CLogConfigurationFile ConfigFile { get; set; } @@ -85,6 +85,8 @@ public ClogSidecar_V2() public string ToJson() { + ModuleUniqueness.Sort(); + JsonSerializerSettings s = new JsonSerializerSettings(); s.Formatting = Formatting.Indented; this.Version = 2; @@ -126,7 +128,7 @@ public class CLogSidecar : ICLogOutputModule { private string _sidecarFileName; - private Dictionary>> ModuleTraceData = new Dictionary>>(); + private SortedDictionary>> ModuleTraceData = new SortedDictionary>>(); private ClogSidecar_V2 _sideCarFile; diff --git a/src/clogutils/ConfigFile/CLogModuleUsageInformation.cs b/src/clogutils/ConfigFile/CLogModuleUsageInformation.cs index 501f495..95f446b 100644 --- a/src/clogutils/ConfigFile/CLogModuleUsageInformation.cs +++ b/src/clogutils/ConfigFile/CLogModuleUsageInformation.cs @@ -36,6 +36,11 @@ public static CLogModuleUsageInformation_V2 ConvertFromV1(CLogModuleUsageInforma } return ret; } + + public void Sort() + { + TraceInformation.Sort((a, b) => a.TraceID.CompareTo(b.TraceID)); + } } From b37a81ff7e666cfed7a52d87bc8e955274db06b6 Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Sun, 27 Feb 2022 15:00:59 -0800 Subject: [PATCH 2/8] Fix build break in Windows - where the output Dictionary needs to be a Sorted Dictionary --- src/clog2text/clog2text_windows/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clog2text/clog2text_windows/Program.cs b/src/clog2text/clog2text_windows/Program.cs index b32d4e7..4ac9257 100644 --- a/src/clog2text/clog2text_windows/Program.cs +++ b/src/clog2text/clog2text_windows/Program.cs @@ -101,7 +101,7 @@ private static int Main(string[] cmdLineArgs) foreach (var b in sidecar.EventBundlesV2) { - Dictionary keys; + SortedDictionary keys; if (!e.IsTraceLogging) { From 3a06ad15b94f4e6876871c6b439d1946ee42426c Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Sun, 27 Feb 2022 15:27:09 -0800 Subject: [PATCH 3/8] Add UserEvents emittor --- src/clog/TraceEmitterModules/CLogDiagTrace.cs | 367 ++++++++++++++++++ src/clog/TraceEmitterModules/CLogSTDOUT.cs | 42 +- src/clog/clog.cs | 3 + 3 files changed, 401 insertions(+), 11 deletions(-) create mode 100644 src/clog/TraceEmitterModules/CLogDiagTrace.cs diff --git a/src/clog/TraceEmitterModules/CLogDiagTrace.cs b/src/clog/TraceEmitterModules/CLogDiagTrace.cs new file mode 100644 index 0000000..a86db34 --- /dev/null +++ b/src/clog/TraceEmitterModules/CLogDiagTrace.cs @@ -0,0 +1,367 @@ +/*++ + + Copyright (c) Microsoft Corporation. + Licensed under the MIT License. + +Abstract: + + A stream manages the send and receive queues for application data. This file + contains the initialization and cleanup functionality for the stream. + +--*/ + +using clogutils; +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using clogutils.MacroDefinations; + +namespace clog.TraceEmitterModules +{ + public class CLogDiagTrace : ICLogOutputModule + { + public string ModuleName + { + get + { + return "UDIAG"; + } + } + + public bool ManditoryModule + { + get + { + return false; + } + } + + public void InitHeader(StringBuilder header) + { + header.AppendLine($"// CLOG DiagTrace Experiment ------"); + + header.AppendLine($"#ifndef CLOG_DIAGTRACE_BPF"); + header.AppendLine($"#include "); + header.AppendLine($"#include "); + header.AppendLine($"#endif"); + + header.AppendLine($"#define DIAG_MIN(a,b) (((a)<(b))?(a):(b))"); + + header.AppendLine($"#ifndef CLOG_DIAGTRACE_UNSET_BPF_TYPES"); + header.AppendLine($"#define DIAG_TRACE_8 char"); + header.AppendLine($"#define DIAG_TRACE_U8 unsigned char"); + + header.AppendLine($"#define DIAG_TRACE_16 short"); + header.AppendLine($"#define DIAG_TRACE_U16 unsigned short"); + + header.AppendLine($"#define DIAG_TRACE_32 int"); + header.AppendLine($"#define DIAG_TRACE_U32 unsigned int"); + + header.AppendLine($"#define DIAG_TRACE_64 long long"); + header.AppendLine($"#define DIAG_TRACE_U64 unsigned long long"); + + header.AppendLine($"#define DIAG_TRACE_POINTER void *"); + header.AppendLine($"#endif"); + } + + + public void FinishedProcessing(CLogOutputInfo outputInfo, StringBuilder header, StringBuilder sourceFile) + { + string ebpfHeaderFileName = Path.Combine(outputInfo.OutputDirectory, "ebpfDecode.h"); + string ebpfHeader = ""; + string myHeader = "#include <" + Path.GetFileName(outputInfo.OutputFileName) + ">\n"; + + if(!Directory.Exists(Path.GetDirectoryName(ebpfHeaderFileName))) + Directory.CreateDirectory(Path.GetDirectoryName(ebpfHeaderFileName)); + + if(File.Exists(ebpfHeaderFileName)) + { + ebpfHeader = File.ReadAllText(ebpfHeaderFileName); + File.Delete(ebpfHeaderFileName); + } + else + { + ebpfHeader = $"#define CLOG_DECODE_ALL(dataPtr, dataLen) \\\n".Replace(".","_"); + } + + if(!ebpfHeader.Contains(myHeader)) + ebpfHeader = myHeader + ebpfHeader; + + + foreach(string s in decoderMacros) + { + if(!ebpfHeader.Contains(s)) + ebpfHeader += s.ToString(); + } + + File.WriteAllText(ebpfHeaderFileName, ebpfHeader); + } + + private HashSet STRUCT_DEFS = new HashSet(); + private List decoderMacros = new List(); + private StringBuilder printFunctions = new StringBuilder(); + + public void TraceLineDiscovered(string sourceFile, CLogOutputInfo outputInfo, CLogDecodedTraceLine decodedTraceLine, CLogSidecar sidecar, StringBuilder headerFile, StringBuilder macroBody, StringBuilder outputSourceFile) + { + CLogExportModuleDefination moduleSettings = decodedTraceLine.GetMacroConfigurationProfile().FindExportModule(this.ModuleName); + + if (!moduleSettings.CustomSettings.ContainsKey("ProviderIndex")) + throw new CLogEnterReadOnlyModeException("ProviderIndex missing from CLOG config file entry for DiagTrace", CLogHandledException.ExceptionType.RequiredConfigParameterUnspecified, decodedTraceLine.match); + + if (!moduleSettings.CustomSettings.ContainsKey("ProviderHandle")) + throw new CLogEnterReadOnlyModeException("ProviderHandle missing from CLOG config file entry for DiagTrace", CLogHandledException.ExceptionType.RequiredConfigParameterUnspecified, decodedTraceLine.match); + + string providerIndex = moduleSettings.CustomSettings["ProviderIndex"]; + string providerHandle = moduleSettings.CustomSettings["ProviderHandle"]; + + headerFile.AppendLine($"extern int " + providerIndex + ";"); + headerFile.AppendLine($"extern int " + providerHandle + ";"); + + StringBuilder structDef = new StringBuilder(); + string structName = "CLOG_DIAGTRACE_" + decodedTraceLine.UniqueId.ToUpper(); + + string traceString = decodedTraceLine.TraceString; + while(traceString.Contains("%!")) + { + traceString = traceString.Replace("%!", "[0x%p]!"); + } + + string existingId = decodedTraceLine.GetConfigFileProperty(ModuleName, "EventID"); + string durableId = decodedTraceLine.UniqueId;//decodedTraceLine.UniqueId.ToString(); + + //durableId = decodedTraceLine.UniqueId.GetHashCode().ToString(); + //if(!durableId.Equals(existingId)) + // throw new CLogEnterReadOnlyModeException("DurableID hash algorithm failed", CLogHandledException.ExceptionType.DuplicateId, decodedTraceLine.match); + + decodedTraceLine.AddConfigFileProperty(ModuleName, "EventID", durableId.ToString()); + + structDef.AppendLine($"#define {structName}_ID \"{durableId}\""); + structDef.AppendLine($"#define {structName}_TEXT \"{decodedTraceLine.DecomposedString.AsPrintF}\""); + + string msg = ""; + msg += $" \\\nDIAG_TYPE({structName}, {structName}_ID, {structName}_FN, dataPtr, dataLen)"; + + decoderMacros.Add(msg); + + // NOTE: do not change this max size without also updating diagtrace.h + int idMaxSize = 48; + int currentOffset = 0; + + structDef.AppendLine("#pragma pack(1)"); + structDef.AppendLine("typedef struct " + structName); + structDef.AppendLine("{"); + structDef.AppendLine($" struct CLOG_UDIAG_EVENT eventHeader;"); + + decodedTraceLine.AddConfigFileProperty(ModuleName, $"clog_ID", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"clog_ID_Len", idMaxSize.ToString()); + currentOffset += idMaxSize; + + macroBody.AppendLine("{\\"); + macroBody.AppendLine(" if (diagtrace_register_page[" + providerIndex + "])\\"); + macroBody.AppendLine(" {\\"); + macroBody.AppendLine(" " +structName + " myEvent = {0};\\"); + macroBody.AppendLine(" strncpy(myEvent.eventHeader.clog_ID, \"" +durableId + "\", "+(idMaxSize-1).ToString()+");\\"); + macroBody.AppendLine(" myEvent.eventHeader.clog_ID["+(idMaxSize-1) +"] = 0;\\"); + + // Only define a struct once + if(STRUCT_DEFS.Contains(decodedTraceLine.UniqueId.ToUpper())) + return; + STRUCT_DEFS.Add(decodedTraceLine.UniqueId.ToUpper()); + + string args = ""; + bool hasAnyDefs = false; + + foreach (var a in decodedTraceLine.splitArgs) + { + CLogFileProcessor.CLogVariableBundle arg = a; + + if (!arg.TypeNode.IsEncodableArg) + continue; + + CLogEncodingCLogTypeSearch node = decodedTraceLine.configFile.FindType(arg, decodedTraceLine); + + switch (node.EncodingType) + { + case CLogEncodingType.Synthesized: + structDef.AppendLine( $" // Skipping Synthesized {arg.VariableInfo.SuggestedTelemetryName};"); + continue; + + case CLogEncodingType.Skip: + structDef.AppendLine( $" // Skipping Skipped {arg.VariableInfo.SuggestedTelemetryName};"); + continue; + } + + switch (node.EncodingType) + { + case CLogEncodingType.ByteArray: + //BUGBUG : Limit to 32. + structDef.AppendLine( $" DIAG_TRACE_8 clog_{arg.VariableInfo.SuggestedTelemetryName}[32]; // BUGBUG Limited"); + macroBody.AppendLine($" if ({arg.MacroVariableName}_len > 0) " +"{ \\"); + macroBody.AppendLine($" CLOG_COPY(myEvent.clog_{arg.VariableInfo.SuggestedTelemetryName}, {arg.MacroVariableName}, DIAG_MIN(32, {arg.MacroVariableName}_len)); \\"); + macroBody.AppendLine( " } \\"); + + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}_Len", "32"); + currentOffset += 32; + + hasAnyDefs = true; + break; + + case CLogEncodingType.Int8: + structDef.AppendLine($" DIAG_TRACE_8 clog_{arg.VariableInfo.SuggestedTelemetryName};"); + macroBody.AppendLine(" myEvent.clog_" + arg.VariableInfo.SuggestedTelemetryName + " = (" + arg.MacroVariableName + "); \\"); + + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}_Len", "1"); + currentOffset += 1; + hasAnyDefs = true; + break; + + case CLogEncodingType.UInt8: + structDef.AppendLine($" DIAG_TRACE_U8 clog_{arg.VariableInfo.SuggestedTelemetryName};"); + macroBody.AppendLine(" myEvent.clog_" + arg.VariableInfo.SuggestedTelemetryName + " = (" + arg.MacroVariableName + "); \\"); + + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}_Len", "1"); + currentOffset += 1; + hasAnyDefs = true; + break; + + case CLogEncodingType.Int16: + structDef.AppendLine($" DIAG_TRACE_16 clog_{arg.VariableInfo.SuggestedTelemetryName};"); + macroBody.AppendLine(" myEvent.clog_" + arg.VariableInfo.SuggestedTelemetryName + " = (" + arg.MacroVariableName + "); \\"); + + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}_Len", "2"); + currentOffset += 2; + hasAnyDefs = true; + break; + + case CLogEncodingType.UInt16: + structDef.AppendLine($" DIAG_TRACE_U16 clog_{arg.VariableInfo.SuggestedTelemetryName};"); + macroBody.AppendLine(" myEvent.clog_" + arg.VariableInfo.SuggestedTelemetryName + " = (" + arg.MacroVariableName + "); \\"); + + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}_Len", "2"); + currentOffset += 2; + hasAnyDefs = true; + break; + + case CLogEncodingType.Int32: + structDef.AppendLine($" DIAG_TRACE_32 clog_{arg.VariableInfo.SuggestedTelemetryName};"); + macroBody.AppendLine(" myEvent.clog_" + arg.VariableInfo.SuggestedTelemetryName + " = (" + arg.MacroVariableName + "); \\"); + + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}_Len", "4"); + currentOffset += 4; + hasAnyDefs = true; + break; + + case CLogEncodingType.UInt32: + structDef.AppendLine($" DIAG_TRACE_U32 clog_{arg.VariableInfo.SuggestedTelemetryName};"); + macroBody.AppendLine(" myEvent.clog_" + arg.VariableInfo.SuggestedTelemetryName + " = (" + arg.MacroVariableName + "); \\"); + + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}_Len", "4"); + currentOffset += 4; + hasAnyDefs = true; + break; + + case CLogEncodingType.Int64: + structDef.AppendLine($" DIAG_TRACE_64 clog_{arg.VariableInfo.SuggestedTelemetryName};"); + macroBody.AppendLine(" myEvent.clog_" + arg.VariableInfo.SuggestedTelemetryName + " = (" + arg.MacroVariableName + "); \\"); + + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}_Len", "8"); + currentOffset += 8; + hasAnyDefs = true; + break; + + case CLogEncodingType.UInt64: + structDef.AppendLine($" DIAG_TRACE_U64 clog_{arg.VariableInfo.SuggestedTelemetryName};"); + macroBody.AppendLine(" myEvent.clog_" + arg.VariableInfo.SuggestedTelemetryName + " = (" + arg.MacroVariableName + "); \\"); + + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}_Len", "8"); + currentOffset += 8; + hasAnyDefs = true; + break; + + case CLogEncodingType.Pointer: + structDef.AppendLine( $" DIAG_TRACE_POINTER clog_{arg.VariableInfo.SuggestedTelemetryName};"); + macroBody.AppendLine(" myEvent.clog_" + arg.VariableInfo.SuggestedTelemetryName + " = (DIAG_TRACE_POINTER)(" + arg.MacroVariableName + "); \\"); + + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}_Len", "8"); + currentOffset += 8; + hasAnyDefs = true; + break; + + case CLogEncodingType.ANSI_String: + //BUGBUG : Limit to 132. + structDef.AppendLine( $" DIAG_TRACE_U8 clog_{arg.VariableInfo.SuggestedTelemetryName}[132]; // BUGBUG Limited"); + macroBody.AppendLine($" CLOG_COPY(myEvent.clog_{arg.VariableInfo.SuggestedTelemetryName}, {arg.MacroVariableName}, DIAG_MIN(132-1, strlen({arg.MacroVariableName}))); \\"); + macroBody.AppendLine($" myEvent.clog_{arg.VariableInfo.SuggestedTelemetryName}[131] = 0;\\"); + + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}_Len", "132"); + currentOffset += 132; + + hasAnyDefs = true; + break; + + case CLogEncodingType.UNICODE_String: + + //BUGBUG : Limit to 32. + structDef.AppendLine($" DIAG_TRACE_U16 clog_{arg.VariableInfo.SuggestedTelemetryName}[32]; // BUGBUG Limited"); + macroBody.AppendLine($" DIAG_TRACE_COPY_WCHAR(myEvent.clog_{arg.VariableInfo.SuggestedTelemetryName}, ({arg.MacroVariableName})); \\"); + macroBody.AppendLine($" myEvent.clog_{arg.VariableInfo.SuggestedTelemetryName}[31] = 0;\\"); + + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"{arg.VariableInfo.SuggestedTelemetryName}_Len", "32"); + currentOffset += 32; + + hasAnyDefs = true; + break; + + default: + throw new CLogEnterReadOnlyModeException("DiagTrace:UnknownType:" + node.EncodingType, CLogHandledException.ExceptionType.EncoderIncompatibleWithType, decodedTraceLine.match); + } + + if(!String.IsNullOrEmpty(args)) + args += ","; + args += $"CLOG_DECODED_ARG->clog_{arg.VariableInfo.SuggestedTelemetryName}"; + + } + + if (!hasAnyDefs) + { + structDef.AppendLine(" DIAG_TRACE_POINTER unused;"); + } + + + structDef.AppendLine($" int alignment;"); + macroBody.AppendLine(" myEvent.alignment = 0xAABBAABB; \\"); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"alignment", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"alignment_Len", "4"); + currentOffset += 4; + + structDef.AppendLine("} " + structName + ";"); + structDef.AppendLine("#pragma pack(0)"); + + + // Only define _ARGS if there are args - this makes build breaks, which are easier to detect + if(!String.IsNullOrEmpty(args)) + args = "," + args; + headerFile.AppendLine($"#define {structName}_ARGS(CLOG_DECODED_ARG) {args}"); + + headerFile.AppendLine(structDef.ToString()); + + macroBody.AppendLine(" DIAG_WRITE(" + providerHandle + ", &myEvent, sizeof(myEvent));\\"); + + macroBody.AppendLine(" }\\"); + macroBody.AppendLine("}\\"); + } + } +} diff --git a/src/clog/TraceEmitterModules/CLogSTDOUT.cs b/src/clog/TraceEmitterModules/CLogSTDOUT.cs index d47527a..aa4796f 100644 --- a/src/clog/TraceEmitterModules/CLogSTDOUT.cs +++ b/src/clog/TraceEmitterModules/CLogSTDOUT.cs @@ -57,9 +57,18 @@ public void TraceLineDiscovered(string sourceFile, CLogOutputInfo outputInfo, CL CLogExportModuleDefination moduleSettings = decodedTraceLine.GetMacroConfigurationProfile().FindExportModule(ModuleName); string printmacro; + string forceCastArg; + bool forceCast = false; + if (!moduleSettings.CustomSettings.TryGetValue("PrintMacro", out printmacro)) printmacro = "printf"; + if (moduleSettings.CustomSettings.TryGetValue("ForceCast", out forceCastArg)) + { + if(0 == forceCastArg.CompareTo("1")) + forceCast = true; + } + if (!emittedHeader) { string printHeader; @@ -122,10 +131,10 @@ public void TraceLineDiscovered(string sourceFile, CLogOutputInfo outputInfo, CL printf += "%u"; break; case CLogEncodingType.Int64: - printf += "%lld"; + printf += "%ld"; break; case CLogEncodingType.UInt64: - printf += "%llu"; + printf += "%lu"; break; case CLogEncodingType.ANSI_String: printf += "%s"; @@ -182,20 +191,26 @@ public void TraceLineDiscovered(string sourceFile, CLogOutputInfo outputInfo, CL switch (arg.TypeNode.EncodingType) { case CLogEncodingType.Int32: - //cast = "(int)"; + if(forceCast) + cast = "(int)"; break; case CLogEncodingType.UInt32: - //cast = "(unsigned int)"; + if(forceCast) + cast = "(unsigned int)"; break; case CLogEncodingType.Int64: - //cast = "(__int64)"; + if(forceCast) + cast = "(long)"; break; case CLogEncodingType.UInt64: - //cast = "(unsigned __int64)"; + if(forceCast) + cast = "(unsigned long)"; break; case CLogEncodingType.ANSI_String: break; case CLogEncodingType.UNICODE_String: + if(forceCast) + cast = "(const wchar_t *)"; break; case CLogEncodingType.Pointer: cast = "(unsigned long long int)"; @@ -204,19 +219,24 @@ public void TraceLineDiscovered(string sourceFile, CLogOutputInfo outputInfo, CL cast = "(void*)"; break; case CLogEncodingType.Int16: - //cast = "(__int16)"; + if(forceCast) + cast = "(short)"; break; case CLogEncodingType.UInt16: - //cast = "(unsigned __int16)"; + if(forceCast) + cast = "(unsigned short)"; break; case CLogEncodingType.Int8: - //cast = "(int)"; + if(forceCast) + cast = "(int)"; break; case CLogEncodingType.UInt8: - //cast = "(int)"; + if(forceCast) + cast = "(int)"; break; case CLogEncodingType.ByteArray: - //cast = "(void*)"; + if(forceCast) + cast = "(void*)"; continue; } inline.Append($", {cast}(" + arg.MacroVariableName + ")"); diff --git a/src/clog/clog.cs b/src/clog/clog.cs index ecea102..b40963b 100644 --- a/src/clog/clog.cs +++ b/src/clog/clog.cs @@ -175,6 +175,9 @@ private static int Main(string[] args) CLogSTDOUT stdout = new CLogSTDOUT(); fullyDecodedMacroEmitter.AddClogModule(stdout); + CLogDiagTrace diagTrace = new CLogDiagTrace(); + fullyDecodedMacroEmitter.AddClogModule(diagTrace); + CLogManifestedETWOutputModule manifestedEtwOutput = new CLogManifestedETWOutputModule(options.ReadOnly); fullyDecodedMacroEmitter.AddClogModule(manifestedEtwOutput); From d3f8feb9eecc199137f18a00399180017e263437 Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Sun, 27 Feb 2022 15:30:10 -0800 Subject: [PATCH 4/8] add diagtrace .c and .h files --- src/diagtrace/consumer/diagtrace.c | 121 +++++++++++++++++++++++++++++ src/diagtrace/consumer/diagtrace.h | 57 ++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 src/diagtrace/consumer/diagtrace.c create mode 100644 src/diagtrace/consumer/diagtrace.h diff --git a/src/diagtrace/consumer/diagtrace.c b/src/diagtrace/consumer/diagtrace.c new file mode 100644 index 0000000..a47af17 --- /dev/null +++ b/src/diagtrace/consumer/diagtrace.c @@ -0,0 +1,121 @@ +#include +#include +#include +#include +#include +#include +#include "diagtrace.h" + +char *diagtrace_register_page = NULL; +int diagtrace_data_fd = -1; + +struct user_reg { + __u32 size; + __u64 name_args; + __u32 status_index; + __u32 write_index; +}; + +#define DIAG_IOC_MAGIC '*' +#define DIAG_IOCSREG _IOWR(DIAG_IOC_MAGIC, 0, struct user_reg*) +#define DIAG_IOCSDEL _IOW(DIAG_IOC_MAGIC, 1, char*) + +int diagtrace_init_no_fallback( + void) +{ + int page_size = sysconf(_SC_PAGESIZE); + + int fd = open("/sys/kernel/debug/tracing/user_events_status", O_RDWR); + + if (fd == -1) + fd = open("/sys/kernel/tracing/user_events_status", O_RDWR); + + if (fd == -1) + { + printf("udiag: Failed opening status FD, %d\n", errno); + return -errno; + } + + diagtrace_data_fd = open("/sys/kernel/debug/tracing/user_events_data", O_RDWR); + + if (diagtrace_data_fd == -1) + diagtrace_data_fd = open("/sys/kernel/debug/tracing/user_events_status", O_RDWR); + + if (diagtrace_data_fd == -1) + { + printf("udiag: Failed opening data FD, %d\n", errno); + return -errno; + } + + char *page = (char*)mmap( + NULL, + page_size, + PROT_READ, + MAP_SHARED, + fd, + 0); + + close(fd); + + if (page == MAP_FAILED) + { + printf("mmap() failed, %d\n", errno); + return -errno; + } + + diagtrace_register_page = page; + + return 0; +} + +int diagtrace_init( + void) +{ + int ret; + + ret = diagtrace_init_no_fallback(); + + if (ret < 0) + { + // fallback to empty page. + int page_size = sysconf(_SC_PAGESIZE); + + diagtrace_register_page = (char*)calloc(page_size, 1); + + if (diagtrace_register_page) + { + ret = 0; + } + else + { + ret = -ENOMEM; + } + } + + return ret; +} + +int diagtrace_register_provider( + const char *provider, + int *handle) +{ + int status; + struct user_reg reg = {0}; + *handle = -1; + + reg.size = sizeof(reg); + reg.name_args = (__u64)provider; + + status = ioctl(diagtrace_data_fd, DIAG_IOCSREG, ®); + + if (status < 0) + { + // return nop provider (callers will never get enabled): + // Caused by too many providers, etc. + return 0; + } + + *handle = reg.write_index; + + return reg.status_index; +} diff --git a/src/diagtrace/consumer/diagtrace.h b/src/diagtrace/consumer/diagtrace.h new file mode 100644 index 0000000..153251b --- /dev/null +++ b/src/diagtrace/consumer/diagtrace.h @@ -0,0 +1,57 @@ +#ifndef DIAGTRACE_H +#define DIAGTRACE_H +#include +#include +#include +#include +#include + +extern char *diagtrace_register_page; +extern int diagtrace_data_fd; + +#ifndef UNREFERENCED_PARAMETER +#define UNREFERENCED_PARAMETER(x) (void)(x) +#endif + +#define DIAG_INIT() diagtrace_init() +#define DIAG_PROVIDER_REG(name, handle) diagtrace_register_provider(name, handle) +#define DIAG_PROVIDER_ON(p) diagtrace_register_page[p] + +//NOTE : do not update this maxsize without also editing CLogDiagTrace.cs +#define CLOG_EVENT_ID_MAXSIZE 48 + +struct CLOG_UDIAG_EVENT +{ + char clog_ID[CLOG_EVENT_ID_MAXSIZE]; +}; + +#define DIAG_WRITE(provider, data, len) \ +do { \ +struct iovec io[2]; \ +io[0].iov_base = &provider; \ +io[0].iov_len = sizeof(provider); \ +io[1].iov_base = data; \ +io[1].iov_len = len; \ +int __result = writev(diagtrace_data_fd, (const struct iovec*)io, 2); \ +UNREFERENCED_PARAMETER(__result); \ +} while (0); + +#if defined(__cplusplus) +extern "C" { +#endif + +int diagtrace_init_no_fallback( + void); + +int diagtrace_init( + void); + +int diagtrace_register_provider( + const char *provider, + int *handle); + +#if defined(__cplusplus) +} +#endif + +#endif /* DIAGTRACE_H */ From 3d7861ad5a738f46061b3651752c8c5e95eb75de Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Mon, 28 Feb 2022 01:15:52 +0000 Subject: [PATCH 5/8] also allow dotnet5 to compile --- src/clog/clog.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/clog/clog.csproj b/src/clog/clog.csproj index a1fcdcd..02b2908 100644 --- a/src/clog/clog.csproj +++ b/src/clog/clog.csproj @@ -3,6 +3,7 @@ Exe net6.0 + net5.0 From 8ca7c9b8d86884e01ac8cfd0b1520a8d8e7825a0 Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Mon, 28 Feb 2022 01:33:08 +0000 Subject: [PATCH 6/8] flip back to dotnet 5 for now --- src/clog/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clog/CMakeLists.txt b/src/clog/CMakeLists.txt index 92f2dac..3f2a795 100644 --- a/src/clog/CMakeLists.txt +++ b/src/clog/CMakeLists.txt @@ -9,4 +9,4 @@ set(SOURCES clog.cs CommandLineArguments.cs) -DOT_NET_BUILD(CLOG_EXE clog.exe ${CMAKE_CURRENT_SOURCE_DIR}/clog.csproj net6.0 ${SOURCES}) +DOT_NET_BUILD(CLOG_EXE clog.exe ${CMAKE_CURRENT_SOURCE_DIR}/clog.csproj net5.0 ${SOURCES}) From 0ef7c3b9dbfd2c154439a03d1f3042b208fec344 Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Mon, 28 Feb 2022 05:31:57 +0000 Subject: [PATCH 7/8] flip to dotnet 6 --- CMakeLists.txt | 1 + src/clog/CMakeLists.txt | 2 +- src/clog/clog.csproj | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6128906..64c0b06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ function(DOT_NET_BUILD) message(STATUS "DOTNET_RUNTIME : ${DOTNET_RUNTIME}") message(STATUS "OUTPUTDIR : ${CLOG_ARTIFACT_DIR}") message(STATUS "DOTNET_FRAMEWORK : ${DOTNET_FRAMEWORK}") + message(STATUS "BUILD WITH : dotnet publish ${PROJECT_PATH} -o ${CLOG_ARTIFACT_DIR} -f ${DOTNET_FRAMEWORK}") foreach(arg IN LISTS ARGV) #message(STATUS "FILE : ${CMAKE_CURRENT_BINARY_DIR}/${arg}") diff --git a/src/clog/CMakeLists.txt b/src/clog/CMakeLists.txt index 3f2a795..92f2dac 100644 --- a/src/clog/CMakeLists.txt +++ b/src/clog/CMakeLists.txt @@ -9,4 +9,4 @@ set(SOURCES clog.cs CommandLineArguments.cs) -DOT_NET_BUILD(CLOG_EXE clog.exe ${CMAKE_CURRENT_SOURCE_DIR}/clog.csproj net5.0 ${SOURCES}) +DOT_NET_BUILD(CLOG_EXE clog.exe ${CMAKE_CURRENT_SOURCE_DIR}/clog.csproj net6.0 ${SOURCES}) diff --git a/src/clog/clog.csproj b/src/clog/clog.csproj index 02b2908..a1fcdcd 100644 --- a/src/clog/clog.csproj +++ b/src/clog/clog.csproj @@ -3,7 +3,6 @@ Exe net6.0 - net5.0 From 868fe81f02ee32c79ecd04f6082afa990ac6dc0a Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Sat, 5 Mar 2022 01:33:29 +0000 Subject: [PATCH 8/8] unsure what the unused pointer is - adding BUGBUG comment, pick it up in CR --- src/clog/TraceEmitterModules/CLogDiagTrace.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/clog/TraceEmitterModules/CLogDiagTrace.cs b/src/clog/TraceEmitterModules/CLogDiagTrace.cs index a86db34..11a342b 100644 --- a/src/clog/TraceEmitterModules/CLogDiagTrace.cs +++ b/src/clog/TraceEmitterModules/CLogDiagTrace.cs @@ -335,9 +335,15 @@ public void TraceLineDiscovered(string sourceFile, CLogOutputInfo outputInfo, CL } + // + // BUGBUG : I'm not sure why we're adding 8 bytes to the end? + // if (!hasAnyDefs) { structDef.AppendLine(" DIAG_TRACE_POINTER unused;"); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"unused", currentOffset.ToString()); + decodedTraceLine.AddConfigFileProperty(ModuleName, $"unused_Len", "4"); + currentOffset += 8; }