From ea3ecb2289d0919cfa9522f38e1d1f77463dc8ad Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Mon, 18 Oct 2021 20:54:34 -0700 Subject: [PATCH 01/19] decomp string and cleaning --- defaults/CLog.cmake | 2 +- examples/README.md | 10 - src/clog/CommandLineArguments.cs | 14 +- .../CLogLTTNGOutputModule.cs | 14 +- src/clog/TraceEmitterModules/CLogSTDOUT.cs | 5 +- src/clog/TraceEmitterModules/CLogSysLog.cs | 2 +- src/clog/clog.cs | 65 +--- src/clog2text/clog2text_lttng/Program.cs | 136 +++++-- src/clog2text/clog2text_windows/Program.cs | 336 ++++++++++-------- src/clogutils/CLogConsoleTrace.cs | 23 +- .../CLogCustomTraceEmittorFactory.cs | 4 +- src/clogutils/CLogDecodedTraceLine.cs | 12 +- src/clogutils/CLogEncodingCLogTypeSearch.cs | 10 +- src/clogutils/CLogEncodingType.cs | 85 ++++- src/clogutils/CLogErrors.cs | 2 +- src/clogutils/CLogFileProcessor.cs | 118 +++++- src/clogutils/CLogFullyDecodedMacroEmitter.cs | 19 +- src/clogutils/CLogHandledException.cs | 12 +- src/clogutils/CLogSidecar.cs | 41 ++- .../ConfigFile/CLogModuleUsageInformation.cs | 2 +- .../CLogTraceMacroDefination.cs | 8 + .../syslog2clog/CommandLineArguments.cs | 2 +- src/converters/syslog2clog/syslog2clog.cs | 14 +- 23 files changed, 586 insertions(+), 350 deletions(-) diff --git a/defaults/CLog.cmake b/defaults/CLog.cmake index c997399..d47a48f 100644 --- a/defaults/CLog.cmake +++ b/defaults/CLog.cmake @@ -23,7 +23,7 @@ function(CLOG_GENERATE_TARGET) foreach(arg IN LISTS ARGV) get_filename_component(RAW_FILENAME ${arg} NAME) set(ARG_CLOG_OUTPUT_DIR ${CMAKE_CLOG_OUTPUT_DIRECTORY}/${library}) - set(ARG_CLOG_FILE ${CMAKE_CLOG_OUTPUT_DIRECTORY}/${library}/${RAW_FILENAME}.clog.h) + set(ARG_CLOG_FILE ${CMAKE_CLOG_OUTPUT_DIRECTORY}/${library}/${RAW_FILENAME}.clog.h) set(ARG_CLOG_C_FILE ${CMAKE_CLOG_OUTPUT_DIRECTORY}/${library}/${library}_${RAW_FILENAME}.clog.h.c) # message(STATUS ">>>>>>> CLOG Source File = ${RAW_FILENAME}") diff --git a/examples/README.md b/examples/README.md index 1422d55..59278a2 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,13 +8,3 @@ Then run ``` dotnet tool install --global --add-source nupkg Microsoft.Logging.CLOG ``` - -# Build (using pwsh) -``` -mkdir build -clog --installDirectory build/clog -cd build -$env:CLOG_DEVELOPMENT_MODE = 1 -cmake .. -cmake --build . -``` diff --git a/src/clog/CommandLineArguments.cs b/src/clog/CommandLineArguments.cs index 01239cd..e9f910b 100644 --- a/src/clog/CommandLineArguments.cs +++ b/src/clog/CommandLineArguments.cs @@ -1,4 +1,4 @@ -/*++ +/*++ Copyright (c) Microsoft Corporation. Licensed under the MIT License. @@ -19,12 +19,6 @@ namespace clog { public class CommandLineArguments { - [Option("installDependencies", SetName = "install", Required = false, HelpText = "Install dependencies such as clog.h can CLog.cmake to the folder specified")] - public string InstallDependencies - { - get; - set; - } [Option("inputFiles", SetName = "build", Required = false, HelpText = "Full path to one (or more) source file for CLOG to generate logging stubs")] public IEnumerable InputFiles @@ -116,7 +110,6 @@ public bool DynamicTracepointProvider get; set; } - public string GetOutputFileName(string inputFile) { string ret = Path.Combine(this.OutputDirectory, Path.GetFileName(inputFile)); @@ -126,11 +119,6 @@ public string GetOutputFileName(string inputFile) public bool IsValid() { - if (!string.IsNullOrWhiteSpace(this.InstallDependencies)) - { - return true; - } - if (!string.IsNullOrEmpty(this.OutputDirectory)) { if (0 == this.InputFiles.Count()) diff --git a/src/clog/TraceEmitterModules/CLogLTTNGOutputModule.cs b/src/clog/TraceEmitterModules/CLogLTTNGOutputModule.cs index 1407295..9f51b19 100644 --- a/src/clog/TraceEmitterModules/CLogLTTNGOutputModule.cs +++ b/src/clog/TraceEmitterModules/CLogLTTNGOutputModule.cs @@ -1,4 +1,4 @@ -/*++ +/*++ Copyright (c) Microsoft Corporation. Licensed under the MIT License. @@ -56,6 +56,8 @@ public bool ManditoryModule public void InitHeader(StringBuilder header) { + string justFile = Path.GetFileName(_lttngHeaderFileName); + header.AppendLine("#undef TRACEPOINT_PROVIDER"); header.AppendLine($"#define TRACEPOINT_PROVIDER {_lttngProviderName}"); @@ -66,7 +68,7 @@ public void InitHeader(StringBuilder header) } header.AppendLine("#undef TRACEPOINT_INCLUDE"); - header.AppendLine($"#define TRACEPOINT_INCLUDE \"{_lttngHeaderFileName}\""); + header.AppendLine($"#define TRACEPOINT_INCLUDE \"{justFile}\""); header.AppendLine($"#if !defined(DEF_{_lttngProviderName}) || defined(TRACEPOINT_HEADER_MULTI_READ)"); header.AppendLine($"#define DEF_{_lttngProviderName}"); @@ -76,7 +78,7 @@ public void InitHeader(StringBuilder header) header.AppendLine("#define __int64 __int64_t"); - header.AppendLine($"#include \"{_lttngHeaderFileName}\""); + header.AppendLine($"#include \"{justFile}\""); header.AppendLine("#endif"); @@ -93,7 +95,6 @@ public void FinishedProcessing(CLogOutputInfo outputInfo, StringBuilder header, } File.WriteAllText(_lttngHeaderFileName, lttngFile.ToString()); - if (_lttngDynamicTracepoint) { sourceFile.AppendLine("#ifdef BUILDING_TRACEPOINT_PROVIDER"); @@ -107,9 +108,8 @@ public void FinishedProcessing(CLogOutputInfo outputInfo, StringBuilder header, sourceFile.AppendLine("#define TRACEPOINT_CREATE_PROBES"); sourceFile.AppendLine("#define TRACEPOINT_DEFINE"); } - - sourceFile.AppendLine($"#include \"{Path.GetFullPath(_clogFile)}\""); - } + sourceFile.AppendLine($"#include \"{Path.GetFileName(_clogFile)}\""); + } public void TraceLineDiscovered(string sourceFile, CLogOutputInfo outputInfo, CLogDecodedTraceLine decodedTraceLine, CLogSidecar sidecar, StringBuilder macroPrefix, StringBuilder inline, StringBuilder function) diff --git a/src/clog/TraceEmitterModules/CLogSTDOUT.cs b/src/clog/TraceEmitterModules/CLogSTDOUT.cs index 2b3b35b..d7b0f82 100644 --- a/src/clog/TraceEmitterModules/CLogSTDOUT.cs +++ b/src/clog/TraceEmitterModules/CLogSTDOUT.cs @@ -1,4 +1,4 @@ -/*++ +/*++ Copyright (c) Microsoft Corporation. Licensed under the MIT License. @@ -58,8 +58,7 @@ public void FinishedProcessing(CLogOutputInfo outputInfo, StringBuilder header, bool emittedHeader = false; public void TraceLineDiscovered(string sourceFile, CLogOutputInfo outputInfo, CLogDecodedTraceLine decodedTraceLine, CLogSidecar sidecar, StringBuilder macroPrefix, StringBuilder inline, StringBuilder function) { - string clean; - + CLogFileProcessor.DecomposedString clean; CLogFileProcessor.CLogTypeContainer[] types = CLogFileProcessor.BuildTypes(decodedTraceLine.configFile, null, decodedTraceLine.TraceString, null, out clean); CLogExportModuleDefination moduleSettings = decodedTraceLine.GetMacroConfigurationProfile().FindExportModule(ModuleName); diff --git a/src/clog/TraceEmitterModules/CLogSysLog.cs b/src/clog/TraceEmitterModules/CLogSysLog.cs index b5f7cca..2f82490 100644 --- a/src/clog/TraceEmitterModules/CLogSysLog.cs +++ b/src/clog/TraceEmitterModules/CLogSysLog.cs @@ -1,4 +1,4 @@ -/*++ +/*++ Copyright (c) Microsoft Corporation. Licensed under the MIT License. diff --git a/src/clog/clog.cs b/src/clog/clog.cs index aa27176..d396538 100644 --- a/src/clog/clog.cs +++ b/src/clog/clog.cs @@ -1,4 +1,4 @@ -/*++ +/*++ Copyright (c) Microsoft Corporation. Licensed under the MIT License. @@ -25,45 +25,6 @@ namespace clog { internal class clog { - private static int PerformInstall(string outputDir) - { - if (File.Exists(outputDir)) - { - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, "Output file for install cannot be a file. It either must not exist, or be a directory"); - return -11; - } - Directory.CreateDirectory(outputDir); - - Assembly utilsAssembly = typeof(CLogConsoleTrace).Assembly; - string baseName = utilsAssembly.GetName().Name; - - void ExtractFile(string name) - { - using Stream embeddedStream = utilsAssembly.GetManifestResourceStream($"{baseName}.{name}"); - using StreamReader reader = new StreamReader(embeddedStream); - string contents = reader.ReadToEnd(); - string fileName = Path.Combine(outputDir, name); - if (File.Exists(fileName)) - { - string existingContents = File.ReadAllText(fileName); - if (existingContents == contents) - { - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Std, $"Skipping file {name} as its up to date"); - return; - } - } - File.WriteAllText(fileName, contents); - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Std, $"Installed file {name}"); - } - - ExtractFile("clog.h"); - ExtractFile("CLog.cmake"); - - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Std, "--installDirectory overrides all arguments. Dependencies successfully installed!"); - - return 0; - } - private static int Main(string[] args) { ParserResult o = Parser.Default.ParseArguments(args); @@ -71,6 +32,7 @@ private static int Main(string[] args) return o.MapResult( options => { + string currentFile = null; try { // @@ -83,11 +45,6 @@ private static int Main(string[] args) return -1; } - if (!string.IsNullOrWhiteSpace(options.InstallDependencies)) - { - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, "Internal error, this should have been handled at a previous step"); - return -1; - } CLogConfigurationFile configFile = CLogConfigurationFile.FromFile(options.ConfigurationFile); configFile.ProfileName = options.ConfigurationProfile; @@ -176,12 +133,12 @@ private static int Main(string[] args) // List batchingModules = new List(); - Console.WriteLine("Number of files : " + (new List(options.InputFiles)).Count); foreach (string inputFile in options.InputFiles) { Console.WriteLine("Processing: " + inputFile); + currentFile = inputFile; string outputFile = options.GetOutputFileName(inputFile); string outputCFile = Path.Combine(Path.GetDirectoryName(outputFile), options.ScopePrefix + "_" + Path.GetFileName(outputFile)) + ".c"; @@ -220,7 +177,7 @@ private static int Main(string[] args) CLogSTDOUT stdout = new CLogSTDOUT(); fullyDecodedMacroEmitter.AddClogModule(stdout); - + CLogManifestedETWOutputModule manifestedEtwOutput = new CLogManifestedETWOutputModule(options.ReadOnly); fullyDecodedMacroEmitter.AddClogModule(manifestedEtwOutput); @@ -241,10 +198,21 @@ private static int Main(string[] args) string content = File.ReadAllText(inputFile); string output = processor.ConvertFile(configFile, outputInfo, fullyDecodedMacroEmitter, content, inputFile, false); + // TODO: BUGBUG - Java won't ever do this + //if (!content.Contains(Path.GetFileName(outputFile))) + //{ + // CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Std, $"You must #include the clog output file {Path.GetFileName(outputFile)}"); + // throw new CLogEnterReadOnlyModeException("MustIncludeCLogHeader", CLogHandledException.ExceptionType.SourceMustIncludeCLOGHeader, null); + //} + fullyDecodedMacroEmitter.FinishedProcessing(outputInfo); StringBuilder clogFile = new StringBuilder(); + + clogFile.AppendLine($"#ifndef CLOG_DO_NOT_INCLUDE_HEADER"); clogFile.AppendLine("#include "); + clogFile.AppendLine($"#endif"); + clogFile.Append(fullyDecodedMacroEmitter.HeaderInit); @@ -314,6 +282,7 @@ private static int Main(string[] args) File.WriteAllText(outputFile, clogFile.ToString()); File.WriteAllText(outputCFile, fullyDecodedMacroEmitter.SourceFile); } + currentFile = null; // // Enumerate batching modules, allowing them to save @@ -328,6 +297,8 @@ private static int Main(string[] args) } catch (CLogHandledException e) { + if(null != currentFile) + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $"Failure in file : {currentFile}"); e.PrintDiagnostics(); return -2; } diff --git a/src/clog2text/clog2text_lttng/Program.cs b/src/clog2text/clog2text_lttng/Program.cs index 61f32dd..50be36b 100644 --- a/src/clog2text/clog2text_lttng/Program.cs +++ b/src/clog2text/clog2text_lttng/Program.cs @@ -90,59 +90,67 @@ private static int Main(string[] args) return o.MapResult( options => { - string sidecarJson = File.ReadAllText(options.SideCarFile); - CLogSidecar textManifest = CLogSidecar.FromJson(sidecarJson); + try + { + string sidecarJson = File.ReadAllText(options.SideCarFile); + CLogSidecar textManifest = CLogSidecar.FromJson(sidecarJson); - TextReader file = Console.In; + TextReader file = Console.In; - if (!string.IsNullOrEmpty(options.BabelTrace)) - { - file = new StreamReader(options.BabelTrace); - } + if (!string.IsNullOrEmpty(options.BabelTrace)) + { + file = new StreamReader(options.BabelTrace); + } - string line; - LTTNGEventDecoder lttngDecoder = new LTTNGEventDecoder(textManifest); - int lines = 0; - StreamWriter outputfile = null; - if (!String.IsNullOrEmpty(options.OutputFile)) - outputfile = new StreamWriter(new FileStream(options.OutputFile, FileMode.Create)); + string line; + LTTNGEventDecoder lttngDecoder = new LTTNGEventDecoder(textManifest); + int lines = 0; + StreamWriter outputfile = null; + if (!String.IsNullOrEmpty(options.OutputFile)) + outputfile = new StreamWriter(new FileStream(options.OutputFile, FileMode.Create)); - DateTimeOffset startTime = DateTimeOffset.Now; + DateTimeOffset startTime = DateTimeOffset.Now; - try - { - while (!string.IsNullOrEmpty(line = file.ReadLine())) + try { - ++lines; - if (0 == lines % 10000) + while (!string.IsNullOrEmpty(line = file.ReadLine())) { - Console.WriteLine($"Line : {lines}"); + ++lines; + if (0 == lines % 10000) + { + Console.WriteLine($"Line : {lines}"); + } + Dictionary valueBag; + EventInformation ei; + CLogDecodedTraceLine bundle = lttngDecoder.DecodedTraceLine(line, out ei, out valueBag); + DecodeAndTraceToConsole(outputfile, bundle, line, textManifest.ConfigFile, valueBag, ei, options.ShowTimestamps, options.ShowCPUInfo); } - Dictionary valueBag; - EventInformation ei; - CLogDecodedTraceLine bundle = lttngDecoder.DecodedTraceLine(line, out ei, out valueBag); - DecodeAndTraceToConsole(outputfile, bundle, line, textManifest.ConfigFile, valueBag, ei, options.ShowTimestamps, options.ShowCPUInfo); } - } - catch (Exception e) - { - CLogConsoleTrace.TraceLine(TraceType.Err, "ERROR : " + e); - if (null != outputfile) + catch (Exception e) { - outputfile.WriteLine("ERROR : " + e); + CLogConsoleTrace.TraceLine(TraceType.Err, "ERROR : " + e); + if (null != outputfile) + { + outputfile.WriteLine("ERROR : " + e); + } } - } - finally - { - if (null != outputfile) + finally { - outputfile.Flush(); - outputfile.Close(); + if (null != outputfile) + { + outputfile.Flush(); + outputfile.Close(); + } } + Console.WriteLine($"Decoded {lines} in {DateTimeOffset.Now - startTime}"); + return 0; + } + catch (CLogHandledException e) + { + e.PrintDiagnostics(); + return -2; } - Console.WriteLine($"Decoded {lines} in {DateTimeOffset.Now - startTime}"); - return 0; }, err => { Console.WriteLine("Bad Args : " + err); @@ -197,7 +205,8 @@ public CLogDecodedTraceLine DecodedTraceLine(string babbleTraceLine, out EventIn ei.ProcessId = Convert.ToInt64(cpuArgs["vpid"].AsString).ToString("x"); } - CLogDecodedTraceLine bundle = _sidecar.FindBundle(args["name"].AsString); + string name = args["name"].AsString.Split(':')[1]; + CLogDecodedTraceLine bundle = _sidecar.FindBundle(name); string fields = args["event.fields"].AsString.Substring(1, args["event.fields"].AsString.Length - 2).Trim(); @@ -226,6 +235,55 @@ public string AsString get; } + public Int64 AsInt64 + { + get + { + var ret = Convert.ToInt64(AsString); + return ret; + } + } + + public System.UInt64 AsUInt64 + { + get + { + var ret = Convert.ToUInt64(AsString); + return ret; + } + } + public System.Int16 AsInt16 + { + get + { + var ret = Convert.ToInt16(AsString); + return ret; + } + } + public System.UInt16 AsUInt16 + { + get + { + var ret = Convert.ToUInt16(AsString); + return ret; + } + } + public sbyte AsInt8 + { + get + { + return Convert.ToSByte(AsUInt8); + } + } + public byte AsUInt8 + { + get + { + var ret = Convert.ToByte(AsString); + return ret; + } + } + public int AsInt32 { get diff --git a/src/clog2text/clog2text_windows/Program.cs b/src/clog2text/clog2text_windows/Program.cs index 4c83ab7..243b319 100644 --- a/src/clog2text/clog2text_windows/Program.cs +++ b/src/clog2text/clog2text_windows/Program.cs @@ -32,204 +32,212 @@ private static int Main(string[] cmdLineArgs) return o.MapResult( options => { - string sidecarJson = File.ReadAllText(options.SideCarFile); - CLogSidecar sidecar = CLogSidecar.FromJson(sidecarJson); - - TextReader file = Console.In; - - if (!File.Exists(options.ETLFile)) + try { - TraceLine(TraceType.Err, $"ETL File {options.ETLFile} doesnt exist"); - return -1; - } + string sidecarJson = File.ReadAllText(options.SideCarFile); + CLogSidecar sidecar = CLogSidecar.FromJson(sidecarJson); - StreamWriter outputfile = null; - if (!String.IsNullOrEmpty(options.OutputFile)) - outputfile = new StreamWriter(new FileStream(options.OutputFile, FileMode.Create)); + TextReader file = Console.In; - try - { - TraceProcessorSettings traceSettings = new TraceProcessorSettings { AllowLostEvents = true, AllowTimeInversion = true }; + if (!File.Exists(options.ETLFile)) + { + TraceLine(TraceType.Err, $"ETL File {options.ETLFile} doesnt exist"); + return -1; + } + + StreamWriter outputfile = null; + if (!String.IsNullOrEmpty(options.OutputFile)) + outputfile = new StreamWriter(new FileStream(options.OutputFile, FileMode.Create)); - using (ITraceProcessor etwfile = TraceProcessor.Create(options.ETLFile, traceSettings)) + try { - HashSet ids = new HashSet(); + TraceProcessorSettings traceSettings = new TraceProcessorSettings { AllowLostEvents = true, AllowTimeInversion = true }; - foreach (var m in sidecar.EventBundlesV2) + using (ITraceProcessor etwfile = TraceProcessor.Create(options.ETLFile, traceSettings)) { - foreach (var prop in m.Value.ModuleProperites) + HashSet ids = new HashSet(); + + foreach (var m in sidecar.EventBundlesV2) { - if (prop.Key.Equals("MANIFESTED_ETW")) + foreach (var prop in m.Value.ModuleProperites) { - ids.Add(new Guid(prop.Value["ETW_Provider"])); - } - else if (prop.Key.Equals("TRACELOGGING")) - { - ids.Add(new Guid(prop.Value["ETW_Provider"])); + if (prop.Key.Equals("MANIFESTED_ETW")) + { + ids.Add(new Guid(prop.Value["ETW_Provider"])); + } + else if (prop.Key.Equals("TRACELOGGING")) + { + ids.Add(new Guid(prop.Value["ETW_Provider"])); + } } } - } - var events = etwfile.UseGenericEvents(ids.ToArray()); - etwfile.Process(); + var events = etwfile.UseGenericEvents(ids.ToArray()); + etwfile.Process(); - foreach (var e in events.Result.Events) - { - string line = ""; - - try + foreach (var e in events.Result.Events) { - Dictionary fixedUpArgs = new Dictionary(); - string errorString = "ERROR"; + string line = ""; - if (null == e.Fields) + try { - continue; - } + Dictionary fixedUpArgs = new Dictionary(); + string errorString = "ERROR"; - Dictionary args = new Dictionary(); + if (null == e.Fields) + { + continue; + } - foreach (var f in e.Fields) - { - args[f.Name] = new ManifestedETWEvent(f); - } + Dictionary args = new Dictionary(); - CLogDecodedTraceLine bundle = null; - int eidAsInt = -1; + foreach (var f in e.Fields) + { + args[f.Name] = new ManifestedETWEvent(f); + } - foreach (var b in sidecar.EventBundlesV2) - { - Dictionary keys; + CLogDecodedTraceLine bundle = null; + int eidAsInt = -1; - if (!e.IsTraceLogging) + foreach (var b in sidecar.EventBundlesV2) { - if (!b.Value.ModuleProperites.TryGetValue("MANIFESTED_ETW", out keys)) + Dictionary keys; + + if (!e.IsTraceLogging) { - continue; + if (!b.Value.ModuleProperites.TryGetValue("MANIFESTED_ETW", out keys)) + { + continue; + } + + string eid; + + if (!keys.TryGetValue("EventID", out eid)) + { + continue; + } + + eidAsInt = Convert.ToInt32(eid); + + if (eidAsInt == e.Id) + { + bundle = b.Value; + errorString = "ERROR:" + eidAsInt; + break; + } } - - string eid; - - if (!keys.TryGetValue("EventID", out eid)) + else { - continue; + if (e.ActivityName.Equals(b.Key)) + { + bundle = b.Value; + errorString = "ERROR:" + b.Key; + break; + } } + } - eidAsInt = Convert.ToInt32(eid); + if (null == bundle) + { + continue; + } + + Dictionary argMap; - if (eidAsInt == e.Id) + if (e.IsTraceLogging) + { + argMap = new Dictionary(); + foreach (var arg in args) { - bundle = b.Value; - errorString = "ERROR:" + eidAsInt; - break; + argMap[arg.Key] = arg.Key; } } else { - if (e.ActivityName.Equals(b.Key)) - { - bundle = b.Value; - errorString = "ERROR:" + b.Key; - break; - } + argMap = sidecar.GetTracelineMetadata(bundle, "MANIFESTED_ETW"); } - } - if (null == bundle) - { - continue; - } - - Dictionary argMap; + var types = CLogFileProcessor.BuildTypes(sidecar.ConfigFile, null, bundle.TraceString, null, out CLogFileProcessor.DecomposedString clean); - if (e.IsTraceLogging) - { - argMap = new Dictionary(); - foreach (var arg in args) + if (0 == types.Length) { - argMap[arg.Key] = arg.Key; + errorString = bundle.TraceString; + goto toPrint; } - } - else - { - argMap = sidecar.GetTracelineMetadata(bundle, "MANIFESTED_ETW"); - } - var types = CLogFileProcessor.BuildTypes(sidecar.ConfigFile, null, bundle.TraceString, null, out string clean); + int argIndex = 0; - if (0 == types.Length) - { - errorString = bundle.TraceString; - goto toPrint; - } + foreach (var type in types) + { + var arg = bundle.splitArgs[argIndex]; + CLogEncodingCLogTypeSearch node = sidecar.ConfigFile.FindType(arg); - int argIndex = 0; + switch (node.EncodingType) + { + case CLogEncodingType.Synthesized: + continue; - foreach (var type in types) - { - var arg = bundle.splitArgs[argIndex]; - CLogEncodingCLogTypeSearch node = sidecar.ConfigFile.FindType(arg); + case CLogEncodingType.Skip: + continue; + } - switch (node.EncodingType) - { - case CLogEncodingType.Synthesized: - continue; + string lookupArgName = argMap[arg.MacroVariableName]; - case CLogEncodingType.Skip: - continue; - } + if (!args.ContainsKey(lookupArgName)) + { + Console.WriteLine($"Argmap missing {lookupArgName}"); + throw new Exception("InvalidType : " + node.DefinationEncoding); + } - string lookupArgName = argMap[arg.MacroVariableName]; + if (0 != node.DefinationEncoding.CompareTo(type.TypeNode.DefinationEncoding)) + { + Console.WriteLine("Invalid Types in Traceline"); + throw new Exception("InvalidType : " + node.DefinationEncoding); + } - if (!args.ContainsKey(lookupArgName)) - { - Console.WriteLine($"Argmap missing {lookupArgName}"); - throw new Exception("InvalidType : " + node.DefinationEncoding); + fixedUpArgs[arg.MacroVariableName] = args[lookupArgName]; + ++argIndex; } - if (0 != node.DefinationEncoding.CompareTo(type.TypeNode.DefinationEncoding)) - { - Console.WriteLine("Invalid Types in Traceline"); - throw new Exception("InvalidType : " + node.DefinationEncoding); - } + toPrint: - fixedUpArgs[arg.MacroVariableName] = args[lookupArgName]; - ++argIndex; + EventInformation ei = new EventInformation(); + ei.Timestamp = e.Timestamp.DateTimeOffset; + ei.ProcessId = e.ProcessId.ToString("x"); + ei.ThreadId = e.ThreadId.ToString("x"); + DecodeAndTraceToConsole(outputfile, bundle, errorString, sidecar.ConfigFile, fixedUpArgs, ei, options.ShowTimestamps, options.ShowCPUInfo); + } + catch (Exception) + { + Console.WriteLine($"Invalid TraceLine : {line}"); } - - toPrint: - - EventInformation ei = new EventInformation(); - ei.Timestamp = e.Timestamp.DateTimeOffset; - ei.ProcessId = e.ProcessId.ToString("x"); - ei.ThreadId = e.ThreadId.ToString("x"); - DecodeAndTraceToConsole(outputfile, bundle, errorString, sidecar.ConfigFile, fixedUpArgs, ei, options.ShowTimestamps, options.ShowCPUInfo); - } - catch (Exception) - { - Console.WriteLine($"Invalid TraceLine : {line}"); } } } - } - catch (Exception e) - { - CLogConsoleTrace.TraceLine(TraceType.Err, "ERROR : " + e); - if (null != outputfile) + catch (Exception e) + { + CLogConsoleTrace.TraceLine(TraceType.Err, "ERROR : " + e); + if (null != outputfile) + { + outputfile.WriteLine("ERROR : " + e); + } + } + finally { - outputfile.WriteLine("ERROR : " + e); + if (null != outputfile) + { + outputfile.Flush(); + outputfile.Close(); + } } + return 0; } - finally + catch (CLogHandledException e) { - if (null != outputfile) - { - outputfile.Flush(); - outputfile.Close(); - } + e.PrintDiagnostics(); + return -2; } - return 0; - }, err => + }, err => { Console.WriteLine("Bad Args : " + err); return -1; @@ -244,7 +252,6 @@ public ManifestedETWEvent(IGenericEventField e) { _event = e; } - public byte[] AsBinary { get @@ -261,6 +268,52 @@ public byte[] AsBinary } } + public Int64 AsInt64 + { + get + { + return _event.AsInt64; + } + } + + public System.UInt64 AsUInt64 + { + get + { + return _event.AsUInt64; + } + } + public System.Int16 AsInt16 + { + get + { + return _event.AsInt16; + } + } + public System.UInt16 AsUInt16 + { + get + { + return _event.AsUInt16; + } + } + public sbyte AsInt8 + { + get + { + return Convert.ToSByte(AsUInt8); + } + } + public byte AsUInt8 + { + get + { + var ret = Convert.ToByte(AsString); + return ret; + } + } + + public ulong AsPointer { get @@ -276,7 +329,6 @@ public ulong AsPointer } } } - public string AsString { get { return _event.ToString(); } diff --git a/src/clogutils/CLogConsoleTrace.cs b/src/clogutils/CLogConsoleTrace.cs index e4a5678..720bc06 100644 --- a/src/clogutils/CLogConsoleTrace.cs +++ b/src/clogutils/CLogConsoleTrace.cs @@ -1,4 +1,4 @@ -/*++ +/*++ Copyright (c) Microsoft Corporation. Licensed under the MIT License. @@ -128,9 +128,8 @@ public static void DecodeAndTraceToConsole(StreamWriter outputfile, CLogDecodedT } } - string clean; - - CLogFileProcessor.CLogTypeContainer[] types = CLogFileProcessor.BuildTypes(config, null, bundle.TraceString, null, out clean); + CLogFileProcessor.DecomposedString decompString; + CLogFileProcessor.CLogTypeContainer[] types = CLogFileProcessor.BuildTypes(config, null, bundle.TraceString, null, out decompString); if (0 == types.Length) { @@ -180,7 +179,8 @@ public static void DecodeAndTraceToConsole(StreamWriter outputfile, CLogDecodedT { if (string.IsNullOrEmpty(type.TypeNode.CustomDecoder)) { - toPrint.Append($"{type.LeadingString}{value.AsString}"); + string typeForPrint = clogutils.ClogEventPrinter.AsCorrectType(type.TypeNode.EncodingType, value); + toPrint.Append($"{type.LeadingString}{typeForPrint}"); } else { @@ -221,18 +221,5 @@ public class EventInformation public string ThreadId { get; set; } public string ProcessId { get; set; } } - - public interface IClogEventArg - { - string AsString { get; } - - int AsInt32 { get; } - - uint AsUInt32 { get; } - - ulong AsPointer { get; } - - byte[] AsBinary { get; } - } } } diff --git a/src/clogutils/CLogCustomTraceEmittorFactory.cs b/src/clogutils/CLogCustomTraceEmittorFactory.cs index 0b0382b..4049548 100644 --- a/src/clogutils/CLogCustomTraceEmittorFactory.cs +++ b/src/clogutils/CLogCustomTraceEmittorFactory.cs @@ -117,7 +117,7 @@ public bool Decode(CLogEncodingCLogTypeSearch type, IClogEventArg value, CLogLin break; case CLogEncodingType.ByteArray: - case CLogEncodingType.UInt64Array: + /*case CLogEncodingType.UInt64Array: case CLogEncodingType.Int32Array: case CLogEncodingType.UInt32Array: case CLogEncodingType.Int64Array: @@ -127,7 +127,7 @@ public bool Decode(CLogEncodingCLogTypeSearch type, IClogEventArg value, CLogLin case CLogEncodingType.GUIDArray: case CLogEncodingType.Int16Array: case CLogEncodingType.UInt16Array: - case CLogEncodingType.Int8Array: + case CLogEncodingType.Int8Array:*/ args[0] = value.AsBinary; break; diff --git a/src/clogutils/CLogDecodedTraceLine.cs b/src/clogutils/CLogDecodedTraceLine.cs index e9242e1..6468020 100644 --- a/src/clogutils/CLogDecodedTraceLine.cs +++ b/src/clogutils/CLogDecodedTraceLine.cs @@ -1,4 +1,4 @@ -/*++ +/*++ Copyright (c) Microsoft Corporation. Licensed under the MIT License. @@ -25,7 +25,7 @@ public class CLogDecodedTraceLine [JsonProperty] public Dictionary> ModuleProperites = new Dictionary>(); public CLogDecodedTraceLine(string uniqueId, string sourceFile, string userString, string userStringNoPrefix, CLogLineMatch m, CLogConfigurationFile c, - CLogTraceMacroDefination mac, CLogFileProcessor.CLogVariableBundle[] args, string cleanedString) + CLogTraceMacroDefination mac, CLogFileProcessor.CLogVariableBundle[] args, CLogFileProcessor.DecomposedString decompString) { SourceFile = sourceFile; macro = mac; @@ -35,16 +35,20 @@ public CLogDecodedTraceLine(string uniqueId, string sourceFile, string userStrin TraceString = userString; splitArgs = args; TraceStringNoPrefix = userStringNoPrefix; - CleanedString = cleanedString; + + DecomposedString = decompString; } [JsonProperty] public string TraceString { get; private set; } - public string CleanedString { get; private set; } + public CLogFileProcessor.DecomposedString DecomposedString; + + public string CleanedString { get { return DecomposedString.AsManifestedETWEncoding; } } public string TraceStringNoPrefix { get; private set; } + [JsonProperty] public string UniqueId { get; private set; } diff --git a/src/clogutils/CLogEncodingCLogTypeSearch.cs b/src/clogutils/CLogEncodingCLogTypeSearch.cs index a112a3b..638be9f 100644 --- a/src/clogutils/CLogEncodingCLogTypeSearch.cs +++ b/src/clogutils/CLogEncodingCLogTypeSearch.cs @@ -1,4 +1,4 @@ -/*++ +/*++ Copyright (c) Microsoft Corporation. Licensed under the MIT License. @@ -25,6 +25,9 @@ public class CLogEncodingCLogTypeSearch [JsonProperty] public string CType { get; set; } + [JsonProperty] + public string JavaType { get; set; } + [JsonProperty] public string DefinationEncoding { get; set; } @@ -37,9 +40,8 @@ public class CLogEncodingCLogTypeSearch public bool IsEncodableArg { - get - { - return !Synthesized && + get { + return !Synthesized && (EncodingType != CLogEncodingType.UniqueAndDurableIdentifier && EncodingType != CLogEncodingType.UserEncodingString); } diff --git a/src/clogutils/CLogEncodingType.cs b/src/clogutils/CLogEncodingType.cs index d2f8d66..0117b16 100644 --- a/src/clogutils/CLogEncodingType.cs +++ b/src/clogutils/CLogEncodingType.cs @@ -37,6 +37,88 @@ public enum CLogEncodingType : uint ByteArray = 15, UserEncodingString = 16, UniqueAndDurableIdentifier = 17, + /* Int32Array = 18, + UInt32Array = 19, + Int64Array = 20, + UInt64Array = 21, + ANSI_StringArray = 22, + UNICODE_StringArray = 23, + PointerArray = 24, + GUIDArray = 25, + Int16Array = 26, + UInt16Array = 27, + Int8Array = 28,*/ + // UInt8Array, // Not used, use ByteArray + //Struct = 29 + } + + public class ClogEventPrinter + { + public static string AsCorrectType(CLogEncodingType type, IClogEventArg arg) + { + switch (type) + { + case CLogEncodingType.Int32: + return arg.AsInt32.ToString(); + case CLogEncodingType.UInt32: + return arg.AsUInt32.ToString(); + case CLogEncodingType.Int64: + return arg.AsInt64.ToString(); + case CLogEncodingType.UInt64: + return arg.AsUInt64.ToString(); + case CLogEncodingType.ANSI_String: + return arg.AsString.ToString(); + /* case CLogEncodingType.UNICODE_String: + return arg.AsInt32.ToString();*/ + case CLogEncodingType.Pointer: + return arg.AsUInt64.ToString(); + /* case CLogEncodingType.GUID: + return arg.ToString();*/ + case CLogEncodingType.Int16: + return arg.AsInt16.ToString(); + case CLogEncodingType.UInt16: + return arg.AsUInt16.ToString(); + case CLogEncodingType.Int8: + return arg.AsInt8.ToString(); + case CLogEncodingType.UInt8: + return arg.AsUInt8.ToString(); + /*case CLogEncodingType.ByteArray: + return arg..ToString(); + case CLogEncodingType.UserEncodingString: + return arg.AsInt32.ToString(); + case CLogEncodingType.UniqueAndDurableIdentifier: + return arg.AsInt32.ToString();*/ + case CLogEncodingType.Unknown: + case CLogEncodingType.Synthesized: + case CLogEncodingType.Skip: + default: + throw new CLogEnterReadOnlyModeException("Invalid Type - likely a missing CLOG feature, consider using a different type or submiting a patch to CLOG", CLogHandledException.ExceptionType.EncoderIncompatibleWithType, null); + } + } + } + + public interface IClogEventArg + { + string AsString { get; } + int AsInt32 { get; } + uint AsUInt32 { get; } + System.Int64 AsInt64 { get; } + System.UInt64 AsUInt64 { get; } + System.Int16 AsInt16 { get; } + System.UInt16 AsUInt16 { get; } + sbyte AsInt8 { get; } + byte AsUInt8 { get; } + ulong AsPointer { get; } + byte[] AsBinary { get; } + + /* + ANSI_String = 7, + UNICODE_String = 8, + + GUID = 10, + ByteArray = 15, + UserEncodingString = 16, + UniqueAndDurableIdentifier = 17, Int32Array = 18, UInt32Array = 19, Int64Array = 20, @@ -48,7 +130,6 @@ public enum CLogEncodingType : uint Int16Array = 26, UInt16Array = 27, Int8Array = 28, - // UInt8Array, // Not used, use ByteArray - Struct = 29 + */ } } diff --git a/src/clogutils/CLogErrors.cs b/src/clogutils/CLogErrors.cs index 8db5ba9..c42b930 100644 --- a/src/clogutils/CLogErrors.cs +++ b/src/clogutils/CLogErrors.cs @@ -1,4 +1,4 @@ -/*++ +/*++ Copyright (c) Microsoft Corporation. Licensed under the MIT License. diff --git a/src/clogutils/CLogFileProcessor.cs b/src/clogutils/CLogFileProcessor.cs index 79d89e5..2cc39d4 100644 --- a/src/clogutils/CLogFileProcessor.cs +++ b/src/clogutils/CLogFileProcessor.cs @@ -1,4 +1,4 @@ -/*++ +/*++ s Copyright (c) Microsoft Corporation. Licensed under the MIT License. @@ -146,7 +146,7 @@ private static string[] SplitWithEscapedQuotes(string info, char splitChar) { if (0 == numParan) { - throw new Exception("Invalid Input"); + throw new CLogEnterReadOnlyModeException($"Unable to split string: {info}", CLogHandledException.ExceptionType.InvalidInput, null); } --numParan; @@ -171,16 +171,90 @@ private static string[] SplitWithEscapedQuotes(string info, char splitChar) return ret.ToArray(); } + + public class DecomposedString + { + public void AddEncoding(EncodingArg arg) + { + encodings.Add(arg); + } + + public EncodingArg CreateNewArg() + { + EncodingArg arg = new EncodingArg(); + encodings.Add(arg); + return arg; + } + public class EncodingArg + { + public string Prefix { get; set; } = ""; + public CLogEncodingCLogTypeSearch Type { get; set; } + } + + public string AsPrintF + { + get + { + string ret = ""; + int idx = 0; + foreach (var e in encodings) + { + if (null != e.Type) + { + switch(e.Type.EncodingType) + { + case CLogEncodingType.ByteArray: + ret += "p"; + break; + default: + ret += e.Type.DefinationEncoding; + break; + } + + ++idx; + } + ret += e.Prefix; + } + + return ret; + } + } + public string AsManifestedETWEncoding { + get { + string ret = ""; + int idx = 0; + foreach(var e in encodings) + { + if(null != e.Type) + { + //ret += e.Type.DefinationEncoding; + ret += idx; + ++idx; + } + ret += e.Prefix; + } + + return ret; + } + } + + private List encodings = new List(); + } + public static CLogTypeContainer[] BuildTypes(CLogConfigurationFile configFile, CLogLineMatch traceLineMatch, string argString, string traceLine, - out string cleanedString) + out DecomposedString decompString) { List ret = new List(); string pieces = string.Empty; int argCount = 0; - cleanedString = string.Empty; + + decompString = new DecomposedString(); + string prefixString = ""; + DecomposedString.EncodingArg currentArg = decompString.CreateNewArg(); + if (string.IsNullOrEmpty(argString)) { return new CLogTypeContainer[0]; @@ -192,6 +266,7 @@ public static CLogTypeContainer[] BuildTypes(CLogConfigurationFile configFile, C for (int i = 0; i < argString.Length; ++i) { pieces += argString[i]; + currentArg.Prefix += argString[i]; if ('%' == argString[i]) { @@ -201,6 +276,8 @@ public static CLogTypeContainer[] BuildTypes(CLogConfigurationFile configFile, C newNode.LeadingString = prefixString; newNode.ArgStartingIndex = i; + currentArg = decompString.CreateNewArg(); + ++i; // Check to see if a custom name is specified for this type @@ -245,6 +322,7 @@ public static CLogTypeContainer[] BuildTypes(CLogConfigurationFile configFile, C { // 'i' will point to the final character on a match (such that i+1 is the next fresh character) t = configFile.FindTypeAndAdvance(argString, traceLineMatch, ref i); + } catch (CLogTypeNotFoundException) { @@ -253,6 +331,7 @@ public static CLogTypeContainer[] BuildTypes(CLogConfigurationFile configFile, C newNode.TypeNode = t; newNode.ArgLength = i - newNode.ArgStartingIndex + 1; + currentArg.Type = t; // If we found a preferred name, the next character after the type should be a closing brace if (preferredName.Length != 0) @@ -276,7 +355,9 @@ public static CLogTypeContainer[] BuildTypes(CLogConfigurationFile configFile, C } } - cleanedString = pieces; + if (!pieces.Equals(decompString.AsManifestedETWEncoding)) + throw new Exception("ETW strings dont match"); + return ret.ToArray(); } @@ -299,13 +380,20 @@ private static (string, string)[] MakeVariable(CLogConfigurationFile configFile, private static CLogDecodedTraceLine BuildArgsFromEncodedArgsX(CLogConfigurationFile configFile, string sourcefile, CLogTraceMacroDefination macroDefination, CLogLineMatch traceLineMatch, string traceLine) { - string userArgs = macroDefination.CombinePrefixWithEncodedString(traceLineMatch.EncodingString); - string cleanedString; + string userArgs = String.Empty; + + if (!String.IsNullOrEmpty(macroDefination.MacroNameConversionName)) + userArgs = ""; + else + userArgs = macroDefination.CombinePrefixWithEncodedString(traceLineMatch.EncodingString); + + CLogFileProcessor.DecomposedString decompString; + // // Loop across all types, ignoring the ones that are not specified in the source code // Queue types = new Queue(); - foreach (var type in BuildTypes(configFile, traceLineMatch, userArgs, traceLine, out cleanedString)) + foreach (var type in BuildTypes(configFile, traceLineMatch, userArgs, traceLine, out decompString)) { if (type.TypeNode.Synthesized) { @@ -347,6 +435,10 @@ private static CLogDecodedTraceLine BuildArgsFromEncodedArgsX(CLogConfigurationF bundle.TypeNode = type; finalArgs.Add(bundle); + } + else if(!String.IsNullOrEmpty(macroDefination.MacroNameConversionName)) + { + } else { @@ -375,7 +467,7 @@ private static CLogDecodedTraceLine BuildArgsFromEncodedArgsX(CLogConfigurationF if (item.PreferredName.Length > configFile.MaximumVariableLength || hasBadChars) { - Console.WriteLine($"WARNING: {item.PreferredName} contains invalid characters (must be <= {configFile.MaximumVariableLength} characters and containing only alphanumeric plus underscore, using {info.SuggestedTelemetryName} instead"); + Console.WriteLine($"WARNING: {item.PreferredName} contains invalid characters (must be <= {configFile.MaximumVariableLength} characters and containing only alphanumeric plus underscore. Ignoring user specified {item.PreferredName} and using {info.SuggestedTelemetryName} instead!"); } else { @@ -404,7 +496,10 @@ private static CLogDecodedTraceLine BuildArgsFromEncodedArgsX(CLogConfigurationF throw new CLogEnterReadOnlyModeException("InvalidUniqueID", CLogHandledException.ExceptionType.InvalidUniqueId, traceLineMatch); } - CLogDecodedTraceLine decodedTraceLine = new CLogDecodedTraceLine(traceLineMatch.UniqueID, sourcefile, userArgs, traceLineMatch.EncodingString, traceLineMatch, configFile, macroDefination, finalArgs.ToArray(), cleanedString); + //if(traceLineMatch.UniqueID.Length >= 47) + // throw new CLogEnterReadOnlyModeException("TooManyCharacters", CLogHandledException.ExceptionType.InvalidUniqueId, traceLineMatch); + + CLogDecodedTraceLine decodedTraceLine = new CLogDecodedTraceLine(traceLineMatch.UniqueID, sourcefile, userArgs, traceLineMatch.EncodingString, traceLineMatch, configFile, macroDefination, finalArgs.ToArray(), decompString); return decodedTraceLine; } @@ -434,9 +529,6 @@ public string ConvertFile(CLogConfigurationFile configFile, CLogOutputInfo outpu try { lastMatch = match; - - string[] splitArgs = SplitWithEscapedQuotes(match.Value.AllArgs, ','); - int idx = match.Value.MatchedRegExX.Index - 1; while (idx > 0 && (contents[idx] == ' ' || contents[idx] == '\t')) diff --git a/src/clogutils/CLogFullyDecodedMacroEmitter.cs b/src/clogutils/CLogFullyDecodedMacroEmitter.cs index 59d9f24..7a7624f 100644 --- a/src/clogutils/CLogFullyDecodedMacroEmitter.cs +++ b/src/clogutils/CLogFullyDecodedMacroEmitter.cs @@ -1,4 +1,4 @@ -/*++ +/*++ Copyright (c) Microsoft Corporation. Licensed under the MIT License. @@ -38,7 +38,6 @@ public CLogFullyDecodedMacroEmitter(string inputSourceFile, CLogSidecar sidecar) { _inputSourceFile = inputSourceFile; _sidecar = sidecar; - _sourceFile.AppendLine($"// CLOG generated {DateTimeOffset.Now}"); _sourceFile.AppendLine("#include "); } @@ -96,7 +95,7 @@ public void TraceLineDiscovered(CLogDecodedTraceLine decodedTraceLine, CLogOutpu throw new CLogEnterReadOnlyModeException("ByteArrayNotUsingCLOG_BYTEARRAY", CLogHandledException.ExceptionType.ArrayMustUseMacro, decodedTraceLine.match); } break; - case CLogEncodingType.Int32Array: + /* case CLogEncodingType.Int32Array: case CLogEncodingType.UInt32Array: case CLogEncodingType.Int64Array: case CLogEncodingType.UInt64Array: @@ -122,7 +121,7 @@ public void TraceLineDiscovered(CLogDecodedTraceLine decodedTraceLine, CLogOutpu throw new CLogEnterReadOnlyModeException("ByteArrayNotUsingCLOG_ARRAY", CLogHandledException.ExceptionType.ArrayMustUseMacro, decodedTraceLine.match); } - break; + break;*/ default: clogArgCountForMacroAlignment++; break; @@ -133,7 +132,7 @@ public void TraceLineDiscovered(CLogDecodedTraceLine decodedTraceLine, CLogOutpu string implSignature = $" clogTraceImpl_{clogArgCountForMacroAlignment}_ARGS_TRACE_{decodedTraceLine.UniqueId}("; string macroName = $"_clog_{clogArgCountForMacroAlignment}_ARGS_TRACE_{decodedTraceLine.UniqueId}"; - _headerFile.AppendLine($"#ifndef {macroName}"); + if (-1 != decodedTraceLine.macro.EncodedArgNumber) { @@ -162,7 +161,7 @@ public void TraceLineDiscovered(CLogDecodedTraceLine decodedTraceLine, CLogOutpu implSignature += $", {v.CType} {arg.VariableInfo.IndexBasedName}"; argsString += $", {arg.VariableInfo.IndexBasedName}"; - if (v.EncodingType == CLogEncodingType.ByteArray || + if (v.EncodingType == CLogEncodingType.ByteArray /*|| v.EncodingType == CLogEncodingType.Int32Array || v.EncodingType == CLogEncodingType.UInt32Array || v.EncodingType == CLogEncodingType.Int64Array || @@ -173,7 +172,7 @@ public void TraceLineDiscovered(CLogDecodedTraceLine decodedTraceLine, CLogOutpu v.EncodingType == CLogEncodingType.GUIDArray || v.EncodingType == CLogEncodingType.Int16Array || v.EncodingType == CLogEncodingType.UInt16Array || - v.EncodingType == CLogEncodingType.Int8Array) + v.EncodingType == CLogEncodingType.Int8Array*/) { implSignature += $", int {arg.VariableInfo.IndexBasedName}_len"; argsString += $", {arg.VariableInfo.IndexBasedName}_len"; @@ -199,9 +198,6 @@ public void TraceLineDiscovered(CLogDecodedTraceLine decodedTraceLine, CLogOutpu StringBuilder macroBody = new StringBuilder(); - _headerFile.AppendLine(""); - _headerFile.AppendLine(""); - _headerFile.AppendLine(""); _headerFile.AppendLine("/*----------------------------------------------------------"); _headerFile.AppendLine($"// Decoder Ring for {decodedTraceLine.UniqueId}"); _headerFile.AppendLine($"// {decodedTraceLine.TraceString}"); @@ -216,6 +212,7 @@ public void TraceLineDiscovered(CLogDecodedTraceLine decodedTraceLine, CLogOutpu } _headerFile.AppendLine("----------------------------------------------------------*/"); + _headerFile.AppendLine($"#ifndef {macroName}"); // // BUGBUG: not fully implemented - the intent of 'implSignature' is to give a turn key @@ -281,7 +278,7 @@ public void TraceLineDiscovered(CLogDecodedTraceLine decodedTraceLine, CLogOutpu CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, " typo in a trace string) - have two options to override/refresh the signature check"); CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Std, ""); CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Std, " Force/Clobber the event signature - indicating you desire breaking the uniqueness contract"); - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Std, $" 1. remove UniquenessHash ({existingTraceInfo.UniquenessHash}) frome this TraceID({existingTraceInfo.TraceID}) in file {decodedTraceLine.configFile.FilePath}"); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Std, $" 1. remove UniquenessHash ({existingTraceInfo.UniquenessHash}) from this TraceID({existingTraceInfo.TraceID}) in file {decodedTraceLine.configFile.FilePath}"); CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Std, $" 2. specify the --overwriteHashCollisions command line argument (good if you're making lots of changes that are all safe)"); CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Tip, $" 3. set the environment variable CLOG_DEVELOPMENT_MODE=1 ($env:CLOG_DEVELOPMENT_MODE=1)"); CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Std, ""); diff --git a/src/clogutils/CLogHandledException.cs b/src/clogutils/CLogHandledException.cs index ae541d2..0f97609 100644 --- a/src/clogutils/CLogHandledException.cs +++ b/src/clogutils/CLogHandledException.cs @@ -46,7 +46,9 @@ public enum ExceptionType InvalidInputFile = 27, UndefinedTypeToLanguageMapping = 28, InvalidNameFormatInTypeSpcifier = 29, - EncodedArgNumberInvalid = 30 + EncodedArgNumberInvalid = 30, + SidecarFileVersionMismatch = 31, + SidecarCorrupted = 32 } public static string TranslateExceptionTypeToErrorMessage(ExceptionType e) @@ -96,7 +98,7 @@ public static string TranslateExceptionTypeToErrorMessage(ExceptionType e) case ExceptionType.ETWTypeMismatch: return "CLOG defined types mismatch with existing ETW manifest - you must fix the ETW manifest manually to align the types"; case ExceptionType.InvalidUniqueId: - return "CLOG Unique IDs must be alphanumeric"; + return "CLOG Unique IDs must be alphanumeric and 47 or fewer characters"; case ExceptionType.WontWriteInReadOnlyMode: return "Wont write while in readonly mode. --readOnly was specified as a command line argument. If you're in a development mode, you can set the environment CLOG_DEVELOPMENT_MODE such that manifests and sidecars will be automatically updated"; case ExceptionType.RequiredConfigParameterUnspecified: @@ -105,6 +107,10 @@ public static string TranslateExceptionTypeToErrorMessage(ExceptionType e) return "Invalid input file"; case ExceptionType.UndefinedTypeToLanguageMapping: return "Must specify conversion from CLOG Type to Language Type in config file"; + case ExceptionType.SidecarFileVersionMismatch: + return "Invalid sidecar file version and unable to update - consider updating clog or correcting the version number"; + case ExceptionType.SidecarCorrupted: + return "Sidecar cannot be opened; it seems to be corrupted - consider deleting and rebuilding, or locating the source of the corruption"; } return "Uknown Error"; @@ -132,6 +138,8 @@ public void PrintDiagnostics() { string fileLine = CLogConsoleTrace.GetFileLine(TraceLine); CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $"{fileLine}: fatal error CLOG{(int)Type}: {TranslateExceptionTypeToErrorMessage(Type)}"); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $"{Exception}"); + } catch (Exception) { diff --git a/src/clogutils/CLogSidecar.cs b/src/clogutils/CLogSidecar.cs index ef20780..d07fec0 100644 --- a/src/clogutils/CLogSidecar.cs +++ b/src/clogutils/CLogSidecar.cs @@ -1,4 +1,4 @@ -/*++ +/*++ Copyright (c) Microsoft Corporation. Licensed under the MIT License. @@ -77,7 +77,6 @@ public CLogModuleUsageInformation_V2 ModuleUniqueness get; set; } - public ClogSidecar_V2() { Version = 2; @@ -95,21 +94,31 @@ public string ToJson() public static ClogSidecar_V2 FromJson(string json) { - JsonSerializerSettings s = new JsonSerializerSettings(); - s.Context = new StreamingContext(StreamingContextStates.Other, json); + try + { + JsonSerializerSettings s = new JsonSerializerSettings(); + s.Context = new StreamingContext(StreamingContextStates.Other, json); - ClogSidecar_V2 ret = JsonConvert.DeserializeObject(json, s); + ClogSidecar_V2 ret = JsonConvert.DeserializeObject(json, s); - if (1 == ret.Version) + if (1 == ret.Version) + { + ClogSidecar_V1 v1 = ClogSidecar_V1.FromJson(json); + ret = new ClogSidecar_V2(); + ret.Version = 2; + ret.EventBundlesV2 = v1.EventBundlesV2; + ret.ConfigFile = v1.ConfigFile; + ret.ModuleUniqueness = CLogModuleUsageInformation_V2.ConvertFromV1(v1.ModuleUniqueness); + } + + if (2 != ret.Version) + throw new CLogEnterReadOnlyModeException($"Invalid sidecar file version {ret.Version}, our max version is 2 - consider updating CLOG", CLogHandledException.ExceptionType.SidecarFileVersionMismatch, null); + return ret; + } + catch(System.Exception e) { - ClogSidecar_V1 v1 = JsonConvert.DeserializeObject(json, s); - ret = new ClogSidecar_V2(); - ret.Version = 2; - ret.EventBundlesV2 = v1.EventBundlesV2; - ret.ConfigFile = v1.ConfigFile; - ret.ModuleUniqueness = CLogModuleUsageInformation_V2.ConvertFromV1(v1.ModuleUniqueness); + throw new CLogEnterReadOnlyModeException($"Invalid sidecar file version - it looks corrupted", CLogHandledException.ExceptionType.SidecarCorrupted, null, e); } - return ret; } } @@ -266,7 +275,7 @@ private void MergeHot() { // // Merge in hot values - // + // foreach (var hot in HotEventBundles) { CLogDecodedTraceLine old; @@ -313,10 +322,8 @@ public bool AreDirty private set { _areDirty = value; } } - public CLogDecodedTraceLine FindBundle(string uid) + public CLogDecodedTraceLine FindBundle(string name) { - string name = uid.Split(':')[1]; - if (!_sideCarFile.EventBundlesV2.ContainsKey(name)) { return null; diff --git a/src/clogutils/ConfigFile/CLogModuleUsageInformation.cs b/src/clogutils/ConfigFile/CLogModuleUsageInformation.cs index 4d1a1c9..501f495 100644 --- a/src/clogutils/ConfigFile/CLogModuleUsageInformation.cs +++ b/src/clogutils/ConfigFile/CLogModuleUsageInformation.cs @@ -1,4 +1,4 @@ -/*++ +/*++ Copyright (c) Microsoft Corporation. Licensed under the MIT License. diff --git a/src/clogutils/MacroDefinations/CLogTraceMacroDefination.cs b/src/clogutils/MacroDefinations/CLogTraceMacroDefination.cs index 9ef6f64..8dfb75e 100644 --- a/src/clogutils/MacroDefinations/CLogTraceMacroDefination.cs +++ b/src/clogutils/MacroDefinations/CLogTraceMacroDefination.cs @@ -31,8 +31,16 @@ public class CLogTraceMacroDefination [JsonProperty] public bool ClassFunctionEncoding { get; set; } + [JsonProperty] public string MacroNameConversionName { get; set; } + + public string ConfigFileWithMacroDefination { get; set; } + public bool ShouldSerializeMacroNameConversionName() + { + return false; + } + public bool ShouldSerializeClassFunctionEncoding() { return (true == ClassFunctionEncoding); diff --git a/src/converters/syslog2clog/CommandLineArguments.cs b/src/converters/syslog2clog/CommandLineArguments.cs index 0213cff..bdab70b 100644 --- a/src/converters/syslog2clog/CommandLineArguments.cs +++ b/src/converters/syslog2clog/CommandLineArguments.cs @@ -51,7 +51,7 @@ public bool ReadOnly set; } - [Option("scopePrefix", SetName = "build", Required = false, HelpText = "scope prefix")] + [Option("scopePrefix", SetName = "build", Required = false, HelpText = "scope prefix; this value will prefix CLOG functions() and helps to provide scope. Typically this value is reused by an entire module or directory (your choice)")] public string ScopePrefix { get; diff --git a/src/converters/syslog2clog/syslog2clog.cs b/src/converters/syslog2clog/syslog2clog.cs index 07f71b1..31ca889 100644 --- a/src/converters/syslog2clog/syslog2clog.cs +++ b/src/converters/syslog2clog/syslog2clog.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Text; @@ -39,8 +39,7 @@ public void TraceLineDiscovered(CLogDecodedTraceLine decodedTraceLine, CLogOutpu ++idx; } - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Std, $"{idx}. "); string choice = null; while (String.IsNullOrEmpty(choice)) @@ -155,17 +154,10 @@ private static int Main(string[] args) CLogFileProcessor processor = new CLogFileProcessor(configFile); SysLogToClog converter = new SysLogToClog(); - - // CLogFullyDecodedMacroEmitter fullyDecodedMacroEmitter = new CLogFullyDecodedMacroEmitter(options.InputFile, sidecar); - - // fullyDecodedMacroEmitter.AddClogModule(converter); - + string content = File.ReadAllText(options.InputFile); - // CLogOutputInfo outputInfo = null; string output = processor.ConvertFile(configFile, null, converter, content, options.InputFile, true); - // fullyDecodedMacroEmitter.FinishedProcessing(); - if (!Directory.Exists(Path.GetDirectoryName(options.OutputFile))) Directory.CreateDirectory(Path.GetDirectoryName(options.OutputFile)); From f2871f416b85d68e6b96d23c5d517bce34d9d1fd Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Mon, 18 Oct 2021 21:03:42 -0700 Subject: [PATCH 02/19] cleanup some formatting issues being flagged by CI/CD policy --- src/clog2text/clog2text_windows/Program.cs | 10 ++++---- .../CLogCustomTraceEmittorFactory.cs | 23 +++++++++---------- src/clogutils/CLogEncodingType.cs | 22 +++++++++--------- src/clogutils/CLogFileProcessor.cs | 15 ++++++------ src/clogutils/CLogSidecar.cs | 2 +- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/clog2text/clog2text_windows/Program.cs b/src/clog2text/clog2text_windows/Program.cs index 243b319..b78e172 100644 --- a/src/clog2text/clog2text_windows/Program.cs +++ b/src/clog2text/clog2text_windows/Program.cs @@ -237,11 +237,11 @@ private static int Main(string[] cmdLineArgs) e.PrintDiagnostics(); return -2; } - }, err => - { - Console.WriteLine("Bad Args : " + err); - return -1; - }); + }, err => + { + Console.WriteLine("Bad Args : " + err); + return -1; + }); } public class ManifestedETWEvent : IClogEventArg diff --git a/src/clogutils/CLogCustomTraceEmittorFactory.cs b/src/clogutils/CLogCustomTraceEmittorFactory.cs index 4049548..14ba265 100644 --- a/src/clogutils/CLogCustomTraceEmittorFactory.cs +++ b/src/clogutils/CLogCustomTraceEmittorFactory.cs @@ -18,7 +18,6 @@ In some cases the human display of a CLOG event will require a type with custom using System.Linq; using System.Reflection; using System.Runtime; -using System.Runtime.InteropServices; using clogutils; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; @@ -117,17 +116,17 @@ public bool Decode(CLogEncodingCLogTypeSearch type, IClogEventArg value, CLogLin break; case CLogEncodingType.ByteArray: - /*case CLogEncodingType.UInt64Array: - case CLogEncodingType.Int32Array: - case CLogEncodingType.UInt32Array: - case CLogEncodingType.Int64Array: - case CLogEncodingType.ANSI_StringArray: - case CLogEncodingType.UNICODE_StringArray: - case CLogEncodingType.PointerArray: - case CLogEncodingType.GUIDArray: - case CLogEncodingType.Int16Array: - case CLogEncodingType.UInt16Array: - case CLogEncodingType.Int8Array:*/ + /*case CLogEncodingType.UInt64Array: + case CLogEncodingType.Int32Array: + case CLogEncodingType.UInt32Array: + case CLogEncodingType.Int64Array: + case CLogEncodingType.ANSI_StringArray: + case CLogEncodingType.UNICODE_StringArray: + case CLogEncodingType.PointerArray: + case CLogEncodingType.GUIDArray: + case CLogEncodingType.Int16Array: + case CLogEncodingType.UInt16Array: + case CLogEncodingType.Int8Array:*/ args[0] = value.AsBinary; break; diff --git a/src/clogutils/CLogEncodingType.cs b/src/clogutils/CLogEncodingType.cs index 0117b16..f30c08c 100644 --- a/src/clogutils/CLogEncodingType.cs +++ b/src/clogutils/CLogEncodingType.cs @@ -37,17 +37,17 @@ public enum CLogEncodingType : uint ByteArray = 15, UserEncodingString = 16, UniqueAndDurableIdentifier = 17, - /* Int32Array = 18, - UInt32Array = 19, - Int64Array = 20, - UInt64Array = 21, - ANSI_StringArray = 22, - UNICODE_StringArray = 23, - PointerArray = 24, - GUIDArray = 25, - Int16Array = 26, - UInt16Array = 27, - Int8Array = 28,*/ + /* Int32Array = 18, + UInt32Array = 19, + Int64Array = 20, + UInt64Array = 21, + ANSI_StringArray = 22, + UNICODE_StringArray = 23, + PointerArray = 24, + GUIDArray = 25, + Int16Array = 26, + UInt16Array = 27, + Int8Array = 28,*/ // UInt8Array, // Not used, use ByteArray //Struct = 29 } diff --git a/src/clogutils/CLogFileProcessor.cs b/src/clogutils/CLogFileProcessor.cs index 2cc39d4..51968ff 100644 --- a/src/clogutils/CLogFileProcessor.cs +++ b/src/clogutils/CLogFileProcessor.cs @@ -11,7 +11,6 @@ Licensed under the MIT License. using System; using System.Collections.Generic; -using System.ComponentModel; using System.Linq; using System.Security.Cryptography; using System.Text; @@ -201,7 +200,7 @@ public string AsPrintF { if (null != e.Type) { - switch(e.Type.EncodingType) + switch (e.Type.EncodingType) { case CLogEncodingType.ByteArray: ret += "p"; @@ -219,13 +218,15 @@ public string AsPrintF return ret; } } - public string AsManifestedETWEncoding { - get { + public string AsManifestedETWEncoding + { + get + { string ret = ""; int idx = 0; - foreach(var e in encodings) + foreach (var e in encodings) { - if(null != e.Type) + if (null != e.Type) { //ret += e.Type.DefinationEncoding; ret += idx; @@ -436,7 +437,7 @@ private static CLogDecodedTraceLine BuildArgsFromEncodedArgsX(CLogConfigurationF bundle.TypeNode = type; finalArgs.Add(bundle); } - else if(!String.IsNullOrEmpty(macroDefination.MacroNameConversionName)) + else if (!String.IsNullOrEmpty(macroDefination.MacroNameConversionName)) { } diff --git a/src/clogutils/CLogSidecar.cs b/src/clogutils/CLogSidecar.cs index d07fec0..eafa9e0 100644 --- a/src/clogutils/CLogSidecar.cs +++ b/src/clogutils/CLogSidecar.cs @@ -115,7 +115,7 @@ public static ClogSidecar_V2 FromJson(string json) throw new CLogEnterReadOnlyModeException($"Invalid sidecar file version {ret.Version}, our max version is 2 - consider updating CLOG", CLogHandledException.ExceptionType.SidecarFileVersionMismatch, null); return ret; } - catch(System.Exception e) + catch (System.Exception e) { throw new CLogEnterReadOnlyModeException($"Invalid sidecar file version - it looks corrupted", CLogHandledException.ExceptionType.SidecarCorrupted, null, e); } From 68e5cbff7686c665aee6a2e2996e2636578b7b28 Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Mon, 18 Oct 2021 21:12:39 -0700 Subject: [PATCH 03/19] code formating --- .../CLogLTTNGOutputModule.cs | 2 +- src/clog/TraceEmitterModules/CLogSysLog.cs | 3 - src/clog/clog.cs | 5 +- src/clogutils/CLogDecodedTraceLine.cs | 2 - src/clogutils/CLogEncodingCLogTypeSearch.cs | 5 +- src/clogutils/CLogFullyDecodedMacroEmitter.cs | 56 +++++++++---------- src/clogutils/CodeDomHelper/Extensions.cs | 2 - .../CodeDomHelper/TargetFramework.cs | 2 - .../ConfigFile/CLogConfigurationFile.cs | 1 - src/converters/syslog2clog/syslog2clog.cs | 2 +- 10 files changed, 34 insertions(+), 46 deletions(-) diff --git a/src/clog/TraceEmitterModules/CLogLTTNGOutputModule.cs b/src/clog/TraceEmitterModules/CLogLTTNGOutputModule.cs index 9f51b19..c297852 100644 --- a/src/clog/TraceEmitterModules/CLogLTTNGOutputModule.cs +++ b/src/clog/TraceEmitterModules/CLogLTTNGOutputModule.cs @@ -109,7 +109,7 @@ public void FinishedProcessing(CLogOutputInfo outputInfo, StringBuilder header, sourceFile.AppendLine("#define TRACEPOINT_DEFINE"); } sourceFile.AppendLine($"#include \"{Path.GetFileName(_clogFile)}\""); - } + } public void TraceLineDiscovered(string sourceFile, CLogOutputInfo outputInfo, CLogDecodedTraceLine decodedTraceLine, CLogSidecar sidecar, StringBuilder macroPrefix, StringBuilder inline, StringBuilder function) diff --git a/src/clog/TraceEmitterModules/CLogSysLog.cs b/src/clog/TraceEmitterModules/CLogSysLog.cs index 2f82490..5a35512 100644 --- a/src/clog/TraceEmitterModules/CLogSysLog.cs +++ b/src/clog/TraceEmitterModules/CLogSysLog.cs @@ -11,11 +11,8 @@ Licensed under the MIT License. --*/ using System; -using System.Collections.Generic; -using System.IO; using System.Text; using clogutils; -using clogutils.MacroDefinations; namespace clog.TraceEmitterModules { diff --git a/src/clog/clog.cs b/src/clog/clog.cs index d396538..4f3c8d5 100644 --- a/src/clog/clog.cs +++ b/src/clog/clog.cs @@ -14,7 +14,6 @@ Licensed under the MIT License. using System; using System.Collections.Generic; using System.IO; -using System.Reflection; using System.Text; using clog.TraceEmitterModules; using clogutils; @@ -177,7 +176,7 @@ private static int Main(string[] args) CLogSTDOUT stdout = new CLogSTDOUT(); fullyDecodedMacroEmitter.AddClogModule(stdout); - + CLogManifestedETWOutputModule manifestedEtwOutput = new CLogManifestedETWOutputModule(options.ReadOnly); fullyDecodedMacroEmitter.AddClogModule(manifestedEtwOutput); @@ -297,7 +296,7 @@ private static int Main(string[] args) } catch (CLogHandledException e) { - if(null != currentFile) + if (null != currentFile) CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $"Failure in file : {currentFile}"); e.PrintDiagnostics(); return -2; diff --git a/src/clogutils/CLogDecodedTraceLine.cs b/src/clogutils/CLogDecodedTraceLine.cs index 6468020..f50b65d 100644 --- a/src/clogutils/CLogDecodedTraceLine.cs +++ b/src/clogutils/CLogDecodedTraceLine.cs @@ -10,8 +10,6 @@ Licensed under the MIT License. --*/ using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Runtime.Serialization; using clogutils.ConfigFile; using clogutils.MacroDefinations; diff --git a/src/clogutils/CLogEncodingCLogTypeSearch.cs b/src/clogutils/CLogEncodingCLogTypeSearch.cs index 638be9f..b9a68c7 100644 --- a/src/clogutils/CLogEncodingCLogTypeSearch.cs +++ b/src/clogutils/CLogEncodingCLogTypeSearch.cs @@ -40,8 +40,9 @@ public class CLogEncodingCLogTypeSearch public bool IsEncodableArg { - get { - return !Synthesized && + get + { + return !Synthesized && (EncodingType != CLogEncodingType.UniqueAndDurableIdentifier && EncodingType != CLogEncodingType.UserEncodingString); } diff --git a/src/clogutils/CLogFullyDecodedMacroEmitter.cs b/src/clogutils/CLogFullyDecodedMacroEmitter.cs index 7a7624f..d6009a4 100644 --- a/src/clogutils/CLogFullyDecodedMacroEmitter.cs +++ b/src/clogutils/CLogFullyDecodedMacroEmitter.cs @@ -13,9 +13,7 @@ Licensed under the MIT License. using System; using System.Collections.Generic; -using System.Security.Cryptography.X509Certificates; using System.Text; -using clogutils; using clogutils.ConfigFile; using clogutils.MacroDefinations; @@ -95,33 +93,33 @@ public void TraceLineDiscovered(CLogDecodedTraceLine decodedTraceLine, CLogOutpu throw new CLogEnterReadOnlyModeException("ByteArrayNotUsingCLOG_BYTEARRAY", CLogHandledException.ExceptionType.ArrayMustUseMacro, decodedTraceLine.match); } break; - /* case CLogEncodingType.Int32Array: - case CLogEncodingType.UInt32Array: - case CLogEncodingType.Int64Array: - case CLogEncodingType.UInt64Array: - case CLogEncodingType.ANSI_StringArray: - case CLogEncodingType.UNICODE_StringArray: - case CLogEncodingType.PointerArray: - case CLogEncodingType.GUIDArray: - case CLogEncodingType.Int16Array: - case CLogEncodingType.UInt16Array: - case CLogEncodingType.Int8Array: - clogArgCountForMacroAlignment += 2; - - // Verify the input argument contains CLOG_ARRAY - this will aid in debugging - if (!arg.VariableInfo.UserSpecifiedUnModified.Contains("CLOG_ARRAY")) - { - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $"Trace ID '{decodedTraceLine.UniqueId}' contains an non-byte array type that is not using the CLOG_ARRAY macro"); - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, " Please encode the following argument with CLOG_ARRAY(length, pointer)"); - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, ""); - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $"// {decodedTraceLine.match.MatchedRegExX}"); - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, ""); - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $"Failing Arg: "); - CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, arg.VariableInfo.UserSuppliedTrimmed); - throw new CLogEnterReadOnlyModeException("ByteArrayNotUsingCLOG_ARRAY", CLogHandledException.ExceptionType.ArrayMustUseMacro, decodedTraceLine.match); - } - - break;*/ + /* case CLogEncodingType.Int32Array: + case CLogEncodingType.UInt32Array: + case CLogEncodingType.Int64Array: + case CLogEncodingType.UInt64Array: + case CLogEncodingType.ANSI_StringArray: + case CLogEncodingType.UNICODE_StringArray: + case CLogEncodingType.PointerArray: + case CLogEncodingType.GUIDArray: + case CLogEncodingType.Int16Array: + case CLogEncodingType.UInt16Array: + case CLogEncodingType.Int8Array: + clogArgCountForMacroAlignment += 2; + + // Verify the input argument contains CLOG_ARRAY - this will aid in debugging + if (!arg.VariableInfo.UserSpecifiedUnModified.Contains("CLOG_ARRAY")) + { + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $"Trace ID '{decodedTraceLine.UniqueId}' contains an non-byte array type that is not using the CLOG_ARRAY macro"); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, " Please encode the following argument with CLOG_ARRAY(length, pointer)"); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, ""); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $"// {decodedTraceLine.match.MatchedRegExX}"); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, ""); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $"Failing Arg: "); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, arg.VariableInfo.UserSuppliedTrimmed); + throw new CLogEnterReadOnlyModeException("ByteArrayNotUsingCLOG_ARRAY", CLogHandledException.ExceptionType.ArrayMustUseMacro, decodedTraceLine.match); + } + + break;*/ default: clogArgCountForMacroAlignment++; break; diff --git a/src/clogutils/CodeDomHelper/Extensions.cs b/src/clogutils/CodeDomHelper/Extensions.cs index 1ea013e..22597c3 100644 --- a/src/clogutils/CodeDomHelper/Extensions.cs +++ b/src/clogutils/CodeDomHelper/Extensions.cs @@ -1,6 +1,4 @@ using System; -using System.Collections.Generic; -using System.Text; using Basic.Reference.Assemblies; using Microsoft.CodeAnalysis.CSharp; diff --git a/src/clogutils/CodeDomHelper/TargetFramework.cs b/src/clogutils/CodeDomHelper/TargetFramework.cs index 5ce69f5..6df7f36 100644 --- a/src/clogutils/CodeDomHelper/TargetFramework.cs +++ b/src/clogutils/CodeDomHelper/TargetFramework.cs @@ -1,5 +1,3 @@ -using Microsoft.CodeAnalysis; - namespace Roslyn.CodeDom { public enum TargetFramework diff --git a/src/clogutils/ConfigFile/CLogConfigurationFile.cs b/src/clogutils/ConfigFile/CLogConfigurationFile.cs index b0c3fe7..c953bcd 100644 --- a/src/clogutils/ConfigFile/CLogConfigurationFile.cs +++ b/src/clogutils/ConfigFile/CLogConfigurationFile.cs @@ -19,7 +19,6 @@ other instances of configuration files. using System.Runtime.Serialization; using clogutils.MacroDefinations; using Newtonsoft.Json; -using static clogutils.CLogConsoleTrace; namespace clogutils.ConfigFile { diff --git a/src/converters/syslog2clog/syslog2clog.cs b/src/converters/syslog2clog/syslog2clog.cs index 31ca889..4652de7 100644 --- a/src/converters/syslog2clog/syslog2clog.cs +++ b/src/converters/syslog2clog/syslog2clog.cs @@ -154,7 +154,7 @@ private static int Main(string[] args) CLogFileProcessor processor = new CLogFileProcessor(configFile); SysLogToClog converter = new SysLogToClog(); - + string content = File.ReadAllText(options.InputFile); string output = processor.ConvertFile(configFile, null, converter, content, options.InputFile, true); From 36faed901744d9ae2e0e1555ec72e1c0a418455d Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Mon, 18 Oct 2021 21:37:32 -0700 Subject: [PATCH 04/19] fix some breaks in tests --- defaults/clog.h | 4 ++ defaults/defaults.clog_config | 60 ------------------ examples/clog.sidecar | 100 ++++++++++++------------------ src/clogutils/CLogEncodingType.cs | 19 +----- 4 files changed, 47 insertions(+), 136 deletions(-) diff --git a/defaults/clog.h b/defaults/clog.h index 3cc0d81..68f54ae 100644 --- a/defaults/clog.h +++ b/defaults/clog.h @@ -42,6 +42,10 @@ #define CLOG_STRUCT(numFields) CLOG_STRUCT_HELPER(numFields) #endif +#define CLOG_ARRAY_HELPER(length, pointer) pointer, length +#define CLOG_ARRAY(length, pointer) CLOG_ARRAY_HELPER(length, pointer) +#define CLOG_STRUCT_HELPER(numFields) numFields +#define CLOG_STRUCT(numFields) CLOG_STRUCT_HELPER(numFields) typedef const void * CLOG_PTR; typedef const unsigned char CLOG_UINT8; typedef const char CLOG_INT8; diff --git a/defaults/defaults.clog_config b/defaults/defaults.clog_config index 50394c3..e9d28a8 100644 --- a/defaults/defaults.clog_config +++ b/defaults/defaults.clog_config @@ -10,66 +10,6 @@ "CType": "CLOG_PTR", "DefinationEncoding": "!BYTEARRAY!" }, - { - "EncodingType": "Int32Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!INT32ARRAY!" - }, - { - "EncodingType": "UInt32Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!UINT32ARRAY!" - }, - { - "EncodingType": "Int64Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!INT64ARRAY!" - }, - { - "EncodingType": "UInt64Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!UINT64ARRAY!" - }, - { - "EncodingType": "ANSI_StringArray", - "CType": "CLOG_PTR", - "DefinationEncoding": "!ANSISTRINGARRAY!" - }, - { - "EncodingType": "UNICODE_StringArray", - "CType": "CLOG_PTR", - "DefinationEncoding": "!UNICODESTRINGARRAY!" - }, - { - "EncodingType": "PointerArray", - "CType": "CLOG_PTR", - "DefinationEncoding": "!POINTERARRAY!" - }, - { - "EncodingType": "GUIDArray", - "CType": "CLOG_PTR", - "DefinationEncoding": "!GUIDARRAY!" - }, - { - "EncodingType": "Int16Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!INT16ARRAY!" - }, - { - "EncodingType": "UInt16Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!UINT16ARRAY!" - }, - { - "EncodingType": "Int8Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!INT8ARRAY!" - }, - { - "EncodingType": "Struct", - "CType": "unsigned long int", - "DefinationEncoding": "!STRUCT!" - }, { "EncodingType": "Int32", "CType": "int", diff --git a/examples/clog.sidecar b/examples/clog.sidecar index 3dec198..d7e69af 100644 --- a/examples/clog.sidecar +++ b/examples/clog.sidecar @@ -151,262 +151,242 @@ { "EncodingType": "ByteArray", "CType": "CLOG_PTR", + "JavaType": null, "DefinationEncoding": "!BYTEARRAY!" }, - { - "EncodingType": "Int32Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!INT32ARRAY!" - }, - { - "EncodingType": "Int64Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!INT64ARRAY!" - }, - { - "EncodingType": "Int16Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!INT16ARRAY!" - }, - { - "EncodingType": "Int8Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!INT8ARRAY!" - }, - { - "EncodingType": "UInt32Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!UINT32ARRAY!" - }, - { - "EncodingType": "UInt64Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!UINT64ARRAY!" - }, - { - "EncodingType": "UInt16Array", - "CType": "CLOG_PTR", - "DefinationEncoding": "!UINT16ARRAY!" - }, - { - "EncodingType": "UNICODE_StringArray", - "CType": "CLOG_PTR", - "DefinationEncoding": "!UNICODESTRINGARRAY!" - }, - { - "EncodingType": "ANSI_StringArray", - "CType": "CLOG_PTR", - "DefinationEncoding": "!ANSISTRINGARRAY!" - }, - { - "EncodingType": "PointerArray", - "CType": "CLOG_PTR", - "DefinationEncoding": "!POINTERARRAY!" - }, - { - "EncodingType": "GUIDArray", - "CType": "CLOG_PTR", - "DefinationEncoding": "!GUIDARRAY!" - }, - { - "EncodingType": "Struct", - "CType": "unsigned long int", - "DefinationEncoding": "!STRUCT!" - }, { "EncodingType": "Int32", "CType": "int", + "JavaType": null, "DefinationEncoding": "d" }, { "EncodingType": "Int8", "CType": "signed char", + "JavaType": null, "DefinationEncoding": "hhd" }, { "EncodingType": "Int8", "CType": "signed char", + "JavaType": null, "DefinationEncoding": "hhi" }, { "EncodingType": "UInt8", "CType": "unsigned char", + "JavaType": null, "DefinationEncoding": "hhu" }, { "EncodingType": "UInt8", "CType": "unsigned char", + "JavaType": null, "DefinationEncoding": "hhx" }, { "EncodingType": "UInt8", "CType": "unsigned char", + "JavaType": null, "DefinationEncoding": "hhX" }, { "EncodingType": "UInt16", "CType": "unsigned short", + "JavaType": null, "DefinationEncoding": "hu" }, { "EncodingType": "UInt16", "CType": "unsigned short", + "JavaType": null, "DefinationEncoding": "hx" }, { "EncodingType": "UInt16", "CType": "unsigned short", + "JavaType": null, "DefinationEncoding": "hX" }, { "EncodingType": "Int32", "CType": "long int", + "JavaType": null, "DefinationEncoding": "ld" }, { "EncodingType": "Int64", "CType": "long long int", + "JavaType": null, "DefinationEncoding": "lld" }, { "EncodingType": "Int64", "CType": "long long int", + "JavaType": null, "DefinationEncoding": "lli" }, { "EncodingType": "UInt64", "CType": "unsigned long long int", + "JavaType": null, "DefinationEncoding": "llu" }, { "EncodingType": "UInt64", "CType": "unsigned long long int", + "JavaType": null, "DefinationEncoding": "llx" }, { "EncodingType": "UInt64", "CType": "unsigned long long int", + "JavaType": null, "DefinationEncoding": "llX" }, { "EncodingType": "Int32", "CType": "long int", + "JavaType": null, "DefinationEncoding": "li" }, { "EncodingType": "UInt32", "CType": "unsigned long int", + "JavaType": null, "DefinationEncoding": "lu" }, { "EncodingType": "UInt32", "CType": "unsigned long int", + "JavaType": null, "DefinationEncoding": "lx" }, { "EncodingType": "UInt32", "CType": "unsigned long int", + "JavaType": null, "DefinationEncoding": "lX" }, { "EncodingType": "UNICODE_String", "CType": "const wchar_t *", + "JavaType": null, "DefinationEncoding": "ls" }, { "EncodingType": "Int64", "CType": "size_t", + "JavaType": null, "DefinationEncoding": "zd" }, { "EncodingType": "Int64", "CType": "size_t", + "JavaType": null, "DefinationEncoding": "zi" }, { "EncodingType": "UInt64", "CType": "size_t", + "JavaType": null, "DefinationEncoding": "zu" }, { "EncodingType": "UInt64", "CType": "size_t", + "JavaType": null, "DefinationEncoding": "zx" }, { "EncodingType": "UInt64", "CType": "size_t", + "JavaType": null, "DefinationEncoding": "zX" }, { "EncodingType": "Int64", "CType": "ptrdiff_t", + "JavaType": null, "DefinationEncoding": "td" }, { "EncodingType": "Int64", "CType": "ptrdiff_t", + "JavaType": null, "DefinationEncoding": "ti" }, { "EncodingType": "UInt64", "CType": "ptrdiff_t", + "JavaType": null, "DefinationEncoding": "tu" }, { "EncodingType": "UInt64", "CType": "ptrdiff_t", + "JavaType": null, "DefinationEncoding": "tx" }, { "EncodingType": "UInt64", "CType": "ptrdiff_t", + "JavaType": null, "DefinationEncoding": "tX" }, { "EncodingType": "Int32", "CType": "int", + "JavaType": null, "DefinationEncoding": "i" }, { "EncodingType": "UInt32", "CType": "unsigned int", + "JavaType": null, "DefinationEncoding": "u" }, { "EncodingType": "UInt32", "CType": "unsigned int", + "JavaType": null, "DefinationEncoding": "x" }, { "EncodingType": "UInt32", "CType": "unsigned int", + "JavaType": null, "DefinationEncoding": "4.4x" }, { "EncodingType": "UInt32", "CType": "unsigned int", + "JavaType": null, "DefinationEncoding": "X" }, { "EncodingType": "UInt8", "CType": "char", + "JavaType": null, "DefinationEncoding": "c", "CustomDecoder": "defaults.clog_config.Types.DecodeChar" }, { "EncodingType": "ANSI_String", "CType": "const char *", + "JavaType": null, "DefinationEncoding": "s" }, { "EncodingType": "UNICODE_String", "CType": "const wchar_t *", + "JavaType": null, "DefinationEncoding": "S" }, { "EncodingType": "Pointer", "CType": "CLOG_PTR", + "JavaType": null, "DefinationEncoding": "p", "CustomDecoder": "defaults.clog_config.Types.DecodePointer" } diff --git a/src/clogutils/CLogEncodingType.cs b/src/clogutils/CLogEncodingType.cs index f30c08c..b472b0d 100644 --- a/src/clogutils/CLogEncodingType.cs +++ b/src/clogutils/CLogEncodingType.cs @@ -36,20 +36,7 @@ public enum CLogEncodingType : uint UInt8 = 14, ByteArray = 15, UserEncodingString = 16, - UniqueAndDurableIdentifier = 17, - /* Int32Array = 18, - UInt32Array = 19, - Int64Array = 20, - UInt64Array = 21, - ANSI_StringArray = 22, - UNICODE_StringArray = 23, - PointerArray = 24, - GUIDArray = 25, - Int16Array = 26, - UInt16Array = 27, - Int8Array = 28,*/ - // UInt8Array, // Not used, use ByteArray - //Struct = 29 + UniqueAndDurableIdentifier = 17 } public class ClogEventPrinter @@ -111,10 +98,10 @@ public interface IClogEventArg ulong AsPointer { get; } byte[] AsBinary { get; } - /* + /* ANSI_String = 7, UNICODE_String = 8, - + GUID = 10, ByteArray = 15, UserEncodingString = 16, From b54e55d560ff5238646984a8bc647e7a27e5006e Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Mon, 18 Oct 2021 22:30:28 -0700 Subject: [PATCH 05/19] working on example/test --- .azure/azure-pipelines.ci.yml | 5 ++--- examples/clogsample/simple.cpp | 2 +- src/clogutils/CLogCustomTraceEmittorFactory.cs | 12 +++++++++++- src/clogutils/CLogEncodingType.cs | 4 ++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.azure/azure-pipelines.ci.yml b/.azure/azure-pipelines.ci.yml index dfe09fc..58917fe 100644 --- a/.azure/azure-pipelines.ci.yml +++ b/.azure/azure-pipelines.ci.yml @@ -103,7 +103,6 @@ stages: mkdir build cd build - $env:CLOG_DEVELOPMENT_MODE = 1 cmake ../examples cmake --build . @@ -120,7 +119,7 @@ stages: lttng stop clogsample babeltrace --names all log/lttng/* > log/clog.babel - ../buildclog/artifacts/clog2text_lttng -i log/clog.babel -s ../../examples/examples/sidecar/clog.sidecar -o log/clog.log --showTimestamp --showCpuInfo + ../buildclog/artifacts/clog2text_lttng -i log/clog.babel -s ../../examples/clog.sidecar -o log/clog.log --showTimestamp --showCpuInfo cat log/clog.log cp log/clog.babel "$UploadPath/clogstatictp.babel" @@ -143,7 +142,7 @@ stages: lttng stop clogsampledyn babeltrace --names all log/lttng/* > log/clog.babel - ../buildclog/artifacts/clog2text_lttng -i log/clog.babel -s ../../examples/examples/sidecar/clog.sidecar -o log/clog.log --showTimestamp --showCpuInfo + ../buildclog/artifacts/clog2text_lttng -i log/clog.babel -s ../../examples/clog.sidecar -o log/clog.log --showTimestamp --showCpuInfo cat log/clog.log cp log/clog.babel "$UploadPath/clogdynamictp.babel" diff --git a/examples/clogsample/simple.cpp b/examples/clogsample/simple.cpp index c1d39fb..8b73ce9 100644 --- a/examples/clogsample/simple.cpp +++ b/examples/clogsample/simple.cpp @@ -19,7 +19,7 @@ int main(int argc, char* argv[]) char *buffer = new char[10]; for(char i=0; i<10; ++i) { - TraceInstanceInfo(INSTANCE_TEST, buffer, "1:%d 2:%s 3:%c 4:%u 5:%u 6:%u", 1, "2", 3, 4, 5, 6); + TraceInstanceInfo(INSTANCE_TEST, buffer, "1:%d 2:%s 3:%c 4:%u 5:%u 6:%u", 1, "2", '3', 4, 5, 6); } TraceInfo(DATABYTEARRAY, "%!BYTEARRAY!", CLOG_BYTEARRAY(5, "hello")); diff --git a/src/clogutils/CLogCustomTraceEmittorFactory.cs b/src/clogutils/CLogCustomTraceEmittorFactory.cs index 14ba265..3ac4a56 100644 --- a/src/clogutils/CLogCustomTraceEmittorFactory.cs +++ b/src/clogutils/CLogCustomTraceEmittorFactory.cs @@ -50,6 +50,9 @@ public bool Inited() public void PrepareAssemblyCompileIfNecessary() { + if (null == CustomTypeDecoder) + return; + if (null != _codeAssembly) return; @@ -94,11 +97,19 @@ internal void ConstructFromAssembly(byte[] assembly) public bool Decode(CLogEncodingCLogTypeSearch type, IClogEventArg value, CLogLineMatch traceLine, out string decodedValue) { + decodedValue = "ERROR:" + type.CustomDecoder; + // // Compiling also caches the assembly // PrepareAssemblyCompileIfNecessary(); + // + // If we dont have a specified decoder, indicate that by returning false (we could not decode) + // + if (null == _codeAssembly) + return true; + object[] args = new object[1]; switch (type.EncodingType) @@ -145,7 +156,6 @@ public bool Decode(CLogEncodingCLogTypeSearch type, IClogEventArg value, CLogLin var newType = _codeAssembly.GetType(customDecoder); var instance = _typesInterface = _codeAssembly.CreateInstance(customDecoder); - decodedValue = "ERROR:" + type.CustomDecoder; if (!_compiledConverterFunctions.ContainsKey(type.CustomDecoder)) { diff --git a/src/clogutils/CLogEncodingType.cs b/src/clogutils/CLogEncodingType.cs index b472b0d..452967a 100644 --- a/src/clogutils/CLogEncodingType.cs +++ b/src/clogutils/CLogEncodingType.cs @@ -69,8 +69,8 @@ public static string AsCorrectType(CLogEncodingType type, IClogEventArg arg) return arg.AsInt8.ToString(); case CLogEncodingType.UInt8: return arg.AsUInt8.ToString(); - /*case CLogEncodingType.ByteArray: - return arg..ToString(); + case CLogEncodingType.ByteArray: + return arg.ToString();/* case CLogEncodingType.UserEncodingString: return arg.AsInt32.ToString(); case CLogEncodingType.UniqueAndDurableIdentifier: From eb0c25f549e365a414c173bc9ef23d95fd2cc358 Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Tue, 19 Oct 2021 08:27:28 -0700 Subject: [PATCH 06/19] trying to make CI/CD work --- examples/clogsample/simple.cpp | 8 +++++++- examples/runTests.ps1 | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 examples/runTests.ps1 diff --git a/examples/clogsample/simple.cpp b/examples/clogsample/simple.cpp index 8b73ce9..95fdcfb 100644 --- a/examples/clogsample/simple.cpp +++ b/examples/clogsample/simple.cpp @@ -13,7 +13,12 @@ int main(int argc, char* argv[]) { - printf("Hello world!\n"); + for(int i=0; i<10000; ++i) + { + if(i % 100) + printf("%d\n", i); + + //printf("Hello world!\n"); TraceInfo(LAUNCHED, "Hello world - we just started here is an int=%d", 20); char *buffer = new char[10]; @@ -27,5 +32,6 @@ int main(int argc, char* argv[]) TraceInfo(DATACHAR, "This is a char: %c", 'a'); TraceInfo(DATAINT, "This is an int: %d", 1234); delete [] buffer; + } return 0; } diff --git a/examples/runTests.ps1 b/examples/runTests.ps1 new file mode 100644 index 0000000..d6dc5da --- /dev/null +++ b/examples/runTests.ps1 @@ -0,0 +1,21 @@ +mkdir log +mkdir log/lttng +lttng create clogsample -o=log/lttng + +lttng enable-event --userspace CLOG_* + +#lttng add-context --userspace --type=vpid --type=vtid +lttng start + +cd ./build/clogsample +$env:LD_PRELOAD="$PWD/libclogsampletracepointprovider.so" + +./clogsampledynamictp +lttng list --userspace | grep CLOG +./clogsample +lttng list --userspace | grep CLOG + +lttng stop clogsample +babeltrace --names all ./log/lttng/* > log/clog.babel +lttng destroy clogsample +cd ../.. \ No newline at end of file From 9dfd84e6d2a0d9005cc26224302c8608e2f626e7 Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Tue, 19 Oct 2021 08:29:21 -0700 Subject: [PATCH 07/19] attempt calling from CI/CD --- .azure/azure-pipelines.ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.azure/azure-pipelines.ci.yml b/.azure/azure-pipelines.ci.yml index 58917fe..528f77e 100644 --- a/.azure/azure-pipelines.ci.yml +++ b/.azure/azure-pipelines.ci.yml @@ -106,6 +106,8 @@ stages: cmake ../examples cmake --build . + ../../examples/runTests.ps1 + if ($IsLinux) { cd clogsample mkdir log From 0c295ae4b7ff0b963a92b16783112fe48ba24886 Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Tue, 19 Oct 2021 10:00:39 -0700 Subject: [PATCH 08/19] this is kinda working --- examples/clogsample/simple.cpp | 2 +- examples/runTests.ps1 | 31 ++++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/examples/clogsample/simple.cpp b/examples/clogsample/simple.cpp index 95fdcfb..9d0c015 100644 --- a/examples/clogsample/simple.cpp +++ b/examples/clogsample/simple.cpp @@ -13,7 +13,7 @@ int main(int argc, char* argv[]) { - for(int i=0; i<10000; ++i) + for(int i=0; i<10; ++i) { if(i % 100) printf("%d\n", i); diff --git a/examples/runTests.ps1 b/examples/runTests.ps1 index d6dc5da..7129684 100644 --- a/examples/runTests.ps1 +++ b/examples/runTests.ps1 @@ -1,21 +1,42 @@ +rm -r -f ./log mkdir log mkdir log/lttng lttng create clogsample -o=log/lttng +if(!$?) +{ + Write-Host "LTTNG Create failed" + Exit +} lttng enable-event --userspace CLOG_* +if(!$?) +{ + Write-Host "LTTNG enable-event failed" + Exit +} -#lttng add-context --userspace --type=vpid --type=vtid +lttng add-context --userspace --type=vpid --type=vtid +if(!$?) +{ + Write-Host "LTTNG add-context failed" + Exit +} lttng start +if(!$?) +{ + Write-Host "LTTNG start failed" + Exit +} -cd ./build/clogsample $env:LD_PRELOAD="$PWD/libclogsampletracepointprovider.so" - ./clogsampledynamictp lttng list --userspace | grep CLOG ./clogsample lttng list --userspace | grep CLOG lttng stop clogsample -babeltrace --names all ./log/lttng/* > log/clog.babel lttng destroy clogsample -cd ../.. \ No newline at end of file + +babeltrace --names all ./log/lttng/* > log/clog.babel + +../buildclog/artifacts/clog2text_lttng -i ./log/clog.babel -s ../../clog.sidecar From 9d03b92e187e4c1435435d3fd6f68adf1659567f Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Tue, 19 Oct 2021 10:21:52 -0700 Subject: [PATCH 09/19] working on CI/CD --- examples/clogsample/simple.cpp | 8 +------- examples/runTests.ps1 | 3 +++ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/examples/clogsample/simple.cpp b/examples/clogsample/simple.cpp index 9d0c015..8b73ce9 100644 --- a/examples/clogsample/simple.cpp +++ b/examples/clogsample/simple.cpp @@ -13,12 +13,7 @@ int main(int argc, char* argv[]) { - for(int i=0; i<10; ++i) - { - if(i % 100) - printf("%d\n", i); - - //printf("Hello world!\n"); + printf("Hello world!\n"); TraceInfo(LAUNCHED, "Hello world - we just started here is an int=%d", 20); char *buffer = new char[10]; @@ -32,6 +27,5 @@ int main(int argc, char* argv[]) TraceInfo(DATACHAR, "This is a char: %c", 'a'); TraceInfo(DATAINT, "This is an int: %d", 1234); delete [] buffer; - } return 0; } diff --git a/examples/runTests.ps1 b/examples/runTests.ps1 index 7129684..41a159b 100644 --- a/examples/runTests.ps1 +++ b/examples/runTests.ps1 @@ -1,6 +1,9 @@ rm -r -f ./log mkdir log mkdir log/lttng + +docker + lttng create clogsample -o=log/lttng if(!$?) { From 6ca8d11ad685c673542563fe4bfe3df9c329dabb Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Tue, 19 Oct 2021 10:47:44 -0700 Subject: [PATCH 10/19] fix path --- .azure/azure-pipelines.ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/azure-pipelines.ci.yml b/.azure/azure-pipelines.ci.yml index 528f77e..2648d0f 100644 --- a/.azure/azure-pipelines.ci.yml +++ b/.azure/azure-pipelines.ci.yml @@ -106,7 +106,7 @@ stages: cmake ../examples cmake --build . - ../../examples/runTests.ps1 + ../examples/runTests.ps1 if ($IsLinux) { cd clogsample From ce944cc33cd520d6941ab167d1b339278106285f Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Tue, 19 Oct 2021 12:46:41 -0700 Subject: [PATCH 11/19] merge issues --- src/clogutils/CLogCustomTraceEmittorFactory.cs | 2 +- src/clogutils/CLogDecodedTraceLine.cs | 2 ++ src/clogutils/CLogHandledException.cs | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/clogutils/CLogCustomTraceEmittorFactory.cs b/src/clogutils/CLogCustomTraceEmittorFactory.cs index 3ac4a56..ebda76a 100644 --- a/src/clogutils/CLogCustomTraceEmittorFactory.cs +++ b/src/clogutils/CLogCustomTraceEmittorFactory.cs @@ -155,7 +155,7 @@ public bool Decode(CLogEncodingCLogTypeSearch type, IClogEventArg value, CLogLin customDecoder = customDecoder.Substring(0, customDecoder.Length - member.Length - 1); var newType = _codeAssembly.GetType(customDecoder); - var instance = _typesInterface = _codeAssembly.CreateInstance(customDecoder); + var instance = _typesInterface = _codeAssembly.CreateInstance(customDecoder); if (!_compiledConverterFunctions.ContainsKey(type.CustomDecoder)) { diff --git a/src/clogutils/CLogDecodedTraceLine.cs b/src/clogutils/CLogDecodedTraceLine.cs index f50b65d..b4e1734 100644 --- a/src/clogutils/CLogDecodedTraceLine.cs +++ b/src/clogutils/CLogDecodedTraceLine.cs @@ -99,6 +99,8 @@ public string macroName public CLogConfigurationProfile GetMacroConfigurationProfile() { + if(!macro.MacroConfiguration.ContainsKey(configFile.ProfileName)) + throw new CLogEnterReadOnlyModeException("MissingProfile:" + configFile.ProfileName + " in macro=" + macro.MacroName, CLogHandledException.ExceptionType.RequiredConfigParameterUnspecified, match); return configFile.MacroConfigurations[macro.MacroConfiguration[configFile.ProfileName]]; } diff --git a/src/clogutils/CLogHandledException.cs b/src/clogutils/CLogHandledException.cs index 0f97609..907bd3b 100644 --- a/src/clogutils/CLogHandledException.cs +++ b/src/clogutils/CLogHandledException.cs @@ -49,6 +49,7 @@ public enum ExceptionType EncodedArgNumberInvalid = 30, SidecarFileVersionMismatch = 31, SidecarCorrupted = 32 + ConfigFileMissingProfile = 33 } public static string TranslateExceptionTypeToErrorMessage(ExceptionType e) @@ -109,6 +110,8 @@ public static string TranslateExceptionTypeToErrorMessage(ExceptionType e) return "Must specify conversion from CLOG Type to Language Type in config file"; case ExceptionType.SidecarFileVersionMismatch: return "Invalid sidecar file version and unable to update - consider updating clog or correcting the version number"; + case ExceptionType.ConfigFileMissingProfile: + return "CLOG config file is missing the specified configuration; please update your clog_config file"; case ExceptionType.SidecarCorrupted: return "Sidecar cannot be opened; it seems to be corrupted - consider deleting and rebuilding, or locating the source of the corruption"; } From 23e7feefdc08c7b84d8305e376f3b2e6fa6991f1 Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Tue, 19 Oct 2021 19:54:25 -0400 Subject: [PATCH 12/19] Add ETW Support to Example (#78) --- examples/clog_examples.clog_config | 22 +++++++++++++++++----- examples/clog_examples.man | 29 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 examples/clog_examples.man diff --git a/examples/clog_examples.clog_config b/examples/clog_examples.clog_config index 5d0a420..603ecbf 100644 --- a/examples/clog_examples.clog_config +++ b/examples/clog_examples.clog_config @@ -1,6 +1,6 @@ { "MacroConfigurations": { - "lttng_plus": { + "lttng": { "Modules": [ { "ExportModule": "LTTNG", @@ -8,6 +8,18 @@ } ] }, + "etw": { + "Modules": [ + { + "ExportModule": "MANIFESTED_ETW", + "CustomSettings": { + "ETWManifestFile": "clog_examples.man", + "ETW_Provider": "6b6c1d19-db92-4787-81fd-70ac01e3ea7f", + "Level": "win:Informational" + } + } + ] + }, "empty": { "Modules": [] }, @@ -24,11 +36,11 @@ "EncodedPrefix": null, "EncodedArgNumber": 1, "MacroConfiguration": { - "linux": "lttng_plus", + "linux": "lttng", "macos": "empty", "stubs": "empty", "windows_kernel": "empty", - "windows": "empty" + "windows": "etw" } }, { @@ -36,11 +48,11 @@ "EncodedPrefix": "[%p]", "EncodedArgNumber": 2, "MacroConfiguration": { - "linux": "lttng_plus", + "linux": "lttng", "macos": "empty", "stubs": "empty", "windows_kernel": "empty", - "windows": "empty" + "windows": "etw" } } ], diff --git a/examples/clog_examples.man b/examples/clog_examples.man new file mode 100644 index 0000000..1385f62 --- /dev/null +++ b/examples/clog_examples.man @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + From d3f6c32eb47b56b97fb7fe79e745bb24979e1661 Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Wed, 20 Oct 2021 08:48:32 -0700 Subject: [PATCH 13/19] make error viewing a little easier on the eyes --- src/clog/CommandLineArguments.cs | 8 ++++++++ src/clog/clog.cs | 22 ++++++++++++++++++++++ src/clogutils/CLogHandledException.cs | 2 +- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/clog/CommandLineArguments.cs b/src/clog/CommandLineArguments.cs index e9f910b..9673f54 100644 --- a/src/clog/CommandLineArguments.cs +++ b/src/clog/CommandLineArguments.cs @@ -110,6 +110,14 @@ public bool DynamicTracepointProvider get; set; } + + [Option("verboseErrors", Required = false, Default = false, HelpText = "[OPTIONAL] Set this to see exceptions information, should you encouter a bug in CLOG")] + public bool VerboseErrors + { + get; + set; + } + public string GetOutputFileName(string inputFile) { string ret = Path.Combine(this.OutputDirectory, Path.GetFileName(inputFile)); diff --git a/src/clog/clog.cs b/src/clog/clog.cs index 4f3c8d5..bc12ccb 100644 --- a/src/clog/clog.cs +++ b/src/clog/clog.cs @@ -301,6 +301,28 @@ private static int Main(string[] args) e.PrintDiagnostics(); return -2; } + catch (Exception e) + { + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $"---------------------------------------------------------------------------"); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $"CLOG has crashed processing : {currentFile}."); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $" if you're reading this, we consider seeing this message a bug. Even if the message"); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $" you're about to read is sufficient to diagnose, we'd like to improve the user experience"); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $" please consider filing a bug, with repro files and --verboseErrors"); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $""); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $""); + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, $"Exception:"); + + if (!options.VerboseErrors) + { + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, " " + e.Message); + return -1; + } + else + { + CLogConsoleTrace.TraceLine(CLogConsoleTrace.TraceType.Err, e.ToString()); + } + + } return 0; }, err => diff --git a/src/clogutils/CLogHandledException.cs b/src/clogutils/CLogHandledException.cs index 907bd3b..7f4191c 100644 --- a/src/clogutils/CLogHandledException.cs +++ b/src/clogutils/CLogHandledException.cs @@ -48,7 +48,7 @@ public enum ExceptionType InvalidNameFormatInTypeSpcifier = 29, EncodedArgNumberInvalid = 30, SidecarFileVersionMismatch = 31, - SidecarCorrupted = 32 + SidecarCorrupted = 32, ConfigFileMissingProfile = 33 } From 8225348357cb3f583ec5e7eaf63ca1f4b300446a Mon Sep 17 00:00:00 2001 From: Chris Gray Date: Wed, 22 Dec 2021 22:46:39 -0700 Subject: [PATCH 14/19] merging cleanup - lots of bugs, clearifying error codes, and samples --- .azure/azure-pipelines.ci.yml | 169 -------------- CMakeLists.txt | 30 ++- clog.sln | 26 ++- defaults/defaults.clog_config | 5 + docs/EXAMPLES.md | 2 +- examples/clog.sidecar | 213 +++++++++++++++--- examples/clog_examples.clog_config | 45 +++- examples/clog_examples.man | 29 --- examples/clogsample/CMakeLists.txt | 27 ++- examples/clogsample/clog_examples.man | 168 ++++++++++++++ examples/clogsample/clog_examples.wprp | 42 ++++ examples/clogsample/simple.cpp | 57 ++++- examples/runTests.ps1 | 148 +++++++++--- examples/test.desired.output.txt | 8 + src/clog/CMakeLists.txt | 3 +- .../CLogLTTNGOutputModule.cs | 2 +- .../CLogManifestedETWOutputModule.cs | 155 ++++++++----- src/clog/TraceEmitterModules/CLogSTDOUT.cs | 186 ++------------- .../CLogSystemTapModule.cs | 151 ------------- .../CLogTraceLoggingOutputModule.cs | 8 - src/clog/clog.cs | 4 +- src/clog/clog.csproj | 2 +- src/clog2text/clog2text_lttng/CMakeLists.txt | 2 +- src/clog2text/clog2text_lttng/Program.cs | 9 +- .../clog2text_lttng/clog2text_lttng.csproj | 2 +- .../clog2text_windows/CMakeLists.txt | 2 +- src/clog2text/clog2text_windows/Program.cs | 19 +- .../clog2text_windows.csproj | 8 +- .../CLogCustomTraceEmittorFactory.cs | 24 +- src/clogutils/CLogDecodedTraceLine.cs | 6 +- src/clogutils/CLogEncodingCLogTypeSearch.cs | 2 +- src/clogutils/CLogEncodingType.cs | 2 +- src/clogutils/CLogFileProcessor.cs | 34 +-- src/clogutils/CLogFullyDecodedMacroEmitter.cs | 71 +++--- src/clogutils/CLogHandledException.cs | 26 ++- src/clogutils/CLogSidecar.cs | 2 +- .../ConfigFile/CLogConfigurationFile.cs | 11 +- .../CLogTraceMacroDefination.cs | 2 + src/converters/syslog2clog/syslog2clog.csproj | 2 +- src/decoderTest/decoderTest.csproj | 2 +- 40 files changed, 916 insertions(+), 790 deletions(-) delete mode 100644 .azure/azure-pipelines.ci.yml delete mode 100644 examples/clog_examples.man create mode 100644 examples/clogsample/clog_examples.man create mode 100644 examples/clogsample/clog_examples.wprp create mode 100644 examples/test.desired.output.txt delete mode 100644 src/clog/TraceEmitterModules/CLogSystemTapModule.cs diff --git a/.azure/azure-pipelines.ci.yml b/.azure/azure-pipelines.ci.yml deleted file mode 100644 index 2648d0f..0000000 --- a/.azure/azure-pipelines.ci.yml +++ /dev/null @@ -1,169 +0,0 @@ -# -# Continuous Integration (CI) Pipeline -# - -trigger: - batch: true - branches: - include: - - main - - release/* - - feature/* -pr: - branches: - include: - - main - - release/* - - feature/* - -variables: -- name: buildConfiguration - value: 'Release' -- name: runCodesignValidationInjection - value: false -- name: skipComponentGovernanceDetection - value: true - -name: 0.$(Date:yyyy).$(Date:MM).$(DayOfMonth).$(Rev:rr).0 - -stages: -- stage: build - displayName: Build - jobs: - - job: build_clog - displayName: Build CLOG - pool: - vmImage: ubuntu-latest - steps: - - task: UseDotNet@2 - displayName: 'Use .NET Core sdk' - inputs: - packageType: sdk - version: 5.x - installationPath: $(Agent.ToolsDirectory)/dotnet - - task: DotNetCoreCLI@2 - inputs: - command: build - projects: clog.sln - arguments: --configuration $(buildConfiguration) - - task: DotNetCoreCLI@2 - inputs: - command: custom - custom: tool - arguments: restore - displayName: Install dotnet-format - - script: dotnet format --check - displayName: Check formatting -- stage: examples - displayName: Examples - jobs: - - job: build_example - displayName: Build Example - strategy: - matrix: - # linux31: - # imageName: 'ubuntu-latest' - # dotnetVersion: 3.1.x - # mac31: - # imageName: 'macos-10.14' - # dotnetVersion: 3.1.x - # windows31: - # imageName: 'windows-latest' - # dotnetVersion: 3.1.x - linux5: - imageName: 'ubuntu-latest' - dotnetVersion: 5.x - mac5: - imageName: 'macos-10.14' - dotnetVersion: 5.x - windows5: - imageName: 'windows-latest' - dotnetVersion: 5.x - pool: - vmImage: $(imageName) - steps: - - task: UseDotNet@2 - condition: ne(variables.imageName, 'windows-latest') - displayName: 'Use .NET Core sdk' - inputs: - packageType: sdk - version: $(dotnetVersion) - installationPath: $(Agent.ToolsDirectory)/dotnet - - pwsh: | - if ($IsLinux) { - sudo apt-add-repository ppa:lttng/stable-2.11 - sudo apt-get update - sudo apt-get install -y liblttng-ust-dev - sudo apt-get install -y lttng-tools - } - - $BaseDir = Get-Location - $UploadPath = Join-Path $BaseDir "Logs" - mkdir $UploadPath - - mkdir build - cd build - cmake ../examples - cmake --build . - - ../examples/runTests.ps1 - - if ($IsLinux) { - cd clogsample - mkdir log - mkdir log/lttng - lttng create clogsample -o=log/lttng - lttng enable-event --userspace CLOG_* - lttng add-context --userspace --type=vpid --type=vtid - lttng start - - ./clogsample - - lttng stop clogsample - babeltrace --names all log/lttng/* > log/clog.babel - ../buildclog/artifacts/clog2text_lttng -i log/clog.babel -s ../../examples/clog.sidecar -o log/clog.log --showTimestamp --showCpuInfo - cat log/clog.log - - cp log/clog.babel "$UploadPath/clogstatictp.babel" - cp ../../examples/examples/sidecar/clog.sidecar "$UploadPath/clog.sidecar" - - rm -rf log/lttng - mkdir log/lttng - - Write-Host "Running with Dynamic Tracepoint Provider" - - lttng create clogsampledyn -o=log/lttng - lttng enable-event --userspace CLOG_* - lttng add-context --userspace --type=vpid --type=vtid - lttng start - - $env:LD_PRELOAD="$PWD/libclogsampletracepointprovider.so" - ./clogsampledynamictp - - $env:LD_PRELOAD="" - - lttng stop clogsampledyn - babeltrace --names all log/lttng/* > log/clog.babel - ../buildclog/artifacts/clog2text_lttng -i log/clog.babel -s ../../examples/clog.sidecar -o log/clog.log --showTimestamp --showCpuInfo - cat log/clog.log - - cp log/clog.babel "$UploadPath/clogdynamictp.babel" - cp ../../examples/examples/sidecar/clog.sidecar "$UploadPath/clog.sidecar" - - ls $UploadPath - - } - - displayName: Test Sample Build - - - task: CopyFiles@2 - displayName: Move Build Artifacts - inputs: - sourceFolder: Logs - targetFolder: $(Build.ArtifactStagingDirectory) - - task: PublishBuildArtifacts@1 - displayName: Upload Build Artifacts - inputs: - artifactName: logs - pathToPublish: $(Build.ArtifactStagingDirectory) - parallel: true diff --git a/CMakeLists.txt b/CMakeLists.txt index e90da60..96e5161 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,15 +11,18 @@ function(DOT_NET_BUILD) set(DEPENDENCY_NAME ${ARGV0}) set(DEST_BINARY ${CLOG_ARTIFACT_DIR}/${ARGV1}) set(PROJECT_PATH ${ARGV2}) + set(DOTNET_FRAMEWORK ${ARGV3}) + list(REMOVE_AT ARGV 0) list(REMOVE_AT ARGV 0) list(REMOVE_AT ARGV 0) list(REMOVE_AT ARGV 0) - #message(STATUS "---------------------- GENERATING : ${DEPENDENCY_NAME} -------------------") - #message(STATUS "PROJECT : ${PROJECT_PATH}") - #message(STATUS "PRODUCES : ${DEST_BINARY}") - #message(STATUS "DOTNET_RUNTIME : ${DOTNET_RUNTIME}") - #message(STATUS "OUTPUTDIR : ${CLOG_ARTIFACT_DIR}") + message(STATUS "---------------------- GENERATING : ${DEPENDENCY_NAME} -------------------") + message(STATUS "PROJECT : ${PROJECT_PATH}") + message(STATUS "PRODUCES : ${DEST_BINARY}") + message(STATUS "DOTNET_RUNTIME : ${DOTNET_RUNTIME}") + message(STATUS "OUTPUTDIR : ${CLOG_ARTIFACT_DIR}") + message(STATUS "DOTNET_FRAMEWORK : ${DOTNET_FRAMEWORK}") foreach(arg IN LISTS ARGV) #message(STATUS "FILE : ${CMAKE_CURRENT_BINARY_DIR}/${arg}") @@ -32,7 +35,7 @@ function(DOT_NET_BUILD) COMMENT "build CLOG, if this fails with nuget packages inspect your nuget.config and make sure nuget.org is present (add from command line using : dotnet nuget add source https://api.nuget.org/v3/index.json -n nuget.org)" COMMAND - dotnet publish ${PROJECT_PATH} -o ${CLOG_ARTIFACT_DIR} -f net5.0 + dotnet publish ${PROJECT_PATH} -o ${CLOG_ARTIFACT_DIR} -f ${DOTNET_FRAMEWORK} DEPENDS ${PROJECT_PATH} DEPENDS @@ -47,14 +50,15 @@ function(DOT_NET_LIBRARY_BUILD) set(DEPENDENCY_NAME ${ARGV0}) set(DEST_BINARY ${CLOG_ARTIFACT_DIR}/${DOTNET_RUNTIME}/${ARGV1}) set(PROJECT_PATH ${ARGV2}) + list(REMOVE_AT ARGV 0) list(REMOVE_AT ARGV 0) list(REMOVE_AT ARGV 0) - #message(STATUS "---------------------- GENERATING : ${DEPENDENCY_NAME} -------------------") - #message(STATUS "PROJECT : ${PROJECT_PATH}") - #message(STATUS "PRODUCES : ${DEST_BINARY}") - #message(STATUS "OUTPUTDIR : ${CLOG_ARTIFACT_DIR}/${DOTNET_RUNTIME}") + ##message(STATUS "---------------------- GENERATING : ${DEPENDENCY_NAME} -------------------") + ##message(STATUS "PROJECT : ${PROJECT_PATH}") + ##message(STATUS "PRODUCES : ${DEST_BINARY}") + ##message(STATUS "OUTPUTDIR : ${CLOG_ARTIFACT_DIR}/${DOTNET_RUNTIME}") foreach(arg IN LISTS ARGV) #message(STATUS "FILE : ${CMAKE_CURRENT_BINARY_DIR}/${arg}") @@ -81,6 +85,10 @@ add_subdirectory(src/clog) add_subdirectory(src/clogutils) if (NOT APPLE) - add_subdirectory(src/clog2text/clog2text_windows) add_subdirectory(src/clog2text/clog2text_lttng) + add_subdirectory(src/clog2text/clog2text_udiag) +endif() + +if (WIN32) + add_subdirectory(src/clog2text/clog2text_windows) endif() diff --git a/clog.sln b/clog.sln index e3a1800..6051582 100644 --- a/clog.sln +++ b/clog.sln @@ -7,7 +7,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clog", "src\clog\clog.cspro EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clogutils", "src\clogutils\clogutils.csproj", "{076A3AA5-DEB6-45AB-B834-5416178BAF44}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "clog2text", "src\clog2text", "{C05DAADF-AA11-4B93-A0B9-CCA2849A6A06}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "clog2text", "clog2text", "{C05DAADF-AA11-4B93-A0B9-CCA2849A6A06}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clog2text_lttng", "src\clog2text\clog2text_lttng\clog2text_lttng.csproj", "{B683F6AC-DEC3-4AFC-B156-77F303EE2986}" EndProject @@ -15,10 +15,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clog2text_windows", "src\cl EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "decoderTest", "src\decoderTest\decoderTest.csproj", "{A2079B4E-FF96-4DE5-80E8-8403A17C4347}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "converters", "src\converters", "{FE0B277E-B292-4FF2-8C69-15EECC1A3787}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "converters", "converters", "{FE0B277E-B292-4FF2-8C69-15EECC1A3787}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "syslog2clog", "src\converters\syslog2clog\syslog2clog.csproj", "{E1FA7999-5C45-46B1-9E83-5467264F928E}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "manifested2clog", "src\converters\manifested2clog\manifested2clog.csproj", "{A8C76D37-6C7C-4617-8A05-7469C07E294D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clog2text_udiag", "src\clog2text\clog2text_udiag\clog2text_udiag.csproj", "{A9D4DA06-7428-4639-BFD0-69D63CCD8F3E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -75,6 +79,22 @@ Global {E1FA7999-5C45-46B1-9E83-5467264F928E}.Release|Any CPU.Build.0 = Release|Any CPU {E1FA7999-5C45-46B1-9E83-5467264F928E}.Release|x64.ActiveCfg = Release|Any CPU {E1FA7999-5C45-46B1-9E83-5467264F928E}.Release|x64.Build.0 = Release|Any CPU + {A8C76D37-6C7C-4617-8A05-7469C07E294D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A8C76D37-6C7C-4617-8A05-7469C07E294D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A8C76D37-6C7C-4617-8A05-7469C07E294D}.Debug|x64.ActiveCfg = Debug|Any CPU + {A8C76D37-6C7C-4617-8A05-7469C07E294D}.Debug|x64.Build.0 = Debug|Any CPU + {A8C76D37-6C7C-4617-8A05-7469C07E294D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A8C76D37-6C7C-4617-8A05-7469C07E294D}.Release|Any CPU.Build.0 = Release|Any CPU + {A8C76D37-6C7C-4617-8A05-7469C07E294D}.Release|x64.ActiveCfg = Release|Any CPU + {A8C76D37-6C7C-4617-8A05-7469C07E294D}.Release|x64.Build.0 = Release|Any CPU + {A9D4DA06-7428-4639-BFD0-69D63CCD8F3E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A9D4DA06-7428-4639-BFD0-69D63CCD8F3E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A9D4DA06-7428-4639-BFD0-69D63CCD8F3E}.Debug|x64.ActiveCfg = Debug|Any CPU + {A9D4DA06-7428-4639-BFD0-69D63CCD8F3E}.Debug|x64.Build.0 = Debug|Any CPU + {A9D4DA06-7428-4639-BFD0-69D63CCD8F3E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A9D4DA06-7428-4639-BFD0-69D63CCD8F3E}.Release|Any CPU.Build.0 = Release|Any CPU + {A9D4DA06-7428-4639-BFD0-69D63CCD8F3E}.Release|x64.ActiveCfg = Release|Any CPU + {A9D4DA06-7428-4639-BFD0-69D63CCD8F3E}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -83,6 +103,8 @@ Global {B683F6AC-DEC3-4AFC-B156-77F303EE2986} = {C05DAADF-AA11-4B93-A0B9-CCA2849A6A06} {1ACE842F-1CA7-49E8-8E43-2C33EAF7E170} = {C05DAADF-AA11-4B93-A0B9-CCA2849A6A06} {E1FA7999-5C45-46B1-9E83-5467264F928E} = {FE0B277E-B292-4FF2-8C69-15EECC1A3787} + {A8C76D37-6C7C-4617-8A05-7469C07E294D} = {FE0B277E-B292-4FF2-8C69-15EECC1A3787} + {A9D4DA06-7428-4639-BFD0-69D63CCD8F3E} = {C05DAADF-AA11-4B93-A0B9-CCA2849A6A06} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {923EC79F-9700-466D-8668-106D7BDECF0D} diff --git a/defaults/defaults.clog_config b/defaults/defaults.clog_config index e9d28a8..f79f794 100644 --- a/defaults/defaults.clog_config +++ b/defaults/defaults.clog_config @@ -125,6 +125,11 @@ "CType": "unsigned short", "DefinationEncoding": "hx", }, + { + "EncodingType": "Int16", + "CType": "short", + "DefinationEncoding": "hd", + }, { "EncodingType": "UInt16", "CType": "unsigned short", diff --git a/docs/EXAMPLES.md b/docs/EXAMPLES.md index f4b4286..01a6bf0 100644 --- a/docs/EXAMPLES.md +++ b/docs/EXAMPLES.md @@ -8,6 +8,6 @@ cmake -G "Unix Makefiles" .. cmake -G "Visual Studio 16 2019" -A x64 .. dotnet publish ./src/clog/clog.csproj --self-contained -o /home/chgray/.dotnet/tools -f net5.0 -r win-x64 -dotnet publish ./src/clog/clog.csproj --self-contained -o /home/chgray/.dotnet/tools -f net5.0 -r linux-x64 +dotnet publish ./src/clog/clog.csproj --self-contained -o /home/chgray/.dotnet/tools -f net6.0 -r linux-x64 -p:PublishReadyToRun=true -p:PublishReadyToRunShowWarnings=true https://github.com/chgray/msquic/blob/53198319bd1460a04823eb6859b776b964f145f6/docs/BUILD.md \ No newline at end of file diff --git a/examples/clog.sidecar b/examples/clog.sidecar index d7e69af..d4fd9ef 100644 --- a/examples/clog.sidecar +++ b/examples/clog.sidecar @@ -1,21 +1,35 @@ { "Version": 2, "EventBundlesV2": { - "LAUNCHED": { - "ModuleProperites": {}, - "TraceString": "Hello world - we just started here is an int=%d", - "UniqueId": "LAUNCHED", + "DATA_STRING": { + "ModuleProperites": { + "MANIFESTED_ETW": { + "EventID": "4", + "ETWManifestFile": ".\\clogsample\\clog_examples.man", + "ETW_Provider": "7EBE92EB-B7AE-4720-B842-FA8798950838", + "Level": "win:Informational" + } + }, + "TraceString": "2. I am string %s=hello", + "UniqueId": "DATA_STRING", "splitArgs": [ { - "DefinationEncoding": "d", + "DefinationEncoding": "s", "MacroVariableName": "arg2" } ], "macroName": "TraceInfo" }, "DATABYTEARRAY": { - "ModuleProperites": {}, - "TraceString": "%!BYTEARRAY!", + "ModuleProperites": { + "MANIFESTED_ETW": { + "EventID": "5", + "ETWManifestFile": ".\\clogsample\\clog_examples.man", + "ETW_Provider": "7EBE92EB-B7AE-4720-B842-FA8798950838", + "Level": "win:Informational" + } + }, + "TraceString": "3. This is a byte array with a custom decoder = %!BYTEARRAY!", "UniqueId": "DATABYTEARRAY", "splitArgs": [ { @@ -26,8 +40,15 @@ "macroName": "TraceInfo" }, "DATACHAR": { - "ModuleProperites": {}, - "TraceString": "This is a char: %c", + "ModuleProperites": { + "MANIFESTED_ETW": { + "EventID": "6", + "ETWManifestFile": ".\\clogsample\\clog_examples.man", + "ETW_Provider": "7EBE92EB-B7AE-4720-B842-FA8798950838", + "Level": "win:Informational" + } + }, + "TraceString": "4. This is a char: %c; it should equal a", "UniqueId": "DATACHAR", "splitArgs": [ { @@ -38,8 +59,15 @@ "macroName": "TraceInfo" }, "DATAINT": { - "ModuleProperites": {}, - "TraceString": "This is an int: %d", + "ModuleProperites": { + "MANIFESTED_ETW": { + "EventID": "7", + "ETWManifestFile": ".\\clogsample\\clog_examples.man", + "ETW_Provider": "7EBE92EB-B7AE-4720-B842-FA8798950838", + "Level": "win:Informational" + } + }, + "TraceString": "5. This is an int: %d; it should be 1234", "UniqueId": "DATAINT", "splitArgs": [ { @@ -49,9 +77,35 @@ ], "macroName": "TraceInfo" }, + "DATAINT5": { + "ModuleProperites": { + "MANIFESTED_ETW": { + "EventID": "8", + "ETWManifestFile": ".\\clogsample\\clog_examples.man", + "ETW_Provider": "7EBE92EB-B7AE-4720-B842-FA8798950838", + "Level": "win:Informational" + } + }, + "TraceString": "6. This is an int: %d; it should be 1234", + "UniqueId": "DATAINT5", + "splitArgs": [ + { + "DefinationEncoding": "d", + "MacroVariableName": "arg2" + } + ], + "macroName": "TraceInfo" + }, "INSTANCE_TEST": { - "ModuleProperites": {}, - "TraceString": "[%p]1:%d 2:%s 3:%c 4:%u 5:%u 6:%u", + "ModuleProperites": { + "MANIFESTED_ETW": { + "EventID": "9", + "ETWManifestFile": ".\\clogsample\\clog_examples.man", + "ETW_Provider": "7EBE92EB-B7AE-4720-B842-FA8798950838", + "Level": "win:Informational" + } + }, + "TraceString": "[%p] 1. 1:%d 2:%s 3:%c 4:%u 5:%hd 6:%lld - you should see 1 2 3 4 5 6", "UniqueId": "INSTANCE_TEST", "splitArgs": [ { @@ -75,20 +129,48 @@ "MacroVariableName": "arg6" }, { - "DefinationEncoding": "u", + "DefinationEncoding": "hd", "MacroVariableName": "arg7" }, { - "DefinationEncoding": "u", + "DefinationEncoding": "lld", "MacroVariableName": "arg8" } ], "macroName": "TraceInstanceInfo" + }, + "INT_ERROR": { + "ModuleProperites": {}, + "TraceString": "7. this is an error %d", + "UniqueId": "INT_ERROR", + "splitArgs": [ + { + "DefinationEncoding": "d", + "MacroVariableName": "arg2" + } + ], + "macroName": "TraceError" + }, + "INT_ERROR_2": { + "ModuleProperites": {}, + "TraceString": "8. this is an error %d with a string %s", + "UniqueId": "INT_ERROR_2", + "splitArgs": [ + { + "DefinationEncoding": "d", + "MacroVariableName": "arg2" + }, + { + "DefinationEncoding": "s", + "MacroVariableName": "arg3" + } + ], + "macroName": "TraceError" } }, "ConfigFile": { "MacroConfigurations": { - "lttng_plus": { + "lttng": { "Modules": [ { "ExportModule": "LTTNG", @@ -96,6 +178,26 @@ } ] }, + "etw_info": { + "Modules": [ + { + "ExportModule": "MANIFESTED_ETW", + "CustomSettings": { + "ETWManifestFile": ".\\clogsample\\clog_examples.man", + "ETW_Provider": "7EBE92EB-B7AE-4720-B842-FA8798950838", + "Level": "win:Informational" + } + } + ] + }, + "etw_errors": { + "Modules": [ + { + "ExportModule": "TRACELOGGING", + "CustomSettings": {} + } + ] + }, "empty": { "Modules": [] } @@ -113,24 +215,50 @@ "EncodedPrefix": null, "EncodedArgNumber": 1, "MacroConfiguration": { - "linux": "lttng_plus", + "linux": "lttng", "macos": "empty", "stubs": "empty", "windows_kernel": "empty", - "windows": "empty" + "windows": "etw_info" }, "CustomSettings": null }, { "MacroName": "TraceInstanceInfo", - "EncodedPrefix": "[%p]", + "EncodedPrefix": "[%p] ", + "EncodedArgNumber": 2, + "MacroConfiguration": { + "linux": "lttng", + "macos": "empty", + "stubs": "empty", + "windows_kernel": "empty", + "windows": "etw_info" + }, + "CustomSettings": null + }, + { + "MacroName": "TraceError", + "EncodedPrefix": null, + "EncodedArgNumber": 1, + "MacroConfiguration": { + "linux": "lttng", + "macos": "empty", + "stubs": "empty", + "windows_kernel": "empty", + "windows": "etw_errors" + }, + "CustomSettings": null + }, + { + "MacroName": "TraceInstanceError", + "EncodedPrefix": "[%p] ", "EncodedArgNumber": 2, "MacroConfiguration": { - "linux": "lttng_plus", + "linux": "lttng", "macos": "empty", "stubs": "empty", "windows_kernel": "empty", - "windows": "empty" + "windows": "etw_errors" }, "CustomSettings": null } @@ -202,6 +330,12 @@ "JavaType": null, "DefinationEncoding": "hx" }, + { + "EncodingType": "Int16", + "CType": "short", + "JavaType": null, + "DefinationEncoding": "hd" + }, { "EncodingType": "UInt16", "CType": "unsigned short", @@ -401,29 +535,44 @@ "ModuleUniqueness": { "TraceInformation": [ { - "UniquenessHash": "1ba94e38-7860-c45a-131c-13e183be74c5", - "TraceID": "LAUNCHED", - "EncodingString": "Hello world - we just started here is an int=%d" + "UniquenessHash": "cd0737ad-4546-9d2c-6890-1132b9b30214", + "TraceID": "DATA_STRING", + "EncodingString": "2. I am string %s=hello" }, { - "UniquenessHash": "368386b1-58a0-bc3e-cf2b-bdcdb5f8a12e", + "UniquenessHash": "32847a97-a325-9319-c339-fe21e3428571", "TraceID": "DATABYTEARRAY", - "EncodingString": "%!BYTEARRAY!" + "EncodingString": "3. This is a byte array with a custom decoder = %!BYTEARRAY!" }, { - "UniquenessHash": "73633e68-7dc6-9310-40dd-35accdad6932", + "UniquenessHash": "231b0811-0852-d02c-e48d-01554d92d7d6", "TraceID": "DATACHAR", - "EncodingString": "This is a char: %c" + "EncodingString": "4. This is a char: %c; it should equal a" }, { - "UniquenessHash": "348407f0-f2f5-c1de-a153-4005cce7a5ce", + "UniquenessHash": "94b6f09a-1a05-c2da-82ea-8cca90a2fd12", "TraceID": "DATAINT", - "EncodingString": "This is an int: %d" + "EncodingString": "5. This is an int: %d; it should be 1234" }, { - "UniquenessHash": "6fe4b3c0-ad93-cd49-fc8b-e30c268668f4", + "UniquenessHash": "30926746-3fa4-1934-c3dc-5c25217666dd", + "TraceID": "DATAINT5", + "EncodingString": "6. This is an int: %d; it should be 1234" + }, + { + "UniquenessHash": "7c626bda-19d6-8240-ec6e-0271f541602b", "TraceID": "INSTANCE_TEST", - "EncodingString": "[%p]1:%d 2:%s 3:%c 4:%u 5:%u 6:%u" + "EncodingString": "[%p] 1. 1:%d 2:%s 3:%c 4:%u 5:%hd 6:%lld - you should see 1 2 3 4 5 6" + }, + { + "UniquenessHash": "0df07de0-3380-500a-02b5-85c45e74d3a3", + "TraceID": "INT_ERROR", + "EncodingString": "7. this is an error %d" + }, + { + "UniquenessHash": "7aff5a89-7dec-e219-e250-098606338503", + "TraceID": "INT_ERROR_2", + "EncodingString": "8. this is an error %d with a string %s" } ] } diff --git a/examples/clog_examples.clog_config b/examples/clog_examples.clog_config index 603ecbf..69a8251 100644 --- a/examples/clog_examples.clog_config +++ b/examples/clog_examples.clog_config @@ -8,18 +8,27 @@ } ] }, - "etw": { + "etw_info": { "Modules": [ { "ExportModule": "MANIFESTED_ETW", "CustomSettings": { - "ETWManifestFile": "clog_examples.man", - "ETW_Provider": "6b6c1d19-db92-4787-81fd-70ac01e3ea7f", + "ETWManifestFile": ".\\clogsample\\clog_examples.man", + "ETW_Provider": "7EBE92EB-B7AE-4720-B842-FA8798950838", "Level": "win:Informational" } } ] }, + "etw_errors": { + "Modules": [ + { + "ExportModule": "TRACELOGGING", + "CustomSettings": { + } + } + ] + }, "empty": { "Modules": [] }, @@ -40,19 +49,43 @@ "macos": "empty", "stubs": "empty", "windows_kernel": "empty", - "windows": "etw" + "windows": "etw_info" } }, { "MacroName": "TraceInstanceInfo", - "EncodedPrefix": "[%p]", + "EncodedPrefix": "[%p] ", + "EncodedArgNumber": 2, + "MacroConfiguration": { + "linux": "lttng", + "macos": "empty", + "stubs": "empty", + "windows_kernel": "empty", + "windows": "etw_info" + } + }, + { + "MacroName": "TraceError", + "EncodedPrefix": null, + "EncodedArgNumber": 1, + "MacroConfiguration": { + "linux": "lttng", + "macos": "empty", + "stubs": "empty", + "windows_kernel": "empty", + "windows": "etw_errors" + } + }, + { + "MacroName": "TraceInstanceError", + "EncodedPrefix": "[%p] ", "EncodedArgNumber": 2, "MacroConfiguration": { "linux": "lttng", "macos": "empty", "stubs": "empty", "windows_kernel": "empty", - "windows": "etw" + "windows": "etw_errors" } } ], diff --git a/examples/clog_examples.man b/examples/clog_examples.man deleted file mode 100644 index 1385f62..0000000 --- a/examples/clog_examples.man +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/examples/clogsample/CMakeLists.txt b/examples/clogsample/CMakeLists.txt index 2e1904c..7764ce2 100644 --- a/examples/clogsample/CMakeLists.txt +++ b/examples/clogsample/CMakeLists.txt @@ -13,12 +13,17 @@ set(SOURCES # as a normal cmake target dependency # CLOG_GENERATE_TARGET(CLOGSAMPLE_LIB STATIC ${SOURCES}) +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/inc) + +if (NOT WIN32) + add_executable(clogsample ${SOURCES}) +else() + add_executable(clogsample ${SOURCES} ${CMAKE_CURRENT_BINARY_DIR}/inc/clog_examples.rc) +endif() -add_executable(clogsample ${SOURCES}) target_link_libraries(clogsample CLOGSAMPLE_LIB) if (NOT WIN32 AND NOT APPLE) - include(FindLTTngUST) target_link_libraries(clogsample ${LTTNGUST_LIBRARIES}) @@ -29,5 +34,23 @@ if (NOT WIN32 AND NOT APPLE) add_library(clogsampletracepointprovider SHARED $) target_link_libraries(clogsampletracepointprovider ${LTTNGUST_LIBRARIES}) +endif() + + +if(WIN32) + message(STATUS "BUILDING CLOG SAMPLE(Windows)") + + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/inc/clog_examples.h + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/inc/clog_examples.rc + DEPENDS clog_examples.man + COMMAND mc.exe -um -h ${CMAKE_CURRENT_BINARY_DIR}/inc -r ${CMAKE_CURRENT_BINARY_DIR}/inc ${CMAKE_CURRENT_SOURCE_DIR}/clog_examples.man) + + set_property(SOURCE simple.cpp APPEND PROPERTY OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/inc/clog_examples.h) + target_include_directories(clogsample PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/inc) + add_definitions(-DCLOG_ETW) + + configure_file(clog_examples.wprp clog_examples.wprp COPYONLY) + configure_file(clog_examples.man clog_examples.man COPYONLY) endif() diff --git a/examples/clogsample/clog_examples.man b/examples/clogsample/clog_examples.man new file mode 100644 index 0000000..b12bcd5 --- /dev/null +++ b/examples/clogsample/clog_examples.man @@ -0,0 +1,168 @@ + + + + + +