Merge pull request #2 from RomeCore/develop

Release v1.2.0

Author: RomeCore

src/LLTSharp/ITemplate.cs

@@ -1,13 +1,10 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
+using System.Collections.Generic;
 using LLTSharp.Metadata;
-using Microsoft.Extensions.AI;
 
 namespace LLTSharp
 {
 	/// <summary>
-	/// Interface for a template that can be used to generate prompts.
+	/// Interface for a template that can be used to generate contents.
 	/// </summary>
 	public interface ITemplate
 	{
@@ -25,7 +22,7 @@ public interface ITemplate
 	}
 
 	/// <summary>
-	/// Interface for a template that can be used to generate prompts.
+	/// Interface for a template that can be used to generate content.
 	/// </summary>
 	/// <typeparam name="TResult">The type of result produced by the template.</typeparam>
 	public interface ITemplate<TResult> : ITemplate
@@ -35,16 +32,16 @@ public interface ITemplate<TResult> : ITemplate
 	}
 
 	/// <summary>
-	/// Represents a prompt template that produces a string result.
+	/// Represents a text template that produces a string result.
 	/// </summary>
-	public interface IPromptTemplate : ITemplate<string>
+	public interface ITextTemplate : ITemplate<string>
 	{
 	}
 
 	/// <summary>
 	/// Represents a messages template that produces a collection of messages.
 	/// </summary>
-	public interface IMessagesTemplate : ITemplate<IEnumerable<ChatMessage>>
+	public interface IMessagesTemplate : ITemplate<IEnumerable<Message>>
 	{
 	}
 }

src/LLTSharp/LLTParser.cs

@@ -86,14 +86,14 @@ private static void DeclareValues(ParserBuilder builder)
 			builder.CreateRule("constant_array")
 				.Literal("[")
 				.ZeroOrMoreSeparated(b => b.Rule("constant"), b => b.Literal(","), allowTrailingSeparator: true)
-					.ConfigureLast(c => c.SkippingStrategy(ParserSkippingStrategy.SkipBeforeParsingGreedy))
+					.ConfigureLast(c => c.Skip(b => b.Rule("skip"), ParserSkippingStrategy.SkipBeforeParsingGreedy))
 				.Literal("]")
 				.Transform(v => new TemplateArrayAccessor(v.Children[1].SelectValues<TemplateDataAccessor>()));
 
 			builder.CreateRule("constant_object")
 				.Literal("{")
 				.ZeroOrMoreSeparated(b => b.Rule("constant_pair"), b => b.Literal(","), allowTrailingSeparator: true)
-					.ConfigureLast(c => c.SkippingStrategy(ParserSkippingStrategy.SkipBeforeParsingGreedy))
+					.ConfigureLast(c => c.Skip(b => b.Rule("skip"), ParserSkippingStrategy.SkipBeforeParsingGreedy))
 				.Literal("}")
 				.Transform(v =>
 				{
@@ -526,18 +526,39 @@ private static void DeclareTextTemplates(ParserBuilder builder)
 					node.Refine(depth: 1);
 
 					var library = v.GetParsingParameter<LLTParsingContext>().LocalLibrary;
-					var template = new PromptTemplate(node, new MetadataCollection(metadata), library);
+					var template = new TextTemplate(node, new MetadataCollection(metadata), library);
 					library.Add(template);
 					return template;
 				});
 
+			var textEscapes = new Dictionary<string, string>
+			{
+				["@@"] = "@",
+				["{{"] = "{",
+				["}}"] = "}"
+			};
+
+			var textForbidden = new string[]
+			{
+				"@", "{", "}", "`````"
+			};
+
 			builder.CreateRule("text_content")
-				.EscapedTextDoubleChars("@{}", allowsEmpty: false)
+				.EscapedText(textEscapes, textForbidden, allowsEmpty: false)
+				.Transform(v => new TextTemplatePlainTextNode(v.GetIntermediateValue<string>()));
+
+			builder.CreateRule("text_multiline_content")
+				.Token(b => b.Between(
+					b => b.Literal("`````"),
+					b => b.TextUntil("`````"),
+					b => b.Literal("`````")
+				))
 				.Transform(v => new TextTemplatePlainTextNode(v.GetIntermediateValue<string>()));
 
 			builder.CreateRule("text_statements")
 				.ZeroOrMore(b => b.Choice(
 					c => c.Rule("text_content"),
+					c => c.Rule("text_multiline_content"),
 					c => c.Rule("text_statement")))
 				.Transform(v =>
 				{
@@ -567,7 +588,7 @@ private static void DeclareTextTemplates(ParserBuilder builder)
 					)
 				.Transform(v => v.GetValue(1));
 
-			builder.CreateRule("text_expression_inner")
+			builder.CreateRule("text_expression")
 				.Rule("simple_expression") // We don't want to use binary expressions in text statements
 				.Optional(b => b
 					.Literal(':')
@@ -578,21 +599,6 @@ private static void DeclareTextTemplates(ParserBuilder builder)
 					return new TextTemplateExpressionNode(v.GetValue<TemplateExpressionNode>(0), format);
 				});
 
-			builder.CreateRule("text_expression")
-				.Custom(
-					(self, ctx, sett, childSett, children, childrenIds) =>
-					{
-						var result = self.ParseRule(childrenIds[0], ctx, childSett);
-						if (!result.success)
-							return result;
-						var modifierChild = result.children[1];
-						if (modifierChild.length == 0)
-							result.length = result.children[0].length;
-						return result;
-					},
-					b => b.Rule("text_expression_inner")
-				);
-
 			builder.CreateRule("text_if")
 				.Keyword("if")
 				.Rule("expression")
@@ -729,15 +735,18 @@ static LLTParser()
 		{
 			var builder = new ParserBuilder();
 
-			// Settings //
-			builder.Settings
-				.Skip(s => s.Choice(
+			// Skip rule //
+			builder.CreateRule("skip")
+				.Choice(
 					c => c.Whitespaces(),
 					c => c.Literal("@/").TextUntil('\n', '\r'), // @/ C#-like comments
 					c => c.Literal("@*").TextUntil("*@").Literal("*@")) // @*...*@ comments
-					.ConfigureForSkip(), // Ignore all errors when parsing comments and unnecessary whitespace
-					ParserSkippingStrategy.TryParseThenSkipLazy) // Allows rules to capture skip-rules contents if can, such as whitespaces
-				.UseCaching().RecordWalkTrace(); // If caching is disabled, prepare to wait for a long time (seconds) when encountering an error :P (you will also get a million of errors, seriously)
+					.ConfigureForSkip(); // Ignore all errors when parsing comments and unnecessary whitespace
+
+			// Settings //
+			builder.Settings
+				.Skip(b => b.Rule("skip"), ParserSkippingStrategy.TryParseThenSkipLazy) // Allows rules to capture skip-rules contents if can, such as whitespaces
+				.UseCaching(); // If caching is disabled, prepare to wait for a long time (seconds) when encountering an error :P (you will also get a million of errors, seriously)
 
 			// ---- Values ---- //
 			DeclareValues(builder);

src/LLTSharp/LLTSharp.csproj

@@ -8,11 +8,11 @@
 
 	<PropertyGroup>
 		<PackageId>LLTSharp</PackageId>
-		<Version>1.1.0</Version>
+		<Version>1.2.0</Version>
 		<Authors>Roman K.</Authors>
 		<Company>RomeCore</Company>
 		<Product>LLTSharp</Product>
-		<Description>A lightweight .NET template engine for LLM with messages and prompt templates support.</Description>
+		<Description>A lightweight .NET template engine for LLM with messages and text templates support.</Description>
 		<PackageReadmeFile>README.md</PackageReadmeFile>
 		<PackageTags>template, llm, template-engine, language-models</PackageTags>
 		<RepositoryUrl>https://github.com/RomeCore/LLTSharp</RepositoryUrl>
@@ -25,8 +25,7 @@
 	</ItemGroup>
 
 	<ItemGroup>
-		<PackageReference Include="Microsoft.Extensions.AI.Abstractions" Version="9.10.0" />
-		<PackageReference Include="RCParsing" Version="4.7.1" />
+		<PackageReference Include="RCParsing" Version="5.1.0" />
 	</ItemGroup>
 
 </Project>

src/LLTSharp/Message.cs

@@ -0,0 +1,68 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace LLTSharp
+{
+	/// <summary>
+	/// Represents a role in a conversation.
+	/// </summary>
+	public enum Role
+	{
+		/// <summary>
+		/// System role, typically used for system instructions or context.
+		/// </summary>
+		System,
+
+		/// <summary>
+		/// User role, typically used for user input or questions.
+		/// </summary>
+		User,
+
+		/// <summary>
+		/// Assistant role, typically used for responses or actions.
+		/// </summary>
+		Assistant,
+
+		/// <summary>
+		/// Tool role, typically used for responses from tool calls.
+		/// </summary>
+		Tool
+	}
+
+	/// <summary>
+	/// Represents a single message in a conversation with LLM.
+	/// </summary>
+	public class Message
+	{
+		/// <summary>
+		/// Gets or sets the role of the message sender. Typically "user" or "assistant".
+		/// </summary>
+		public Role Role { get; }
+
+		/// <summary>
+		/// Gets or sets the content of the message.
+		/// </summary>
+		public string Content { get; }
+
+		/// <summary>
+		/// Initializes a new instance of the <see cref="Message"/> class.
+		/// </summary>
+		public Message()
+		{
+			Role = Role.System;
+			Content = string.Empty;
+		}
+
+		/// <summary>
+		/// Initializes a new instance of the <see cref="Message"/> class.
+		/// </summary>
+		/// <param name="role">The role of the message sender.</param>
+		/// <param name="content">The content of the message.</param>
+		public Message(Role role, string content)
+		{
+			Role = role;
+			Content = content;
+		}
+	}
+}

src/LLTSharp/MessagesTemplate.cs

@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using System.Text;
 using LLTSharp.Metadata;
-using Microsoft.Extensions.AI;
 
 namespace LLTSharp
 {
@@ -39,7 +38,7 @@ public MessagesTemplate(MessagesTemplateNode mainNode, IMetadataCollection metad
 		/// </summary>
 		/// <param name="context">The context accessor to use for rendering.</param>
 		/// <returns>The rendered prompt as a collection of messages.</returns>
-		public IEnumerable<ChatMessage> Render(object? context = null)
+		public IEnumerable<Message> Render(object? context = null)
 		{
 			var ctx = new TemplateContextAccessor(TemplateDataAccessor.Create(context), Metadata, library: LocalLibrary);
 			return _node.Render(ctx);

src/LLTSharp/MessagesTemplateNode.cs

@@ -1,7 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Text;
-using Microsoft.Extensions.AI;
 
 namespace LLTSharp
 {
@@ -15,7 +13,7 @@ public abstract class MessagesTemplateNode
 		/// </summary>
 		/// <param name="context">The context accessor containing the data to render.</param>
 		/// <returns>A collection of messages representing the rendered template node.</returns>
-		public abstract IEnumerable<ChatMessage> Render(TemplateContextAccessor context);
+		public abstract IEnumerable<Message> Render(TemplateContextAccessor context);
 
 		/// <summary>
 		/// Refines the template after parsing an AST to remove indents and unnecessary leading/trailing whitespaces.

src/LLTSharp/TemplateContextAccessor.cs

@@ -5,7 +5,6 @@
 using System.Text;
 using LLTSharp.DataAccessors;
 using LLTSharp.Metadata;
-using Microsoft.Extensions.AI;
 
 namespace LLTSharp
 {
@@ -223,7 +222,7 @@ public string RenderTemplate(string identifier, TemplateDataAccessor? newContext
 			if (template == null)
 				throw new TemplateRuntimeException($"Template '{identifier}' not found.");
 
-			if (template is not PromptTemplate && template is not PlaintextTemplate)
+			if (template is not TextTemplate && template is not PlaintextTemplate)
 				throw new TemplateRuntimeException($"Template '{identifier}' is not a text template.");
 
 			var context = newContext ?? this;
@@ -239,7 +238,7 @@ public string RenderTemplate(string identifier, TemplateDataAccessor? newContext
 		/// <param name="identifier">The identifier of the messages template to render.</param>
 		/// <param name="newContext">The new context to use for rendering the template. If null, uses the current context.</param>
 		/// <returns>The rendered template as a collection of messages.</returns>
-		public IEnumerable<ChatMessage> RenderMessagesTemplate(string identifier, TemplateDataAccessor? newContext)
+		public IEnumerable<Message> RenderMessagesTemplate(string identifier, TemplateDataAccessor? newContext)
 		{
 			var template = Library.TryRetrieve(identifier);
 			if (template == null)

src/LLTSharp/TemplateNodes/MessagesTemplateEntryNode.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
-using Microsoft.Extensions.AI;
 
 namespace LLTSharp.TemplateNodes
 {
@@ -32,21 +31,21 @@ public MessagesTemplateEntryNode(TemplateExpressionNode role, TextTemplateNode c
 			Child = child ?? throw new ArgumentNullException(nameof(child));
 		}
 
-		public override IEnumerable<ChatMessage> Render(TemplateContextAccessor context)
+		public override IEnumerable<Message> Render(TemplateContextAccessor context)
 		{
 			var role = Role.Evaluate(context);
 			var content = Child.Render(context);
 
-			ChatMessage message = role.ToString() switch
+			Message message = role.ToString() switch
 			{
-				"system" => new ChatMessage(ChatRole.System, content),
-				"user" => new ChatMessage(ChatRole.User, content),
-				"assistant" => new ChatMessage(ChatRole.Assistant, content),
+				"system" => new Message(LLTSharp.Role.System, content),
+				"user" => new Message(LLTSharp.Role.User, content),
+				"assistant" => new Message(LLTSharp.Role.Assistant, content),
 				// "tool" => new ToolMessage(content), // TODO: Add support for tool call ids and tool names
 				"tool" => throw new TemplateRuntimeException($"Tool messages are not yet supported.", dataAccessor: context, messagesTemplateNode: this),
 				_ => throw new TemplateRuntimeException($"Invalid role '{role}'.", dataAccessor: context, messagesTemplateNode: this),
 			};
-			return new ChatMessage[] { message };
+			return new Message[] { message };
 		}
 
 		public override void Refine(int depth)

src/LLTSharp/TemplateNodes/MessagesTemplateForeachNode.cs

@@ -1,8 +1,6 @@
 using System;
 using System.Collections.Generic;
-using System.Text;
 using LLTSharp.DataAccessors;
-using Microsoft.Extensions.AI;
 
 namespace LLTSharp.TemplateNodes
 {
@@ -41,15 +39,15 @@ public MessagesTemplateForeachNode(TemplateExpressionNode source, MessagesTempla
 			IterableName = string.IsNullOrEmpty(iterableName) ? throw new ArgumentException("The iterable name cannot be null or empty.", nameof(iterableName)) : iterableName;
 		}
 
-		public override IEnumerable<ChatMessage> Render(TemplateContextAccessor context)
+		public override IEnumerable<Message> Render(TemplateContextAccessor context)
 		{
 			var source = Source.Evaluate(context);
 
 			if (source is not IEnumerableTemplateDataAccessor enumerableSource)
 				throw new TemplateRuntimeException($"The source expression does not provide an enumerable data source.",
 					dataAccessor: context, expressionNode: Source);
 
-			List<ChatMessage> messages = new List<ChatMessage>();
+			List<Message> messages = new List<Message>();
 
 			context.PushFrame();
 			foreach (var item in enumerableSource)