From a05903df4dc2279235119962cc2d2b9d28cdb44e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominic=20H=C3=B6glinger?= Date: Sat, 25 Apr 2026 01:08:54 +0200 Subject: [PATCH 1/4] Add stricter semicolon grammar, require() function, fix multi-rule xn To easily spot semicolon errors, they are now mandatory delimiters after each construct. Add require() to raise an error if a match is unsucessful. Multiple rules in one xn files now possible. --- Parser/XnParser.cs | 32 ++++++++++++++++++++---------- Program.cs | 25 +++++++++++++++++++---- Transformation/ContextFunctions.cs | 13 ++++++++++++ Transformation/RequireException.cs | 15 ++++++++++++++ Transformation/Rule.cs | 16 +++++++++------ 5 files changed, 80 insertions(+), 21 deletions(-) create mode 100644 Transformation/RequireException.cs diff --git a/Parser/XnParser.cs b/Parser/XnParser.cs index f959107..57c282e 100644 --- a/Parser/XnParser.cs +++ b/Parser/XnParser.cs @@ -199,11 +199,13 @@ namespace XNeedle.Parser from ws1 in Parse.WhiteSpace.Many().Optional() from open in Parse.Char('{') from ws2 in Parse.WhiteSpace.Many().Optional() - from expressions in Parse.Ref(() => BangObject) - .Or(Parse.Ref(() => RuleObject)) - .Or(ExprPart(ctx).Select(e => (Part)e)) - .DelimitedBy(Parse.Char(';').Token()) - from tailingsemi in Parse.Char(';').Optional() + from expressions in ( + from expr in Parse.Ref(() => BangObject) + .Or(Parse.Ref(() => RuleObject)) + .Or(ExprPart(ctx).Select(e => (Part)e)) + from semi in Parse.Char(';').Token() + select expr + ).Many() from ws3 in Parse.WhiteSpace.Many().Optional() from close in Parse.Char('}') select new Bang{Name = new string([.. name]), Arguments = arguments, Body = expressions}; @@ -214,15 +216,23 @@ namespace XNeedle.Parser from ws1 in Parse.WhiteSpace.Many().Optional() from open in Parse.Char('{') from ws2 in Parse.WhiteSpace.Many().Optional() - from expressions in Parse.Ref(() => BangObject) - .Or(Parse.Ref(() => RuleObject)) - .Or(ExprPart(ctx).Select(e => (Part)e)) - .DelimitedBy(Parse.Char(';').Token()) - from tailingsemi in Parse.Char(';').Optional() + from expressions in ( + from expr in Parse.Ref(() => BangObject) + .Or(Parse.Ref(() => RuleObject)) + .Or(ExprPart(ctx).Select(e => (Part)e)) + from semi in Parse.Char(';').Token() + select expr + ).Many() from ws3 in Parse.WhiteSpace.Many().Optional() from close in Parse.Char('}') select new Rule{XPathSelector = selector, Body = expressions}; - + + public static readonly Parser> Rules = + (from rule in RuleObject.Token() + from semi in Parse.Char(';').Token() + select rule) + .Many() + .End(); static public string StripComments(string source) { diff --git a/Program.cs b/Program.cs index ae34d3e..36a84f6 100644 --- a/Program.cs +++ b/Program.cs @@ -40,10 +40,10 @@ namespace XNeedle var rawSource = File.ReadAllText(opts.XnDefinition); var xnSource = XnParser.StripComments(rawSource); - Rule masterRule; + IEnumerable rules; try { - masterRule = (Rule)XnParser.RuleObject.Parse(xnSource); + rules = XnParser.Rules.Parse(xnSource); } catch (ParseException ex) { @@ -54,11 +54,19 @@ namespace XNeedle if (showPosition) { - Console.Error.WriteLine($"{opts.XnDefinition}({line},{col}): parse error: {ex.Message}"); + var msg = ex.Message; + var hint = (msg.Contains("expected }") && !msg.Contains("expected ';'")) + ? "Hint: a missing ';' terminator may be located just after this position" + : ""; + Console.Error.WriteLine($"{opts.XnDefinition}({line},{col}): parse error: {msg}"); var srcLine = line <= lines.Length ? lines[line - 1].TrimEnd() : ""; var pointer = new string(' ', Math.Max(0, col - 1)) + "^"; Console.Error.WriteLine($" {srcLine}"); Console.Error.WriteLine($" {pointer}"); + if (!string.IsNullOrEmpty(hint)) + { + Console.Error.WriteLine($"\n{hint}"); + } } else { @@ -69,7 +77,16 @@ namespace XNeedle var doc = XDocument.Load(opts.Input, LoadOptions.PreserveWhitespace); var encoding = XmlRoundtripWriter.DetectEncoding(File.ReadAllBytes(opts.Input)); var sloc = new SourceLocation(opts.Input, encoding); - masterRule.Transform(doc, sloc); + try + { + foreach (var rule in rules.OfType()) + rule.Transform(doc, sloc); + } + catch (RequireException ex) + { + Console.Error.WriteLine($"{ex.SLoc.FilePath}({ex.SLoc.LineNumber}): require error: {ex.UserMessage}"); + return; + } using var writer = XmlRoundtripWriter.Create(opts.Input, opts.Output); doc.WriteTo(writer); Console.WriteLine($"Applied \"{opts.XnDefinition}\" to \"{opts.Input}\" and saved result to \"{opts.Output}\""); diff --git a/Transformation/ContextFunctions.cs b/Transformation/ContextFunctions.cs index 008b16f..8bf926d 100644 --- a/Transformation/ContextFunctions.cs +++ b/Transformation/ContextFunctions.cs @@ -187,5 +187,18 @@ namespace XNeedle.Transformation } return ""; } + + public static string require(Context ctx, string message = "Required node not present") + { + if (ctx.Node == null) + { + throw new RequireException{SLoc=ctx.SLoc, UserMessage=message}; + } + else + { + Console.WriteLine(ctx.Node.ToString()); + } + return ""; + } } } \ No newline at end of file diff --git a/Transformation/RequireException.cs b/Transformation/RequireException.cs new file mode 100644 index 0000000..64485c1 --- /dev/null +++ b/Transformation/RequireException.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; +using System.Xml; +using System.Xml.Linq; +using System.Xml.XPath; +using XNeedle; +using XNeedle.Parser.Elements; + +namespace XNeedle.Transformation +{ + public class RequireException : Exception + { + public SourceLocation SLoc { get; set; } + public required string UserMessage { get; set; } + } +} \ No newline at end of file diff --git a/Transformation/Rule.cs b/Transformation/Rule.cs index a51e897..7eabc60 100644 --- a/Transformation/Rule.cs +++ b/Transformation/Rule.cs @@ -22,15 +22,19 @@ namespace XNeedle.Transformation public override void Operate(Context ctx) { - if(ctx.Node is null) - { - return; - } - var nodes = ctx.Node.XPathSelectElements(XPathSelector); - if(nodes is null) + var nodes = ctx.Node?.XPathSelectElements(XPathSelector); + if(nodes is null || nodes.Count() == 0) { + var nullCtx = ctx.Clone(); + nullCtx.Node = null; + nullCtx.SLoc = ctx.SLoc.WithLineNumber(-1); + foreach (var b in Body) + { + b.Operate(nullCtx); + } return; } + foreach (XElement n in nodes) { var loopCtx = ctx.Clone(); -- 2.39.5 From 4d9ce95f6febf4700d21858d848c0c1b30933848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominic=20H=C3=B6glinger?= Date: Sat, 25 Apr 2026 11:57:15 +0200 Subject: [PATCH 2/4] Rename and parameterize crc function to crc32 --- Parser/XnParser.cs | 41 +++++++++++++++++-------- Transformation/ContextFunctions.cs | 48 +++++++++++++++++++++++++----- 2 files changed, 69 insertions(+), 20 deletions(-) diff --git a/Parser/XnParser.cs b/Parser/XnParser.cs index 57c282e..e226488 100644 --- a/Parser/XnParser.cs +++ b/Parser/XnParser.cs @@ -35,7 +35,7 @@ namespace XNeedle.Parser static Parser Function(Context ctx) { return - from name in Parse.Letter.AtLeastOnce().Text() + from name in Identifier from lparen in Parse.Char('(') from expr in Parse.Ref(() => Expr(ctx)).DelimitedBy(Parse.Char(',').Token()).Optional() from rparen in Parse.Char(')') @@ -46,23 +46,44 @@ namespace XNeedle.Parser { var mathMethodInfo = typeof(Math).GetTypeInfo().GetMethod(name, parameters.Select(e => e.Type).ToArray()); if (mathMethodInfo != null) - { return Expression.Call(mathMethodInfo, parameters); - } - var ctxMethodInfo = typeof(ContextFunctions).GetTypeInfo().GetMethod(name, parameters.Select(e => e.Type).Prepend(typeof(Context)).ToArray()); + + // Find by name and fill in optional parameters with their default values + var ctxMethodInfo = typeof(ContextFunctions).GetTypeInfo() + .GetMethods() + .FirstOrDefault(m => + { + if (m.Name != name) return false; + var mp = m.GetParameters(); + if (mp.Length == 0 || mp[0].ParameterType != typeof(Context)) return false; + int required = mp.Skip(1).Count(p => !p.HasDefaultValue); + return parameters.Length >= required && parameters.Length <= mp.Length - 1; + }); + if (ctxMethodInfo != null) { - return Expression.Call(ctxMethodInfo, parameters.Prepend(CtxParam).ToArray()); + var mp = ctxMethodInfo.GetParameters(); + var args = mp.Skip(1).Select((p, i) => + i < parameters.Length + ? parameters[i] + : Expression.Constant(p.DefaultValue, p.ParameterType) + ).ToArray(); + return Expression.Call(ctxMethodInfo, args.Prepend(CtxParam).ToArray()); } throw new ParseException(string.Format("Function '{0}({1})' does not exist.", name, string.Join(",", parameters.Select(e => e.Type.Name)))); } + public static readonly Parser HexNumber = + from prefix in Parse.String("0x").Token() + from number in Parse.Char(char.IsAsciiHexDigit, "hexadecimal digit").AtLeastOnce().Text() + select number; + static readonly Parser NumberConstant = - Parse.Decimal - .Select(x => Expression.Constant(double.Parse(x))) - .Named("number"); + HexNumber.Select(x => Expression.Constant(int.Parse(x, System.Globalization.NumberStyles.HexNumber))) + .XOr(Parse.Decimal.Select(x => Expression.Constant(double.Parse(x)))) + .Named("number"); private static readonly Parser StringConstant = from open in Parse.Char('"') @@ -183,10 +204,6 @@ namespace XNeedle.Parser from arguments in Argument.DelimitedBy(Parse.Char(',').Token()) from close in Parse.Char(')') select new Call{Identifier = new string(identifier.ToArray()), Arguments = arguments}; - - /*public static readonly Parser ExpressionObject = - from expression in CallObject - */ public static CommentParser Comment = new CommentParser { Single = "#", NewLine = Environment.NewLine }; diff --git a/Transformation/ContextFunctions.cs b/Transformation/ContextFunctions.cs index 8bf926d..e2aa770 100644 --- a/Transformation/ContextFunctions.cs +++ b/Transformation/ContextFunctions.cs @@ -26,7 +26,41 @@ namespace XNeedle.Transformation level++; return level; } - public static double crc(Context ctx) + + private static uint ComputeCrc32(byte[] data, uint poly, uint init, uint xorOut, bool reflected) + { + uint crc = init; + uint polyReflected = ReflectBits(poly, 32); + foreach (byte b in data) + { + if (reflected) + { + crc ^= b; + for (int i = 0; i < 8; i++) + crc = (crc & 1) != 0 ? (crc >> 1) ^ polyReflected : crc >> 1; + } + else + { + crc ^= (uint)b << 24; + for (int i = 0; i < 8; i++) + crc = (crc & 0x80000000u) != 0 ? (crc << 1) ^ poly : crc << 1; + } + } + return crc ^ xorOut; + } + + private static uint ReflectBits(uint value, int bits) + { + uint reflected = 0; + for (int i = 0; i < bits; i++) + { + if ((value & (1u << i)) != 0) + reflected |= 1u << (bits - 1 - i); + } + return reflected; + } + + public static double crc32(Context ctx, int poly = 0x04C11DB7, int init = 0, int xorOut = 0, bool reflected = false) { if (ctx.Node is null) return 0; @@ -36,17 +70,15 @@ namespace XNeedle.Transformation ctx.Node.WriteTo(writer); writer.Flush(); string xml = buffer.ToString(); - + // Step 2: Extract all text after the opening tag through and including the closing tag string tagName = ctx.Node.Name.LocalName; string data = XmlFragParser.ChecksumData(tagName).Parse(xml); + //byte[] fragBytes = ctx.SLoc.Encoding.GetBytes(data); + byte[] fragBytes = Encoding.UTF8.GetBytes(data); - // Step 3: Digest into CRC32 using the source encoding - var crc32 = new System.IO.Hashing.Crc32(); - crc32.Append(ctx.SLoc.Encoding.GetBytes(data)); - uint result = BitConverter.ToUInt32(crc32.GetCurrentHash()); - - return (double)result; + // Step 3: Digest into CRC32 using parameterized CRC32 - by default MPEG-2 + return (double)ComputeCrc32(fragBytes, (uint)poly, (uint)init, (uint)xorOut, reflected); } public static string text(Context ctx) -- 2.39.5 From af4239e198a841a2de4daf4c8ead53c0b2cb6bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominic=20H=C3=B6glinger?= Date: Sat, 25 Apr 2026 12:04:15 +0200 Subject: [PATCH 3/4] Rename hex() to hexlit(), use lowercase by default, add string case converstions --- Transformation/ContextFunctions.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Transformation/ContextFunctions.cs b/Transformation/ContextFunctions.cs index e2aa770..9920d43 100644 --- a/Transformation/ContextFunctions.cs +++ b/Transformation/ContextFunctions.cs @@ -12,10 +12,20 @@ namespace XNeedle.Transformation { public class ContextFunctions { - public static string hex(Context ctx, double value) + public static string hexlit(Context ctx, double value) { uint v = (uint)value; - return $"#x{v:X8}"; + return $"#x{v:x8}"; + } + + public static string lower(Context ctx, string text) + { + return text.ToLower(); + } + + public static string upper(Context ctx, string text) + { + return text.ToUpper(); } public static double GetLevel(Context ctx) -- 2.39.5 From 3ae964e033c2f27c38ce4381f828e8436ebda5ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominic=20H=C3=B6glinger?= Date: Sat, 25 Apr 2026 12:14:28 +0200 Subject: [PATCH 4/4] Add bitwise operations, rename power operator to "**" --- Parser/XnParser.cs | 9 ++++++--- Transformation/ContextFunctions.cs | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Parser/XnParser.cs b/Parser/XnParser.cs index e226488..18e58c3 100644 --- a/Parser/XnParser.cs +++ b/Parser/XnParser.cs @@ -30,8 +30,11 @@ namespace XNeedle.Parser static readonly Parser Multiply = Operator("*", ExpressionType.MultiplyChecked); static readonly Parser Divide = Operator("/", ExpressionType.Divide); static readonly Parser Modulo = Operator("%", ExpressionType.Modulo); - static readonly Parser Power = Operator("^", ExpressionType.Power); - + static readonly Parser Power = Operator("**", ExpressionType.Power); + static readonly Parser BitwiseXor = Operator("^", ExpressionType.ExclusiveOr); + static readonly Parser BitwiseAnd = Operator("&", ExpressionType.And); + static readonly Parser BitwiseOr = Operator("|", ExpressionType.Or); + static Parser Function(Context ctx) { return @@ -116,7 +119,7 @@ namespace XNeedle.Parser static Parser Term(Context ctx) { - return Parse.ChainOperator(Multiply.Or(Divide).Or(Modulo), InnerTerm(ctx), Expression.MakeBinary); + return Parse.ChainOperator(Multiply.Or(Divide).Or(Modulo).Or(BitwiseXor).Or(BitwiseAnd).Or(BitwiseOr), InnerTerm(ctx), Expression.MakeBinary); } static Parser ConcatExpr(Context ctx) diff --git a/Transformation/ContextFunctions.cs b/Transformation/ContextFunctions.cs index 9920d43..1580970 100644 --- a/Transformation/ContextFunctions.cs +++ b/Transformation/ContextFunctions.cs @@ -12,7 +12,7 @@ namespace XNeedle.Transformation { public class ContextFunctions { - public static string hexlit(Context ctx, double value) + public static string hexlit(Context ctx, int value) { uint v = (uint)value; return $"#x{v:x8}"; @@ -70,7 +70,7 @@ namespace XNeedle.Transformation return reflected; } - public static double crc32(Context ctx, int poly = 0x04C11DB7, int init = 0, int xorOut = 0, bool reflected = false) + public static int crc32(Context ctx, int poly = 0x04C11DB7, int init = 0, int xorOut = 0, bool reflected = false) { if (ctx.Node is null) return 0; @@ -88,7 +88,7 @@ namespace XNeedle.Transformation byte[] fragBytes = Encoding.UTF8.GetBytes(data); // Step 3: Digest into CRC32 using parameterized CRC32 - by default MPEG-2 - return (double)ComputeCrc32(fragBytes, (uint)poly, (uint)init, (uint)xorOut, reflected); + return (int)ComputeCrc32(fragBytes, (uint)poly, (uint)init, (uint)xorOut, reflected); } public static string text(Context ctx) -- 2.39.5