Implement performance optimizations and caching improvements

build/_build.csproj

@@ -10,18 +10,20 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="chocolatey" Version="2.5.1" />
-    <PackageReference Include="GitVersion.Core" Version="6.5.1" />
-    <PackageReference Include="Microsoft.Build" Version="18.0.2" />
-    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="18.0.2" />
-    <PackageReference Include="NuGet.CommandLine" Version="7.0.3">
+    <PackageReference Include="chocolatey" Version="2.7.1" />
+    <PackageReference Include="GitVersion.Core" Version="6.7.0" />
+    <PackageReference Include="Microsoft.Build" Version="18.4.0" />
+    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="18.4.0" />
+    <PackageReference Include="NuGet.CommandLine" Version="7.3.1">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="NuGet.Versioning" Version="7.0.1" />
-    <PackageReference Include="Nuke.Common" Version="10.0.0" />
+    <PackageReference Include="NuGet.Packaging" Version="7.3.1" />
+    <PackageReference Include="NuGet.Versioning" Version="7.3.1" />
+    <PackageReference Include="Nuke.Common" Version="10.1.0" />
     <PackageReference Include="Nuke.GitHub" Version="7.0.0" />
-    <PackageReference Include="NUnit.ConsoleRunner" Version="3.20.2" />
+    <PackageReference Include="NUnit.ConsoleRunner" Version="3.22.0" />
+    <PackageReference Include="System.Security.Cryptography.Xml" Version="10.0.6" />
   </ItemGroup>
 
   <ItemGroup>

global.json

@@ -1,6 +1,6 @@
 {
   "sdk": {
-    "version": "10.0.200",
+    "version": "10.0.100",
     "rollForward": "latestPatch"
   }
 }

src/ColumnizerLib/LogLine.cs

@@ -4,30 +4,19 @@ namespace ColumnizerLib;
 /// Represents a single log line, including its full text and line number.
 /// </summary>
 /// <remarks>
-/// <para>
-/// <b>Purpose:</b> <br/>
-/// The <c>LogLine</c> struct encapsulates the content and line number of a log entry. It is used throughout the
-/// columnizer and log processing infrastructure to provide a strongly-typed, immutable representation of a log line.
-/// </para>
-/// <para>
-/// <b>Usage:</b> <br/>
-/// This struct implements the <see cref="ILogLineMemory"/> interface, allowing it to be used wherever an <c>ILogLineMemory</c>
-/// is expected. It provides value semantics and is intended to be lightweight and efficiently passed by value.
-/// </para>
-/// <para>
-/// <b>Relationship to ILogLineMemory:</b> <br/>
-/// <c>LogLine</c> is a concrete, immutable implementation of the <see cref="ILogLineMemory"/> interface, providing
-/// properties for the full line text and its line number.
-/// </para>
-/// <para>
-/// <b>Why struct instead of record:</b> <br/>
-/// A <c>struct</c> is preferred over a <c>record</c> here to avoid heap allocations and to provide value-type
-/// semantics, which are beneficial for performance when processing large numbers of log lines. The struct is
-/// immutable (readonly), ensuring thread safety and predictability. The previous <c>record</c> implementation
-/// was replaced to better align with these performance and semantic requirements.
-/// </para>
+/// <para> <b> Purpose:</b> <br/> The <c> LogLine</c> struct encapsulates the content and line number of a log entry. It
+/// is used throughout the columnizer and log processing infrastructure to provide a strongly-typed, immutable
+/// representation of a log line. </para>
+/// <para> <b> Usage:</b> <br/> This struct implements the
+/// <see cref="ILogLineMemory"/> interface, allowing it to be used wherever an <c> ILogLineMemory</c> is expected. It
+/// provides value semantics and is intended to be lightweight and efficiently passed by value. </para> <para> <b>
+/// Relationship to ILogLineMemory:</b> <br/> <c> LogLine</c> is a concrete, immutable implementation of the
+/// <see cref="ILogLineMemory"/> interface, providing properties for the full line text and its line number. </para>
+/// This is a readonly record struct implementing
+/// <see cref="ILogLineMemory"/>. Stored inline in <c> List&lt;LogLine&gt;</c> to avoid boxing and heap allocation.
+/// Boxing occurs only when returned through the <c> ILogLineMemory</c> interface boundary.
 /// </remarks>
-public class LogLine : ILogLineMemory
+public readonly record struct LogLine : ILogLineMemory
 {
     public int LineNumber { get; }
 

src/Directory.Build.props

@@ -1,9 +1,9 @@
 <Project>
   <PropertyGroup>
-    <Version>1.31.0.0</Version>
-    <AssemblyVersion>1.31.0.0</AssemblyVersion>
-    <FileVersion>1.31.0.0</FileVersion>
-    <InformationalVersion>1.31.0.0</InformationalVersion>
+    <Version>1.40.0.0</Version>
+    <AssemblyVersion>1.40.0.0</AssemblyVersion>
+    <FileVersion>1.40.0.0</FileVersion>
+    <InformationalVersion>1.40.0.0</InformationalVersion>
     <Authors>Hirogen, zarunbal, RandallFlagg, TheNicker</Authors>
     <Company>LogExperts</Company>
     <ImplicitUsings>enable</ImplicitUsings>
@@ -21,8 +21,8 @@
     <RepositoryUrl>https://github.com/LogExperts/LogExpert</RepositoryUrl>
     <PackageTags>LogExpert, Columnizer, Logging, Windows, Winforms</PackageTags>
     <RepositoryType>git</RepositoryType>
-    <PackageReleaseNotes>https://github.com/LogExperts/LogExpert/releases/tag/v.1.31.0</PackageReleaseNotes>
-    <PackageVersion>1.31.0.0</PackageVersion>
+    <PackageReleaseNotes>https://github.com/LogExperts/LogExpert/releases/tag/v.1.40.0</PackageReleaseNotes>
+    <PackageVersion>1.40.0.0</PackageVersion>
     <Optimize Condition="'$(Configuration)' == 'Release'">true</Optimize>
     <Product>LogExpert</Product>
     <Copyright>Copyright © LogExpert 2025</Copyright>

src/Directory.Packages.props

@@ -10,24 +10,24 @@
     <PackageVersion Include="CsvHelper" Version="33.1.0" />
     <PackageVersion Include="DockPanelSuite.ThemeVS2015" Version="3.1.1" />
     <PackageVersion Include="GitVersion.Core" Version="6.3.0" />
-    <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
+    <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.4.0" />
     <PackageVersion Include="Moq" Version="4.20.72" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.4" />
-    <PackageVersion Include="NLog" Version="6.0.6" />
+    <PackageVersion Include="NLog" Version="6.1.2" />
     <PackageVersion Include="NuGet.CommandLine" Version="6.14.0" />
-    <PackageVersion Include="NuGet.Versioning" Version="7.0.1" />
+    <PackageVersion Include="NuGet.Versioning" Version="7.3.1" />
     <PackageVersion Include="Nuke.Common" Version="9.0.4" />
     <PackageVersion Include="Nuke.GitHub" Version="7.0.0" />
-    <PackageVersion Include="NUnit" Version="4.4.0" />
+    <PackageVersion Include="NUnit" Version="4.5.1" />
     <PackageVersion Include="NUnit.ConsoleRunner" Version="3.20.1" />
-    <PackageVersion Include="NUnit3TestAdapter" Version="5.2.0" />
+    <PackageVersion Include="NUnit3TestAdapter" Version="6.2.0" />
     <PackageVersion Include="SSH.NET" Version="2025.1.0" />
-    <PackageVersion Include="System.Drawing.Common" Version="10.0.0" />
-    <PackageVersion Include="System.Resources.Extensions" Version="10.0.0" />
-    <PackageVersion Include="Vanara.Library" Version="4.2.1" />
-    <PackageVersion Include="Vanara.PInvoke.DwmApi" Version="4.2.1" />
-    <PackageVersion Include="Vanara.PInvoke.RstrtMgr" Version="4.2.1" />
-    <PackageVersion Include="Vanara.PInvoke.Shell32" Version="4.2.1" />
-    <PackageVersion Include="Vanara.PInvoke.User32" Version="4.2.1" />
+    <PackageVersion Include="System.Drawing.Common" Version="10.0.6" />
+    <PackageVersion Include="System.Resources.Extensions" Version="10.0.6" />
+    <PackageVersion Include="Vanara.Library" Version="5.0.4" />
+    <PackageVersion Include="Vanara.PInvoke.DwmApi" Version="5.0.4" />
+    <PackageVersion Include="Vanara.PInvoke.RstrtMgr" Version="5.0.4" />
+    <PackageVersion Include="Vanara.PInvoke.Shell32" Version="5.0.4" />
+    <PackageVersion Include="Vanara.PInvoke.User32" Version="5.0.4" />
   </ItemGroup>
 </Project>

src/LogExpert.Core/Classes/Log/LineOffsetIndex.cs

@@ -0,0 +1,53 @@
+namespace LogExpert.Core.Classes.Log;
+
+/// <summary>
+/// Stores byte offsets for each line start in a file. Supports incremental appending for tail mode.
+/// </summary>
+internal sealed class LineOffsetIndex (int initialCapacity = 4096)
+{
+    private long[] _offsets = new long[initialCapacity];
+
+    public int LineCount { get; private set; }
+
+    /// <summary>
+    /// Appends a line-start offset.
+    /// </summary>
+    public void Add (long offset)
+    {
+        if (LineCount == _offsets.Length)
+        {
+            Array.Resize(ref _offsets, _offsets.Length * 2);
+        }
+
+        _offsets[LineCount++] = offset;
+    }
+
+    /// <summary>
+    /// Returns the byte offset of the start of the given line.
+    /// </summary>
+    public long GetOffset (int lineNum)
+    {
+        return (uint)lineNum < (uint)LineCount ? _offsets[lineNum] : -1;
+    }
+
+    /// <summary>
+    /// Returns the byte length of the given line (from its start to the next line's start).
+    /// For the last line, returns -1 (unknown length, read to end or newline).
+    /// </summary>
+    public long GetLineLength (int lineNum)
+    {
+        return (uint)lineNum >= (uint)LineCount
+            ? -1
+            : lineNum + 1 < LineCount
+                ? _offsets[lineNum + 1] - _offsets[lineNum]
+                : -1;
+    }
+
+    /// <summary>
+    /// Removes all offsets, resetting the index.
+    /// </summary>
+    public void Clear ()
+    {
+        LineCount = 0;
+    }
+}

src/LogExpert.Core/Classes/Log/LogBuffer.cs

@@ -1,3 +1,5 @@
+using System.Buffers;
+
 using ColumnizerLib;
 
 using NLog;
@@ -8,29 +10,33 @@ public class LogBuffer
 {
     #region Fields
 
+    private SpinLock _contentLock = new(enableThreadOwnerTracking: false);
     private static readonly Logger _logger = LogManager.GetCurrentClassLogger();
 
 #if DEBUG
-    private readonly IList<long> _filePositions = []; // file position for every line
+    private readonly List<long> _filePositions; // file position for every line
 #endif
 
-    private readonly List<ILogLineMemory> _lineList = [];
+    private LogLine[] _lineArray;
+    private int _lineArrayLength; // capacity of the rented array
 
     private int MAX_LINES = 500;
-    private long _size;
 
     #endregion
 
     #region cTor
 
-    //public LogBuffer() { }
-
     // Don't use a primary constructor here: field initializers (like MAX_LINES) run before primary constructor parameters are assigned,
     // so MAX_LINES would always be set to its default value before the constructor body can assign it. Use a regular constructor instead.
     public LogBuffer (ILogFileInfo fileInfo, int maxLines)
     {
         FileInfo = fileInfo;
         MAX_LINES = maxLines;
+        _lineArray = ArrayPool<LogLine>.Shared.Rent(maxLines);
+        _lineArrayLength = _lineArray.Length;
+#if DEBUG
+        _filePositions = new(MAX_LINES);
+#endif
     }
 
     #endregion
@@ -43,18 +49,18 @@ public long Size
     {
         set
         {
-            _size = value;
+            field = value;
 #if DEBUG
             if (_filePositions.Count > 0)
             {
-                if (_size < _filePositions[_filePositions.Count - 1] - StartPos)
+                if (field < _filePositions[^1] - StartPos)
                 {
                     _logger.Error("LogBuffer overall Size must be greater than last line file position!");
                 }
             }
 #endif
         }
-        get => _size;
+        get;
     }
 
     public int EndLine => StartLine + LineCount;
@@ -75,36 +81,91 @@ public long Size
 
     #region Public methods
 
-    public void AddLine (ILogLineMemory lineMemory, long filePos)
+    public void AddLine (LogLine lineMemory, long filePos)
     {
-        _lineList.Add(lineMemory);
+        if (LineCount < _lineArrayLength)
+        {
+            _lineArray[LineCount] = lineMemory;
+            LineCount++;
+        }
+#if DEBUG
+        else
+        {
+            _logger.Error("AddLine overflow: LineCount={0} >= _lineArrayLength={1}", LineCount, _lineArrayLength);
+        }
+#endif
+
 #if DEBUG
         _filePositions.Add(filePos);
 #endif
-        LineCount++;
         IsDisposed = false;
     }
 
     public void ClearLines ()
     {
-        _lineList.Clear();
+        Array.Clear(_lineArray, 0, LineCount);
         LineCount = 0;
     }
 
+    /// <summary>
+    /// Prepares the buffer for reuse from the pool.
+    /// </summary>
+    public void Reinitialise (ILogFileInfo fileInfo, int maxLines)
+    {
+        FileInfo = fileInfo;
+        MAX_LINES = maxLines;
+        StartLine = 0;
+        StartPos = 0;
+        Size = 0;
+        LineCount = 0;
+        DroppedLinesCount = 0;
+        PrevBuffersDroppedLinesSum = 0;
+        IsDisposed = false;
+        _lineArray = ArrayPool<LogLine>.Shared.Rent(maxLines);
+        _lineArrayLength = _lineArray.Length;
+#if DEBUG
+        _filePositions.Clear();
+        DisposeCount = 0;
+#endif
+    }
+
     public void DisposeContent ()
     {
-        _lineList.Clear();
+        if (_lineArray != null)
+        {
+            Array.Clear(_lineArray, 0, LineCount);
+            ArrayPool<LogLine>.Shared.Return(_lineArray);
+            _lineArray = null;
+            LineCount = 0;
+        }
+
         IsDisposed = true;
 #if DEBUG
         DisposeCount++;
 #endif
     }
 
-    public ILogLineMemory GetLineMemoryOfBlock (int num)
+    public LogLine? GetLineMemoryOfBlock (int num)
+    {
+        return num < LineCount && num >= 0
+        ? _lineArray[num]
+        : null;
+    }
+
+    /// <summary>
+    /// Acquires the content lock. The caller MUST call <see cref="ReleaseContentLock"/> in a finally block.
+    /// </summary>
+    public void AcquireContentLock (ref bool lockTaken)
+    {
+        _contentLock.Enter(ref lockTaken);
+    }
+
+    /// <summary>
+    /// Releases the content lock previously acquired via <see cref="AcquireContentLock"/>.
+    /// </summary>
+    public void ReleaseContentLock ()
     {
-        return num < _lineList.Count && num >= 0
-            ? _lineList[num]
-            : null;
+        _contentLock.Exit(useMemoryBarrier: false);
     }
 
     #endregion