decimal support

language-extensions/dotnet-core-CSharp/build/windows/build-dotnet-core-CSharp-extension.cmd

@@ -38,7 +38,7 @@ REM Do not call VsDevCmd if the environment is already set. Otherwise, it will k
 REM to the PATH environment variable and it will be too long for windows to handle.
 REM
 IF NOT DEFINED DevEnvDir (
-	CALL "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64
+	CALL "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64
 )
 
 REM VSCMD_START_DIR set the working directory to this variable after calling VsDevCmd.bat
@@ -59,9 +59,9 @@ SET EXTENSION_HOST_INCLUDE=%ENL_ROOT%\extension-host\include
 SET DOTNET_NATIVE_LIB=%DOTNET_EXTENSION_HOME%\lib
 
 IF /I %BUILD_CONFIGURATION%==debug (
-	cl.exe /LD %DOTNET_NATIVE_SRC%\nativecsharpextension.cpp %DOTNET_NATIVE_SRC%\*.cpp /I %DOTNET_NATIVE_INCLUDE% /I %EXTENSION_HOST_INCLUDE% /D WINDOWS /D DEBUG /EHsc /Zi
+	cl.exe /LD %DOTNET_NATIVE_SRC%\nativecsharpextension.cpp %DOTNET_NATIVE_SRC%\*.cpp /I %DOTNET_NATIVE_INCLUDE% /I %EXTENSION_HOST_INCLUDE% /D WINDOWS /D DEBUG /EHsc /Zi /link /MACHINE:X64
 ) ELSE (
-	cl.exe /LD %DOTNET_NATIVE_SRC%\nativecsharpextension.cpp %DOTNET_NATIVE_SRC%\*.cpp /I %DOTNET_NATIVE_INCLUDE% /I %EXTENSION_HOST_INCLUDE% /D WINDOWS /EHsc /Zi
+	cl.exe /LD %DOTNET_NATIVE_SRC%\nativecsharpextension.cpp %DOTNET_NATIVE_SRC%\*.cpp /I %DOTNET_NATIVE_INCLUDE% /I %EXTENSION_HOST_INCLUDE% /D WINDOWS /EHsc /Zi /link /MACHINE:X64
 )
 
 CALL :CHECKERROR %ERRORLEVEL% "Error: Failed to build nativecsharpextension for configuration=%BUILD_CONFIGURATION%" || EXIT /b %ERRORLEVEL%

language-extensions/dotnet-core-CSharp/src/managed/CSharpInputDataSet.cs

@@ -11,6 +11,7 @@
 using System;
 using Microsoft.Data.Analysis;
 using static Microsoft.SqlServer.CSharpExtension.Sql;
+using static Microsoft.SqlServer.CSharpExtension.SqlNumericHelper;
 
 namespace Microsoft.SqlServer.CSharpExtension
 {
@@ -126,6 +127,9 @@ private unsafe void AddColumn(
                 case SqlDataType.DotNetReal:
                     AddDataFrameColumn<float>(columnNumber, rowsNumber, colData, colMap);
                     break;
+                case SqlDataType.DotNetNumeric:
+                    AddNumericDataFrameColumn(columnNumber, rowsNumber, colData, colMap);
+                    break;
                 case SqlDataType.DotNetChar:
                     int[] strLens = new int[rowsNumber];
                     Interop.Copy((int*)colMap, strLens, 0, (int)rowsNumber);
@@ -185,5 +189,47 @@ private unsafe void AddDataFrameColumn<T>(
 
             CSharpDataFrame.Columns.Add(colDataFrame);
         }
+
+        /// <summary>
+        /// This method adds NUMERIC/DECIMAL column data by converting from SQL_NUMERIC_STRUCT
+        /// to C# decimal values, creating a PrimitiveDataFrameColumn<decimal>, and adding it to the DataFrame.
+        /// </summary>
+        /// <param name="columnNumber">The column index.</param>
+        /// <param name="rowsNumber">Number of rows in this column.</param>
+        /// <param name="colData">Pointer to array of SQL_NUMERIC_STRUCT structures.</param>
+        /// <param name="colMap">Pointer to null indicator array (SQL_NULL_DATA for null values).</param>
+        private unsafe void AddNumericDataFrameColumn(
+            ushort columnNumber,
+            ulong  rowsNumber,
+            void   *colData,
+            int    *colMap)
+        {
+            // Cast the raw pointer to SQL_NUMERIC_STRUCT array
+            SqlNumericStruct* numericArray = (SqlNumericStruct*)colData;
+
+            // Create a DataFrame column for decimal values
+            PrimitiveDataFrameColumn<decimal> colDataFrame = 
+                new PrimitiveDataFrameColumn<decimal>(_columns[columnNumber].Name, (int)rowsNumber);
+
+            // Convert each SQL_NUMERIC_STRUCT to decimal, handling nulls
+            Span<int> nullSpan = new Span<int>(colMap, (int)rowsNumber);
+            for (int i = 0; i < (int)rowsNumber; ++i)
+            {
+                // Check if this row has a null value
+                // 
+                // Why check both Nullable == 0 and SQL_NULL_DATA?
+                // - Nullable == 0 means column is declared NOT NULL (cannot contain nulls)
+                // - For NOT NULL columns, skip null checking for performance (nullSpan[i] is undefined)
+                // - For nullable columns (Nullable != 0), check if nullSpan[i] == SQL_NULL_DATA (-1)
+                // - This matches the pattern used by other numeric types in the codebase
+                if (_columns[columnNumber].Nullable == 0 || nullSpan[i] != SQL_NULL_DATA)
+                {
+                    // Convert SQL_NUMERIC_STRUCT to C# decimal
+                    colDataFrame[i] = ToDecimal(numericArray[i]);
+                }
+            }
+
+            CSharpDataFrame.Columns.Add(colDataFrame);
+        }
     }
 }

language-extensions/dotnet-core-CSharp/src/managed/CSharpOutputDataSet.cs

@@ -15,6 +15,7 @@
 using System.Collections.Generic;
 using Microsoft.Data.Analysis;
 using static Microsoft.SqlServer.CSharpExtension.Sql;
+using static Microsoft.SqlServer.CSharpExtension.SqlNumericHelper;
 
 namespace Microsoft.SqlServer.CSharpExtension
 {
@@ -174,6 +175,9 @@ DataFrameColumn column
                 case SqlDataType.DotNetDouble:
                     SetDataPtrs<double>(columnNumber, GetArray<double>(column));
                     break;
+                case SqlDataType.DotNetNumeric:
+                    ExtractNumericColumn(columnNumber, column);
+                    break;
                 case SqlDataType.DotNetChar:
                     // Calculate column size from actual data.
                     // columnSize = max UTF-8 byte length across all rows.
@@ -203,7 +207,7 @@ DataFrameColumn column
         /// <summary>
         /// This method sets data pointer for the column and append the array to the handle list.
         /// </summary>
-        private unsafe void SetDataPtrs<T>(
+        private void SetDataPtrs<T>(
             ushort  columnNumber,
             T[]     array
         ) where T : unmanaged
@@ -213,6 +217,135 @@ T[]     array
             _handleList.Add(handle);
         }
 
+        /// <summary>
+        /// This method extracts NUMERIC/DECIMAL column data by converting C# decimal values
+        /// to SQL_NUMERIC_STRUCT array, pinning it, and storing the pointer.
+        /// </summary>
+        /// <param name="columnNumber">The column index.</param>
+        /// <param name="column">The DataFrameColumn containing decimal values.</param>
+        private void ExtractNumericColumn(
+            ushort          columnNumber,
+            DataFrameColumn column)
+        {
+            if (column == null)
+            {
+                SetDataPtrs<SqlNumericStruct>(columnNumber, Array.Empty<SqlNumericStruct>());
+            }
+            else
+            {
+
+            // For NUMERIC/DECIMAL, we need to determine appropriate precision and scale from the data.
+            // SQL Server supports precision 1-38 and scale 0-precision.
+            // We'll calculate both precision and scale by examining the actual decimal values.
+            //
+            // WHY calculate from data instead of hardcoding?
+            // - The extension doesn't have access to the input column's original precision
+            // - SQL Server validates returned precision against WITH RESULT SETS declaration
+            // - Using precision=38 for all values causes "Invalid data for type numeric" errors
+            // - We must calculate the minimum precision needed to represent the data
+            //
+            byte precision = 0;
+            byte scale = (byte)_columns[columnNumber].DecimalDigits;
+
+            // Calculate precision and scale by examining all non-null values
+            // We need to find the maximum precision and scale to ensure no data loss
+            //
+            // WHY examine ALL rows instead of just sampling?
+            // - A previous implementation only checked first 10 rows (optimization attempt)
+            // - This caused data loss when higher-precision values appeared later in the dataset
+            // - Example: rows 1-10 need precision 6, but row 100 needs precision 14
+            // - If we use precision=6 for the entire column, row 100 gets truncated (data loss!)
+            // - Must examine ALL rows to find maximum precision and scale
+            //
+            for (int rowNumber = 0; rowNumber < column.Length; ++rowNumber)
+            {
+                if (column[rowNumber] != null)
+                {
+                    decimal value = (decimal)column[rowNumber];
+
+                    // Get the scale from the decimal value itself
+                    // Scale is in bits 16-23 of flags field (bits[3])
+                    int[] bits = decimal.GetBits(value);
+                    byte valueScale = (byte)((bits[3] >> 16) & 0x7F);
+                    scale = Math.Max(scale, valueScale);
+
+                    // Calculate precision by counting significant digits
+                    // Remove the scale (decimal places) to get the integer part,
+                    // then count digits in both parts
+                    decimal absValue = Math.Abs(value);
+                    decimal integerPart = Math.Truncate(absValue);
+
+                    // Count digits in integer part (or 1 if zero)
+                    byte integerDigits;
+                    if (integerPart == 0)
+                    {
+                        integerDigits = 1;
+                    }
+                    else
+                    {
+                        // Log10 gives us the magnitude, +1 for digit count
+                        integerDigits = (byte)(Math.Floor(Math.Log10((double)integerPart)) + 1);
+                    }
+
+                    // Precision = digits before decimal + digits after decimal
+                    byte valuePrecision = (byte)(integerDigits + valueScale);
+                    precision = Math.Max(precision, valuePrecision);
+                }
+            }
+
+            // Ensure minimum precision of 1 and maximum of 38
+            precision = Math.Max(precision, (byte)1);
+            precision = Math.Min(precision, (byte)38);
+
+            // Ensure scale doesn't exceed precision
+            if (scale > precision)
+            {
+                precision = scale;
+            }
+
+            // Update column metadata with calculated precision and scale
+            // Size contains the precision for DECIMAL/NUMERIC types (not bytes)
+            // DecimalDigits contains the scale
+            _columns[columnNumber].Size = precision;
+            _columns[columnNumber].DecimalDigits = scale;
+
+            Logging.Trace($"ExtractNumericColumn: Column {columnNumber}, Precision={precision}, Scale={scale}, RowCount={column.Length}");
+
+            // Convert each decimal value to SQL_NUMERIC_STRUCT
+            SqlNumericStruct[] numericArray = new SqlNumericStruct[column.Length];
+            for (int rowNumber = 0; rowNumber < column.Length; ++rowNumber)
+            {
+                if (column[rowNumber] != null)
+                {
+                    decimal value = (decimal)column[rowNumber];
+                    numericArray[rowNumber] = FromDecimal(value, precision, scale);
+                    Logging.Trace($"ExtractNumericColumn: Row {rowNumber}, Value={value} converted to SqlNumericStruct");
+                }
+                else
+                {
+                    // For null values, create a zero-initialized struct
+                    // The null indicator in strLenOrNullMap will mark this as SQL_NULL_DATA
+                    //
+                    // WHY create a struct for NULL values instead of leaving uninitialized?
+                    // - ODBC requires a valid struct pointer even for NULL values
+                    // - The strLenOrNullMap array separately tracks which values are NULL
+                    // - Native code reads from the struct pointer, so it must be valid memory
+                    // - We use sign=1 (positive) by convention for NULL placeholders
+                    numericArray[rowNumber] = new SqlNumericStruct
+                    {
+                        precision = precision,
+                        scale = (sbyte)scale,
+                        sign = 1  // Positive sign convention for NULL placeholders
+                    };
+                    Logging.Trace($"ExtractNumericColumn: Row {rowNumber} is NULL");
+                }
+            }
+
+                // Pin the SqlNumericStruct array and store pointer
+                SetDataPtrs<SqlNumericStruct>(columnNumber, numericArray);
+            }
+        }
+
         /// <summary>
         /// This method gets the array from a DataFrameColumn Column for numeric types.
         /// </summary>

language-extensions/dotnet-core-CSharp/src/managed/CSharpParamContainer.cs

@@ -14,6 +14,7 @@
 using System.Collections.Generic;
 using System.Runtime.InteropServices;
 using static Microsoft.SqlServer.CSharpExtension.Sql;
+using static Microsoft.SqlServer.CSharpExtension.SqlNumericHelper;
 
 namespace Microsoft.SqlServer.CSharpExtension
 {
@@ -132,6 +133,11 @@ public unsafe void AddParam(
                 case SqlDataType.DotNetBit:
                     _params[paramNumber].Value = *(bool*)paramValue;
                     break;
+                case SqlDataType.DotNetNumeric:
+                    // Convert SQL_NUMERIC_STRUCT to C# decimal
+                    SqlNumericStruct* numericPtr = (SqlNumericStruct*)paramValue;
+                    _params[paramNumber].Value = ToDecimal(*numericPtr);
+                    break;
                 case SqlDataType.DotNetChar:
                     _params[paramNumber].Value = Interop.UTF8PtrToStr((char*)paramValue, (ulong)strLenOrNullMap);
                     break;
@@ -214,6 +220,25 @@ public unsafe void ReplaceParam(
                     bool boolValue = Convert.ToBoolean(param.Value);
                     ReplaceNumericParam<bool>(boolValue, paramValue);
                     break;
+                case SqlDataType.DotNetNumeric:
+                    // Convert C# decimal to SQL_NUMERIC_STRUCT
+                    // Use the precision and scale from the parameter metadata
+                    decimal decimalValue = Convert.ToDecimal(param.Value);
+                    // WHY use param.Size for precision?
+                    // - For DECIMAL/NUMERIC parameters, param.Size contains the declared precision (not bytes)
+                    // - This follows standard ODBC behavior where ColumnSize = precision for SQL_NUMERIC/SQL_DECIMAL
+                    // - CRITICAL: The SqlNumericStruct precision MUST match the declared parameter precision
+                    //   or SQL Server rejects it with "Invalid data for type decimal" (Msg 9803)
+                    // - Example: DECIMAL(3,3) parameter MUST have precision=3 in the struct, not precision=38
+                    byte precision = (byte)param.Size;
+                    byte scale = (byte)param.DecimalDigits;
+                    // WHY set strLenOrNullMap to 19?
+                    // - For fixed-size types like SQL_NUMERIC_STRUCT, strLenOrNullMap contains the byte size
+                    // - SQL_NUMERIC_STRUCT is exactly 19 bytes: precision(1) + scale(1) + sign(1) + val(16)
+                    // - This tells ODBC how many bytes to read from the paramValue pointer
+                    *strLenOrNullMap = 19; // sizeof(SqlNumericStruct)
+                    ReplaceNumericStructParam(decimalValue, precision, scale, paramValue);
+                    break;
                 case SqlDataType.DotNetChar:
                     // For CHAR/VARCHAR, strLenOrNullMap is in bytes (1 byte per character for ANSI).
                     // param.Size is the declared parameter size in characters (from SQL Server's CHAR(n)/VARCHAR(n)).
@@ -275,6 +300,50 @@ private unsafe void ReplaceNumericParam<T>(
             *paramValue = (void*)handle.AddrOfPinnedObject();
         }
 
+        /// <summary>
+        /// This method replaces parameter value for NUMERIC/DECIMAL data types.
+        /// Converts C# decimal to SQL_NUMERIC_STRUCT and uses proper memory pinning.
+        /// Follows the same pattern as Java extension's numeric parameter handling.
+        /// </summary>
+        /// <param name="value">The C# decimal value to convert.</param>
+        /// <param name="precision">Total number of digits (1-38).</param>
+        /// <param name="scale">Number of digits after decimal point (0-precision).</param>
+        /// <param name="paramValue">Output pointer to receive the pinned SqlNumericStruct.</param>
+        private unsafe void ReplaceNumericStructParam(
+            decimal value,
+            byte    precision,
+            byte    scale,
+            void    **paramValue)
+        {
+            // Convert C# decimal to SQL_NUMERIC_STRUCT
+            SqlNumericStruct numericStruct = FromDecimal(value, precision, scale);
+
+            // Box the struct into a single-element array to create a heap-allocated copy, then pin it.
+            // 
+            // WHY box into an array before pinning?
+            // - Local struct 'numericStruct' is stack-allocated and will be destroyed when method returns
+            // - We need a heap-allocated copy that survives after this method returns
+            // - GCHandle.Alloc requires a heap object; structs must be boxed first
+            // - Single-element array is the simplest way to create a heap-allocated struct
+            //
+            // WHY pin with GCHandle?
+            // - Native code will dereference the paramValue pointer during execution
+            // - Without pinning, garbage collector could move the object, invalidating the pointer
+            // - GCHandleType.Pinned prevents GC from moving the object until we free the handle
+            //
+            // WHY add handle to _handleList?
+            // - If we don't keep a reference, GC could free the handle immediately
+            // - _handleList keeps handles alive until container is disposed/reset
+            // - Handles are freed in ResetParams or class disposal, ensuring proper cleanup
+            //
+            SqlNumericStruct[] valueArray = new SqlNumericStruct[1] { numericStruct };
+            GCHandle handle = GCHandle.Alloc(valueArray, GCHandleType.Pinned);
+            _handleList.Add(handle);
+            *paramValue = (void*)handle.AddrOfPinnedObject();
+
+            Logging.Trace($"ReplaceNumericStructParam: Converted decimal {value} to SqlNumericStruct (precision={precision}, scale={scale})");
+        }
+
         /// <summary>
         /// This method replaces parameter value for string data types.
         /// If the string is not empty, the address of underlying bytes will be assigned to paramValue.