From f24de6697026e99eeca0bb7100ae07dad82ce644 Mon Sep 17 00:00:00 2001 From: Chrissy LeMaire Date: Thu, 4 Dec 2025 19:32:46 +0100 Subject: [PATCH 1/2] Add SQL Server schema inference for CSV files Introduces CsvSchemaInference and ColumnTypeAnalyzer classes to analyze CSV data and infer optimal SQL Server column types, supporting both sample-based and full-file scans. Adds InferredColumn model, utilities for generating CREATE TABLE statements and column type mappings, and comprehensive tests covering various data scenarios, options, and edge cases. Updates package metadata and version to 1.1.10. --- project/Dataplat.Dbatools.Csv/CHANGELOG.md | 13 + .../Dataplat.Dbatools.Csv.csproj | 8 +- project/Dataplat.Dbatools.Csv/README.md | 60 ++ .../Csv/CsvSchemaInferenceTest.cs | 681 ++++++++++++++++++ .../dbatools/Csv/Reader/ColumnTypeAnalyzer.cs | 438 +++++++++++ .../dbatools/Csv/Reader/CsvSchemaInference.cs | 479 ++++++++++++ project/dbatools/Csv/Reader/InferredColumn.cs | 86 +++ 7 files changed, 1761 insertions(+), 4 deletions(-) create mode 100644 project/dbatools.Tests/Csv/CsvSchemaInferenceTest.cs create mode 100644 project/dbatools/Csv/Reader/ColumnTypeAnalyzer.cs create mode 100644 project/dbatools/Csv/Reader/CsvSchemaInference.cs create mode 100644 project/dbatools/Csv/Reader/InferredColumn.cs diff --git a/project/Dataplat.Dbatools.Csv/CHANGELOG.md b/project/Dataplat.Dbatools.Csv/CHANGELOG.md index d621992..44f3d61 100644 --- a/project/Dataplat.Dbatools.Csv/CHANGELOG.md +++ b/project/Dataplat.Dbatools.Csv/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.1.10] - 2025-12-04 + +### Added +- **SQL Server schema inference** - New `CsvSchemaInference` class that analyzes CSV data to determine optimal SQL Server column types. Two modes available: + - `InferSchemaFromSample()` - Fast inference from first N rows (default 1000) + - `InferSchema()` - Full file scan with progress callback for zero-risk type detection +- `InferredColumn` class containing column name, SQL data type, max length, nullability, unicode flag, and decimal precision/scale +- Type detection for: `uniqueidentifier`, `bit`, `int`, `bigint`, `decimal(p,s)`, `datetime2`, `varchar(n)`, `nvarchar(n)` +- `GenerateCreateTableStatement()` utility to produce SQL DDL from inferred schema +- `ToColumnTypes()` utility to convert inferred schema to `CsvReaderOptions.ColumnTypes` dictionary +- Early exit optimization: types are eliminated as values fail validation, reducing unnecessary checks +- Progress callback support for full-scan mode (fires every ~1% or 10K rows) + ## [1.1.1] - 2025-12-04 ### Changed diff --git a/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj b/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj index 39b72bb..b0ef36c 100644 --- a/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj +++ b/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj @@ -7,20 +7,20 @@ Dataplat.Dbatools.Csv - 1.1.1 + 1.1.10 Chrissy LeMaire Dataplat Dataplat.Dbatools.Csv - High-performance CSV reader and writer for .NET. Features streaming IDataReader for SqlBulkCopy, automatic compression (GZip, Deflate, Brotli, ZLib), multi-character delimiters, parallel processing, string interning, and robust error handling. 20%+ faster than LumenWorks CsvReader. From the trusted dbatools project. + High-performance CSV reader with native IDataReader for SqlBulkCopy - 6x faster than legacy solutions for database imports. Database-first design with culture-aware parsing, intelligent null handling, and robust support for messy real-world data (duplicate headers, field mismatches). Features automatic compression (GZip, Deflate, Brotli, ZLib), progress reporting with rows/second metrics, and cancellation support. From the trusted dbatools project. Copyright (c) 2025 Chrissy LeMaire csv;parser;reader;writer;datareader;idatareader;sqlbulkcopy;compression;gzip;brotli;dbatools;high-performance;parallel MIT - https://github.com/dataplat/dbatools.library + https://dataplat.dbatools.io/csv https://github.com/dataplat/dbatools.library git main README.md - Initial release with high-performance CSV parsing, parallel processing support, and comprehensive edge case handling. + v1.1.10: SQL Server schema inference - auto-detect column types (int, bigint, decimal, datetime2, bit, uniqueidentifier, varchar/nvarchar). v1.1.5: Updated package metadata and URL. v1.1.1: ~25% performance improvement for all-columns reads. v1.1.0: Added CancellationToken and progress reporting support. true true false diff --git a/project/Dataplat.Dbatools.Csv/README.md b/project/Dataplat.Dbatools.Csv/README.md index 8949851..ca5e226 100644 --- a/project/Dataplat.Dbatools.Csv/README.md +++ b/project/Dataplat.Dbatools.Csv/README.md @@ -8,6 +8,7 @@ **What makes this library unique:** - **Native IDataReader** - Stream directly to SqlBulkCopy with zero intermediate allocations +- **Schema Inference** - Auto-detect SQL Server column types (int, bigint, decimal, datetime2, bit, uniqueidentifier, varchar/nvarchar) - **Built-in compression** - GZip, Brotli, Deflate, ZLib with decompression bomb protection - **Real-world data handling** - Lenient parsing, smart quotes, duplicate headers, field count mismatches - **Faster than LumenWorks & CsvHelper** - ~1.5x faster with modern .NET (Span, ArrayPool) @@ -28,6 +29,7 @@ Install-Package Dataplat.Dbatools.Csv ## Features - **Streaming IDataReader** - Works seamlessly with SqlBulkCopy and other ADO.NET consumers +- **Schema Inference** - Analyze CSV data to determine optimal SQL Server column types - **High Performance** - ~1.5x faster than LumenWorks/CsvHelper with ArrayPool-based memory management - **Parallel Processing** - Optional multi-threaded parsing for large files (25K+ rows/sec) - **String Interning** - Reduce memory for files with repeated values @@ -271,6 +273,64 @@ while (reader.Read()) } ``` +### Schema Inference + +Automatically detect optimal SQL Server column types from CSV data. No more `nvarchar(MAX)` for everything: + +```csharp +using Dataplat.Dbatools.Csv.Reader; + +// Fast: Sample first 1000 rows (tiny risk if data changes after sample) +var columns = CsvSchemaInference.InferSchemaFromSample("data.csv"); + +// Safe: Scan entire file with progress reporting (zero risk of type mismatches) +var columns = CsvSchemaInference.InferSchema("data.csv", null, progress => { + Console.WriteLine($"Progress: {progress:P0}"); +}); + +// Examine inferred types +foreach (var col in columns) +{ + Console.WriteLine($"{col.ColumnName}: {col.SqlDataType} {(col.IsNullable ? "NULL" : "NOT NULL")}"); +} +// Output: +// Id: int NOT NULL +// Name: nvarchar(100) NULL +// Price: decimal(10,2) NOT NULL +// Created: datetime2 NULL + +// Generate CREATE TABLE statement +string sql = CsvSchemaInference.GenerateCreateTableStatement(columns, "Products", "dbo"); +// CREATE TABLE [dbo].[Products] ( +// [Id] int NOT NULL, +// [Name] nvarchar(100) NULL, +// [Price] decimal(10,2) NOT NULL, +// [Created] datetime2 NULL +// ); + +// Use inferred types with CsvDataReader +var typeMap = CsvSchemaInference.ToColumnTypes(columns); +var options = new CsvReaderOptions { ColumnTypes = typeMap }; +using var reader = new CsvDataReader("data.csv", options); +``` + +**Detected types:** `uniqueidentifier`, `bit`, `int`, `bigint`, `decimal(p,s)`, `datetime2`, `varchar(n)`, `nvarchar(n)` (when Unicode is detected) + +**InferredColumn properties:** + +| Property | Type | Description | +|----------|------|-------------| +| `ColumnName` | string | Column header name | +| `SqlDataType` | string | SQL Server data type (e.g., `int`, `decimal(10,2)`, `nvarchar(50)`) | +| `IsNullable` | bool | True if any NULL/empty values were found | +| `IsUnicode` | bool | True if non-ASCII characters detected | +| `MaxLength` | int | Maximum string length observed | +| `Precision` | int | Decimal precision (total digits) | +| `Scale` | int | Decimal scale (digits after decimal point) | +| `Ordinal` | int | Column position (0-based) | +| `TotalCount` | long | Total rows analyzed | +| `NonNullCount` | long | Rows with non-null values | + ### Null vs Empty String Handling CSV files can represent missing data in two ways: an empty field (`,,`) or an explicitly quoted empty string (`,"",...`). The `DistinguishEmptyFromNull` option controls how these are interpreted. diff --git a/project/dbatools.Tests/Csv/CsvSchemaInferenceTest.cs b/project/dbatools.Tests/Csv/CsvSchemaInferenceTest.cs new file mode 100644 index 0000000..4b7f4aa --- /dev/null +++ b/project/dbatools.Tests/Csv/CsvSchemaInferenceTest.cs @@ -0,0 +1,681 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Dataplat.Dbatools.Csv.Reader; + +namespace Dataplat.Dbatools.Csv.Tests +{ + [TestClass] + public class CsvSchemaInferenceTest + { + private string _tempDir; + + [TestInitialize] + public void Setup() + { + _tempDir = Path.Combine(Path.GetTempPath(), "CsvSchemaInferenceTests_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempDir); + } + + [TestCleanup] + public void Cleanup() + { + if (Directory.Exists(_tempDir)) + { + try { Directory.Delete(_tempDir, true); } catch { } + } + } + + #region File-Based Tests + + [TestMethod] + public void TestInferSchema_RealFile_MixedTypes() + { + string csvPath = Path.Combine(_tempDir, "mixed_types.csv"); + File.WriteAllText(csvPath, @"Id,Name,Price,Quantity,IsActive,Created,UniqueId +1,Widget A,19.99,100,true,2024-01-15,550e8400-e29b-41d4-a716-446655440000 +2,Widget B,29.50,50,false,2024-02-20,6ba7b810-9dad-11d1-80b4-00c04fd430c8 +3,Gadget C,99.00,25,yes,2024-03-25,f47ac10b-58cc-4372-a567-0e02b2c3d479 +4,Thing D,5.99,1000,no,2024-04-30,7c9e6679-7425-40de-944b-e07fc1f90ae7 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(7, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); // Id + Assert.IsTrue(columns[1].SqlDataType.StartsWith("varchar(")); // Name + Assert.IsTrue(columns[2].SqlDataType.StartsWith("decimal(")); // Price + Assert.AreEqual("int", columns[3].SqlDataType); // Quantity + Assert.AreEqual("bit", columns[4].SqlDataType); // IsActive + Assert.AreEqual("datetime2", columns[5].SqlDataType); // Created + Assert.AreEqual("uniqueidentifier", columns[6].SqlDataType); // UniqueId + } + + [TestMethod] + public void TestInferSchema_RealFile_LargeIntegers() + { + string csvPath = Path.Combine(_tempDir, "large_ints.csv"); + var sb = new StringBuilder(); + sb.AppendLine("SmallInt,RegularInt,BigInt,TooBig"); + sb.AppendLine("100,2000000000,9000000000000000000,99999999999999999999"); + sb.AppendLine("200,1500000000,8000000000000000000,88888888888888888888"); + File.WriteAllText(csvPath, sb.ToString()); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual("int", columns[0].SqlDataType); // SmallInt fits in int + Assert.AreEqual("int", columns[1].SqlDataType); // RegularInt fits in int + Assert.AreEqual("bigint", columns[2].SqlDataType); // BigInt needs bigint + Assert.IsTrue(columns[3].SqlDataType.StartsWith("varchar(") || + columns[3].SqlDataType.StartsWith("decimal(")); // TooBig overflows + } + + [TestMethod] + public void TestInferSchema_RealFile_DecimalPrecision() + { + string csvPath = Path.Combine(_tempDir, "decimals.csv"); + File.WriteAllText(csvPath, @"Price,Tax,Total,Tiny +19.99,1.50,21.49,0.001 +199.99,15.00,214.99,0.002 +1999.99,150.00,2149.99,0.003 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + // All should be decimal with appropriate precision + Assert.IsTrue(columns[0].SqlDataType.Contains("decimal")); + Assert.IsTrue(columns[1].SqlDataType.Contains("decimal")); + Assert.IsTrue(columns[2].SqlDataType.Contains("decimal")); + Assert.IsTrue(columns[3].SqlDataType.Contains("decimal")); + Assert.AreEqual(3, columns[3].Scale); // Tiny has 3 decimal places + } + + [TestMethod] + public void TestInferSchema_RealFile_NegativeNumbers() + { + string csvPath = Path.Combine(_tempDir, "negatives.csv"); + File.WriteAllText(csvPath, @"Temperature,Balance,Change +-10,1000.50,-5.25 +25,-500.00,10.00 +-40,-1234.56,-0.01 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual("int", columns[0].SqlDataType); // Temperature - integers + Assert.IsTrue(columns[1].SqlDataType.Contains("decimal")); // Balance + Assert.IsTrue(columns[2].SqlDataType.Contains("decimal")); // Change + } + + [TestMethod] + public void TestInferSchema_RealFile_UnicodeStrings() + { + string csvPath = Path.Combine(_tempDir, "unicode.csv"); + File.WriteAllText(csvPath, @"Name,City,Description +José García,São Paulo,Développeur senior +田中太郎,東京,ソフトウェアエンジニア +Müller,München,Geschäftsführer +", Encoding.UTF8); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.IsTrue(columns[0].SqlDataType.StartsWith("nvarchar(")); + Assert.IsTrue(columns[0].IsUnicode); + Assert.IsTrue(columns[1].SqlDataType.StartsWith("nvarchar(")); + Assert.IsTrue(columns[2].SqlDataType.StartsWith("nvarchar(")); + } + + [TestMethod] + public void TestInferSchema_RealFile_DateFormats() + { + string csvPath = Path.Combine(_tempDir, "dates.csv"); + File.WriteAllText(csvPath, @"ISO,US,WithTime +2024-01-15,01/15/2024,2024-01-15 14:30:00 +2024-02-20,02/20/2024,2024-02-20 09:15:30 +2024-03-25,03/25/2024,2024-03-25 18:45:00 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual("datetime2", columns[0].SqlDataType); + Assert.AreEqual("datetime2", columns[1].SqlDataType); + Assert.AreEqual("datetime2", columns[2].SqlDataType); + } + + [TestMethod] + public void TestInferSchema_RealFile_BooleanVariants() + { + string csvPath = Path.Combine(_tempDir, "booleans.csv"); + File.WriteAllText(csvPath, @"TrueFalse,YesNo,OnOff,TF,YN +true,yes,on,t,y +false,no,off,f,n +TRUE,YES,ON,T,Y +FALSE,NO,OFF,F,N +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + foreach (var col in columns) + { + Assert.AreEqual("bit", col.SqlDataType, $"Column {col.ColumnName} should be bit"); + } + } + + [TestMethod] + public void TestInferSchema_RealFile_NullableColumns() + { + string csvPath = Path.Combine(_tempDir, "nullable.csv"); + File.WriteAllText(csvPath, @"Id,Name,OptionalValue +1,John,100 +2,Jane, +3,,200 +4,Bob, +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.IsFalse(columns[0].IsNullable); // Id has all values + Assert.IsTrue(columns[1].IsNullable); // Name has empty + Assert.IsTrue(columns[2].IsNullable); // OptionalValue has empty + } + + [TestMethod] + public void TestInferSchema_RealFile_VeryLongStrings() + { + string csvPath = Path.Combine(_tempDir, "longstrings.csv"); + string longString = new string('x', 5000); + string veryLongString = new string('y', 10000); + File.WriteAllText(csvPath, $"Short,Long,VeryLong\nabc,{longString},{veryLongString}\n"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual("varchar(3)", columns[0].SqlDataType); + Assert.AreEqual("varchar(5000)", columns[1].SqlDataType); + Assert.AreEqual("varchar(max)", columns[2].SqlDataType); // > 8000 + } + + #endregion + + #region Full Scan Tests + + [TestMethod] + public void TestInferSchema_FullScan_10000Rows() + { + string csvPath = Path.Combine(_tempDir, "large.csv"); + using (var writer = new StreamWriter(csvPath)) + { + writer.WriteLine("Id,Value,Category"); + for (int i = 0; i < 10000; i++) + { + // Mix of values to test type detection + string category = i % 10 == 0 ? "A" : (i % 10 == 1 ? "B" : "C"); + writer.WriteLine($"{i},{i * 1.5m:F2},{category}"); + } + } + + var progressValues = new List(); + var columns = CsvSchemaInference.InferSchema(csvPath, null, p => progressValues.Add(p)); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + Assert.IsTrue(columns[1].SqlDataType.Contains("decimal")); + Assert.IsTrue(columns[2].SqlDataType.StartsWith("varchar(")); + + // Progress should have been reported + Assert.IsTrue(progressValues.Count > 0); + Assert.AreEqual(1.0, progressValues.Last(), 0.01); + + // Verify row counts + Assert.AreEqual(10000, columns[0].TotalCount); + } + + [TestMethod] + public void TestInferSchema_FullScan_WithCancellation() + { + string csvPath = Path.Combine(_tempDir, "cancellable.csv"); + using (var writer = new StreamWriter(csvPath)) + { + writer.WriteLine("Id,Value"); + for (int i = 0; i < 1000; i++) + { + writer.WriteLine($"{i},{i * 10}"); + } + } + + // Test 1: Pre-cancelled token should throw immediately + var preCancelledCts = new CancellationTokenSource(); + preCancelledCts.Cancel(); + + try + { + CsvSchemaInference.InferSchema(csvPath, null, null, preCancelledCts.Token); + Assert.Fail("Should have thrown OperationCanceledException for pre-cancelled token"); + } + catch (OperationCanceledException) + { + // Expected + } + + // Test 2: Stream-based inference with cancellation + var cts = new CancellationTokenSource(); + + try + { + // Create data in memory + var sb = new StringBuilder(); + sb.AppendLine("Id,Value"); + for (int i = 0; i < 10000; i++) + { + sb.AppendLine($"{i},{i * 10}"); + } + + using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString()))) + { + // Cancel after very short time to trigger during read + cts.CancelAfter(1); + + // This may or may not throw depending on timing - just verify it handles gracefully + var columns = CsvSchemaInference.InferSchemaFromSample(stream, null, 10000, cts.Token); + + // If we got here, data was small enough to complete before cancellation + // That's acceptable - cancellation is best-effort + } + } + catch (OperationCanceledException) + { + // Expected if cancellation kicked in + } + } + + #endregion + + #region Sample vs Full Scan Comparison + + [TestMethod] + public void TestInferSchema_SampleVsFullScan_ConsistentResults() + { + string csvPath = Path.Combine(_tempDir, "consistent.csv"); + // Use consistent value ranges so sample and full scan produce same type classifications + using (var writer = new StreamWriter(csvPath)) + { + writer.WriteLine("Id,Price,Name"); + for (int i = 0; i < 5000; i++) + { + // Keep all values in same range (1-100, price ~20) + writer.WriteLine($"{(i % 100) + 1},{19.99m + (i % 10) * 0.01m:F2},Product{(i % 10)}"); + } + } + + var sampleColumns = CsvSchemaInference.InferSchemaFromSample(csvPath, null, 100); + var fullColumns = CsvSchemaInference.InferSchema(csvPath); + + // Base type categories should match (int vs decimal vs string), precision may vary + Assert.AreEqual("int", sampleColumns[0].SqlDataType); + Assert.AreEqual("int", fullColumns[0].SqlDataType); + Assert.IsTrue(sampleColumns[1].SqlDataType.Contains("decimal")); + Assert.IsTrue(fullColumns[1].SqlDataType.Contains("decimal")); + Assert.IsTrue(sampleColumns[2].SqlDataType.StartsWith("varchar(")); + Assert.IsTrue(fullColumns[2].SqlDataType.StartsWith("varchar(")); + } + + #endregion + + #region Compressed File Tests + + [TestMethod] + public void TestInferSchema_GzipCompressed() + { + string csvPath = Path.Combine(_tempDir, "data.csv.gz"); + string csvContent = @"Id,Name,Value +1,Test,100 +2,Demo,200 +3,Sample,300 +"; + using (var fs = File.Create(csvPath)) + using (var gz = new GZipStream(fs, CompressionMode.Compress)) + using (var writer = new StreamWriter(gz)) + { + writer.Write(csvContent); + } + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + Assert.IsTrue(columns[1].SqlDataType.StartsWith("varchar(")); + Assert.AreEqual("int", columns[2].SqlDataType); + } + + #endregion + + #region Custom Options Tests + + [TestMethod] + public void TestInferSchema_CustomDelimiter() + { + string csvPath = Path.Combine(_tempDir, "semicolon.csv"); + File.WriteAllText(csvPath, @"Id;Name;Value +1;John;100 +2;Jane;200 +"); + + var options = new CsvReaderOptions { Delimiter = ";" }; + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath, options); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("Id", columns[0].ColumnName); + Assert.AreEqual("Name", columns[1].ColumnName); + Assert.AreEqual("Value", columns[2].ColumnName); + } + + [TestMethod] + public void TestInferSchema_TabDelimited() + { + string csvPath = Path.Combine(_tempDir, "tabs.tsv"); + File.WriteAllText(csvPath, "Id\tName\tValue\n1\tJohn\t100\n2\tJane\t200\n"); + + var options = new CsvReaderOptions { Delimiter = "\t" }; + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath, options); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + } + + [TestMethod] + public void TestInferSchema_CustomDateFormat() + { + string csvPath = Path.Combine(_tempDir, "customdate.csv"); + File.WriteAllText(csvPath, @"Id,Date +1,25-Dec-2024 +2,15-Jan-2025 +3,01-Feb-2025 +"); + + var options = new CsvReaderOptions + { + DateTimeFormats = new[] { "dd-MMM-yyyy" } + }; + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath, options); + + Assert.AreEqual("datetime2", columns[1].SqlDataType); + } + + [TestMethod] + public void TestInferSchema_NoHeaderRow() + { + string csvPath = Path.Combine(_tempDir, "noheader.csv"); + File.WriteAllText(csvPath, @"1,John,100 +2,Jane,200 +3,Bob,300 +"); + + var options = new CsvReaderOptions { HasHeaderRow = false }; + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath, options); + + Assert.AreEqual(3, columns.Count); + // Column names are auto-generated by CsvDataReader (0-based: Column0, Column1, Column2) + Assert.AreEqual("Column0", columns[0].ColumnName); + Assert.AreEqual("Column1", columns[1].ColumnName); + Assert.AreEqual("Column2", columns[2].ColumnName); + } + + #endregion + + #region Edge Cases + + [TestMethod] + public void TestInferSchema_EmptyFile() + { + string csvPath = Path.Combine(_tempDir, "empty.csv"); + File.WriteAllText(csvPath, ""); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(0, columns.Count); + } + + [TestMethod] + public void TestInferSchema_HeaderOnly() + { + string csvPath = Path.Combine(_tempDir, "headeronly.csv"); + File.WriteAllText(csvPath, "Id,Name,Value\n"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("varchar(1)", columns[0].SqlDataType); + Assert.IsTrue(columns[0].IsNullable); + } + + [TestMethod] + public void TestInferSchema_SingleRow() + { + string csvPath = Path.Combine(_tempDir, "singlerow.csv"); + File.WriteAllText(csvPath, "Id,Name,Value\n1,Test,100\n"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + Assert.AreEqual(1, columns[0].TotalCount); + } + + [TestMethod] + public void TestInferSchema_ScientificNotation() + { + string csvPath = Path.Combine(_tempDir, "scientific.csv"); + File.WriteAllText(csvPath, @"Value,BigValue +1.5e2,1.0E10 +2.5e2,2.0E10 +3.5e2,3.0E10 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + // Scientific notation should be handled + Assert.IsTrue(columns[0].SqlDataType.Contains("decimal") || + columns[0].SqlDataType.StartsWith("varchar(")); + } + + [TestMethod] + public void TestInferSchema_MixedTypesInColumn_FallsBackToVarchar() + { + string csvPath = Path.Combine(_tempDir, "mixed.csv"); + File.WriteAllText(csvPath, @"Value +100 +abc +200 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.IsTrue(columns[0].SqlDataType.StartsWith("varchar(")); + } + + [TestMethod] + public void TestInferSchema_QuotedFields() + { + string csvPath = Path.Combine(_tempDir, "quoted.csv"); + // RFC 4180: quotes inside quoted fields are escaped by doubling them + var sb = new StringBuilder(); + sb.AppendLine("Id,Name,Description"); + sb.AppendLine("1,\"John Smith\",\"A \"\"quoted\"\" value\""); + sb.AppendLine("2,\"Jane Doe\",\"Another, with comma\""); + File.WriteAllText(csvPath, sb.ToString()); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(3, columns.Count); + Assert.IsTrue(columns[2].MaxLength > 10); // Should capture full quoted content + } + + [TestMethod] + public void TestInferSchema_LeadingZeros_TreatedAsString() + { + string csvPath = Path.Combine(_tempDir, "leadingzeros.csv"); + File.WriteAllText(csvPath, @"ZipCode,Phone +01234,0123456789 +02345,0234567890 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + // Leading zeros should preserve as integer since parsing ignores them + // but this tests that we handle them gracefully + Assert.IsNotNull(columns[0].SqlDataType); + Assert.IsNotNull(columns[1].SqlDataType); + } + + #endregion + + #region Utility Method Tests + + [TestMethod] + public void TestGenerateCreateTableStatement_ComplexTable() + { + string csvPath = Path.Combine(_tempDir, "complex.csv"); + File.WriteAllText(csvPath, @"Id,Name,Price,IsActive,Created,UniqueId +1,Widget,19.99,true,2024-01-15,550e8400-e29b-41d4-a716-446655440000 +2,,29.50,false,2024-02-20,6ba7b810-9dad-11d1-80b4-00c04fd430c8 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + string sql = CsvSchemaInference.GenerateCreateTableStatement(columns, "Products", "sales"); + + Assert.IsTrue(sql.Contains("CREATE TABLE [sales].[Products]")); + Assert.IsTrue(sql.Contains("[Id] int NOT NULL")); + Assert.IsTrue(sql.Contains("[Name]") && sql.Contains("NULL")); // Name is nullable + Assert.IsTrue(sql.Contains("[Price] decimal")); + Assert.IsTrue(sql.Contains("[IsActive] bit")); + Assert.IsTrue(sql.Contains("[Created] datetime2")); + Assert.IsTrue(sql.Contains("[UniqueId] uniqueidentifier")); + } + + [TestMethod] + public void TestToColumnTypes_Mapping() + { + string csvPath = Path.Combine(_tempDir, "types.csv"); + File.WriteAllText(csvPath, @"IntCol,DecCol,BoolCol,DateCol,GuidCol,StrCol +1,1.5,true,2024-01-01,550e8400-e29b-41d4-a716-446655440000,text +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + var typeMap = CsvSchemaInference.ToColumnTypes(columns); + + Assert.AreEqual(typeof(int), typeMap["IntCol"]); + Assert.AreEqual(typeof(decimal), typeMap["DecCol"]); + Assert.AreEqual(typeof(bool), typeMap["BoolCol"]); + Assert.AreEqual(typeof(DateTime), typeMap["DateCol"]); + Assert.AreEqual(typeof(Guid), typeMap["GuidCol"]); + Assert.AreEqual(typeof(string), typeMap["StrCol"]); + } + + [TestMethod] + public void TestInferredColumn_Properties() + { + string csvPath = Path.Combine(_tempDir, "props.csv"); + File.WriteAllText(csvPath, @"Name,Value +John,100 +Jane,200 +Bob,300 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + // Check Name column + Assert.AreEqual("Name", columns[0].ColumnName); + Assert.AreEqual(0, columns[0].Ordinal); + Assert.AreEqual(4, columns[0].MaxLength); // "John" is longest + Assert.IsFalse(columns[0].IsNullable); + Assert.IsFalse(columns[0].IsUnicode); + Assert.AreEqual(3, columns[0].TotalCount); + Assert.AreEqual(3, columns[0].NonNullCount); + } + + #endregion + + #region Stream-Based Tests + + [TestMethod] + public void TestInferSchema_FromStream() + { + string csv = "Id,Name,Value\n1,John,100\n2,Jane,200\n"; + using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(csv))) + { + var columns = CsvSchemaInference.InferSchemaFromSample(stream); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + } + } + + [TestMethod] + public void TestInferSchema_FromTextReader() + { + string csv = "Id,Name,Value\n1,John,100\n2,Jane,200\n"; + using (var reader = new StringReader(csv)) + { + var columns = CsvSchemaInference.InferSchemaFromSample(reader); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + } + } + + #endregion + + #region Real-World Scenario Tests + + [TestMethod] + public void TestInferSchema_SalesData() + { + string csvPath = Path.Combine(_tempDir, "sales.csv"); + File.WriteAllText(csvPath, @"OrderId,CustomerId,ProductName,Quantity,UnitPrice,Discount,OrderDate,ShipCountry +10248,VINET,Queso Cabrales,12,14.00,0.00,1996-07-04,France +10249,TOMSP,Tofu,9,18.60,0.00,1996-07-05,Germany +10250,HANAR,Sir Rodney's Scones,40,8.00,0.05,1996-07-08,Brazil +10251,VICTE,Manjimup Dried Apples,35,42.40,0.15,1996-07-08,France +10252,SUPRD,Filo Mix,48,5.60,0.10,1996-07-09,Belgium +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(8, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); // OrderId + Assert.IsTrue(columns[1].SqlDataType.StartsWith("varchar(")); // CustomerId + Assert.IsTrue(columns[2].SqlDataType.StartsWith("varchar(")); // ProductName + Assert.AreEqual("int", columns[3].SqlDataType); // Quantity + Assert.IsTrue(columns[4].SqlDataType.Contains("decimal")); // UnitPrice + Assert.IsTrue(columns[5].SqlDataType.Contains("decimal")); // Discount + Assert.AreEqual("datetime2", columns[6].SqlDataType); // OrderDate + Assert.IsTrue(columns[7].SqlDataType.StartsWith("varchar(")); // ShipCountry + } + + [TestMethod] + public void TestInferSchema_EmployeeData() + { + string csvPath = Path.Combine(_tempDir, "employees.csv"); + File.WriteAllText(csvPath, @"EmployeeId,FirstName,LastName,Email,HireDate,Salary,IsManager,DepartmentCode +E001,John,Smith,john.smith@company.com,2020-03-15,75000.00,true,IT +E002,Jane,Doe,jane.doe@company.com,2019-07-22,85000.00,true,HR +E003,Bob,Johnson,bob.j@company.com,2021-01-10,65000.00,false,IT +E004,Alice,Williams,alice.w@company.com,2018-11-05,95000.00,true,FIN +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(8, columns.Count); + Assert.IsTrue(columns[0].SqlDataType.StartsWith("varchar(")); // EmployeeId (has letter prefix) + Assert.AreEqual("datetime2", columns[4].SqlDataType); // HireDate + Assert.IsTrue(columns[5].SqlDataType.Contains("decimal")); // Salary + Assert.AreEqual("bit", columns[6].SqlDataType); // IsManager + } + + #endregion + } +} diff --git a/project/dbatools/Csv/Reader/ColumnTypeAnalyzer.cs b/project/dbatools/Csv/Reader/ColumnTypeAnalyzer.cs new file mode 100644 index 0000000..c1d8e17 --- /dev/null +++ b/project/dbatools/Csv/Reader/ColumnTypeAnalyzer.cs @@ -0,0 +1,438 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.RegularExpressions; + +namespace Dataplat.Dbatools.Csv.Reader +{ + /// + /// Analyzes values for a single column to determine the optimal SQL Server data type. + /// Uses incremental analysis with early exit when types are eliminated. + /// + internal sealed class ColumnTypeAnalyzer + { + // Type flags - tracks which types are still possible + [Flags] + private enum PossibleTypes + { + None = 0, + Guid = 1 << 0, + Boolean = 1 << 1, + Int = 1 << 2, + BigInt = 1 << 3, + Decimal = 1 << 4, + DateTime = 1 << 5, + String = 1 << 6, // Always possible as fallback + All = Guid | Boolean | Int | BigInt | Decimal | DateTime | String + } + + private static readonly HashSet BooleanTrueValues = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "true", "yes", "1", "on", "y", "t" + }; + + private static readonly HashSet BooleanFalseValues = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "false", "no", "0", "off", "n", "f" + }; + + // Standard DateTime formats to try + private static readonly string[] StandardDateTimeFormats = new[] + { + "yyyy-MM-dd HH:mm:ss.fff", + "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd", + "yyyy/MM/dd HH:mm:ss", + "yyyy/MM/dd", + "MM/dd/yyyy HH:mm:ss", + "MM/dd/yyyy", + "dd/MM/yyyy HH:mm:ss", + "dd/MM/yyyy", + "dd-MM-yyyy HH:mm:ss", + "dd-MM-yyyy", + "M/d/yyyy HH:mm:ss", + "M/d/yyyy", + "yyyyMMdd", + "yyyyMMddHHmmss", + "yyyy-MM-ddTHH:mm:ss", + "yyyy-MM-ddTHH:mm:ss.fff", + "yyyy-MM-ddTHH:mm:ssZ", + "yyyy-MM-ddTHH:mm:ss.fffZ", + }; + + private readonly string _columnName; + private readonly int _ordinal; + private readonly string[] _customDateTimeFormats; + private readonly CultureInfo _culture; + + private PossibleTypes _possibleTypes = PossibleTypes.All; + private long _totalCount; + private long _nullCount; + private int _maxLength; + private bool _hasUnicode; + + // Decimal tracking + private int _maxPrecision; // Max total digits + private int _maxScale; // Max digits after decimal point + private int _maxIntegerDigits; // Max digits before decimal point + + /// + /// Creates a new column type analyzer. + /// + /// The column name from the CSV header. + /// The zero-based column position. + /// Optional custom DateTime formats to try first. + /// The culture for parsing numbers and dates. + public ColumnTypeAnalyzer(string columnName, int ordinal, string[] customDateTimeFormats, CultureInfo culture) + { + _columnName = columnName; + _ordinal = ordinal; + _customDateTimeFormats = customDateTimeFormats; + _culture = culture ?? CultureInfo.InvariantCulture; + } + + /// + /// Analyzes a single value and updates type statistics. + /// + /// The string value to analyze. + public void AnalyzeValue(string value) + { + _totalCount++; + + // Handle null/empty + if (string.IsNullOrEmpty(value) || string.IsNullOrWhiteSpace(value)) + { + _nullCount++; + return; + } + + string trimmed = value.Trim(); + if (trimmed.Length == 0) + { + _nullCount++; + return; + } + + // Track max length for string types + if (trimmed.Length > _maxLength) + { + _maxLength = trimmed.Length; + } + + // Check for Unicode characters (non-ASCII) + if (!_hasUnicode) + { + foreach (char c in trimmed) + { + if (c > 127) + { + _hasUnicode = true; + break; + } + } + } + + // Try each type in priority order, eliminating as they fail + // Early exit: if a type is already eliminated, skip checking it + + // 1. GUID check (most specific) + if ((_possibleTypes & PossibleTypes.Guid) != 0) + { + if (!Guid.TryParse(trimmed, out _)) + { + _possibleTypes &= ~PossibleTypes.Guid; + } + } + + // 2. Boolean check + if ((_possibleTypes & PossibleTypes.Boolean) != 0) + { + if (!BooleanTrueValues.Contains(trimmed) && !BooleanFalseValues.Contains(trimmed)) + { + _possibleTypes &= ~PossibleTypes.Boolean; + } + } + + // 3. Integer checks (int then bigint) + if ((_possibleTypes & PossibleTypes.Int) != 0) + { + if (!int.TryParse(trimmed, NumberStyles.Integer, _culture, out _)) + { + _possibleTypes &= ~PossibleTypes.Int; + } + } + + if ((_possibleTypes & PossibleTypes.BigInt) != 0) + { + if (!long.TryParse(trimmed, NumberStyles.Integer, _culture, out _)) + { + _possibleTypes &= ~PossibleTypes.BigInt; + } + } + + // 4. Decimal check - track precision and scale + if ((_possibleTypes & PossibleTypes.Decimal) != 0) + { + if (decimal.TryParse(trimmed, NumberStyles.Number, _culture, out decimal decVal)) + { + // Calculate precision and scale + AnalyzeDecimalPrecision(trimmed, decVal); + } + else + { + _possibleTypes &= ~PossibleTypes.Decimal; + } + } + + // 5. DateTime check + if ((_possibleTypes & PossibleTypes.DateTime) != 0) + { + if (!TryParseDateTime(trimmed)) + { + _possibleTypes &= ~PossibleTypes.DateTime; + } + } + } + + /// + /// Analyzes decimal precision and scale from a parsed value. + /// + private void AnalyzeDecimalPrecision(string original, decimal value) + { + // Use the string representation to count actual digits + // This handles scientific notation and trailing zeros correctly + + string normalized = original.Trim(); + + // Remove sign + if (normalized.StartsWith("-") || normalized.StartsWith("+")) + { + normalized = normalized.Substring(1); + } + + // Handle scientific notation - fall back to string analysis of the decimal + if (normalized.IndexOf('e') >= 0 || normalized.IndexOf('E') >= 0) + { + // Use decimal's string representation for scientific notation + normalized = Math.Abs(value).ToString(CultureInfo.InvariantCulture); + } + + // Remove thousands separators + normalized = normalized.Replace(",", "").Replace(" ", ""); + + // Find decimal point + int decimalIndex = normalized.IndexOf('.'); + if (decimalIndex < 0) + { + // Use culture-specific decimal separator + string decSep = _culture.NumberFormat.NumberDecimalSeparator; + decimalIndex = normalized.IndexOf(decSep, StringComparison.Ordinal); + } + + int integerDigits; + int fractionalDigits; + + if (decimalIndex >= 0) + { + // Count integer part digits (excluding leading zeros for values < 1) + string intPart = normalized.Substring(0, decimalIndex); + integerDigits = CountSignificantDigits(intPart, true); + + // Count fractional digits (including trailing zeros as they indicate precision) + string fracPart = normalized.Substring(decimalIndex + 1); + fractionalDigits = fracPart.Length; + } + else + { + integerDigits = CountSignificantDigits(normalized, true); + fractionalDigits = 0; + } + + // Track maximums + if (integerDigits > _maxIntegerDigits) + { + _maxIntegerDigits = integerDigits; + } + if (fractionalDigits > _maxScale) + { + _maxScale = fractionalDigits; + } + + int totalPrecision = integerDigits + fractionalDigits; + if (totalPrecision > _maxPrecision) + { + _maxPrecision = totalPrecision; + } + } + + /// + /// Counts significant digits in a numeric string. + /// + private static int CountSignificantDigits(string value, bool isIntegerPart) + { + int count = 0; + bool foundNonZero = false; + + foreach (char c in value) + { + if (char.IsDigit(c)) + { + if (isIntegerPart) + { + // For integer part, count after first non-zero (or all if it's just "0") + if (c != '0') + { + foundNonZero = true; + } + if (foundNonZero || value.Length == 1) + { + count++; + } + } + else + { + // For fractional part, count all digits + count++; + } + } + } + + return count > 0 ? count : 1; // At least 1 digit + } + + /// + /// Attempts to parse a value as DateTime using custom and standard formats. + /// + private bool TryParseDateTime(string value) + { + DateTimeStyles styles = DateTimeStyles.AllowWhiteSpaces; + + // Try custom formats first + if (_customDateTimeFormats != null && _customDateTimeFormats.Length > 0) + { + if (DateTime.TryParseExact(value, _customDateTimeFormats, _culture, styles, out _)) + { + return true; + } + } + + // Try standard formats + if (DateTime.TryParseExact(value, StandardDateTimeFormats, _culture, styles, out _)) + { + return true; + } + + // Try general parsing as fallback + return DateTime.TryParse(value, _culture, styles, out _); + } + + /// + /// Returns the inferred column based on all analyzed values. + /// + public InferredColumn GetInferredColumn() + { + var column = new InferredColumn + { + ColumnName = _columnName, + Ordinal = _ordinal, + TotalCount = _totalCount, + NonNullCount = _totalCount - _nullCount, + IsNullable = _nullCount > 0, + IsUnicode = _hasUnicode, + MaxLength = _maxLength + }; + + // If all values were null/empty + if (_totalCount == _nullCount) + { + column.SqlDataType = "varchar(1)"; + column.IsNullable = true; + return column; + } + + // Determine type in priority order + // Priority: GUID > Int > BigInt > Decimal > DateTime > Boolean > String + // Note: Int/BigInt are checked before Boolean because "1" and "0" are valid for both, + // and integer types are more restrictive (if we saw "2", boolean is eliminated but int remains) + + if ((_possibleTypes & PossibleTypes.Guid) != 0) + { + column.SqlDataType = "uniqueidentifier"; + } + else if ((_possibleTypes & PossibleTypes.Int) != 0) + { + column.SqlDataType = "int"; + } + else if ((_possibleTypes & PossibleTypes.BigInt) != 0) + { + column.SqlDataType = "bigint"; + } + else if ((_possibleTypes & PossibleTypes.Decimal) != 0) + { + // Calculate SQL decimal precision and scale + // SQL Server decimal: precision 1-38, scale 0-precision + int precision = _maxIntegerDigits + _maxScale; + int scale = _maxScale; + + // Ensure valid SQL Server decimal bounds + if (precision < 1) precision = 1; + if (precision > 38) precision = 38; + if (scale > precision) scale = precision; + if (scale < 0) scale = 0; + + // If it's effectively an integer in decimal form + if (scale == 0 && precision <= 10 && (_possibleTypes & PossibleTypes.Int) != 0) + { + column.SqlDataType = "int"; + } + else if (scale == 0 && precision <= 19 && (_possibleTypes & PossibleTypes.BigInt) != 0) + { + column.SqlDataType = "bigint"; + } + else + { + column.SqlDataType = $"decimal({precision},{scale})"; + column.Precision = precision; + column.Scale = scale; + } + } + else if ((_possibleTypes & PossibleTypes.Boolean) != 0) + { + column.SqlDataType = "bit"; + } + else if ((_possibleTypes & PossibleTypes.DateTime) != 0) + { + column.SqlDataType = "datetime2"; + } + else + { + // Fall back to string type + column.SqlDataType = GetStringType(column); + } + + return column; + } + + /// + /// Determines the appropriate string type (varchar/nvarchar with length). + /// + private string GetStringType(InferredColumn column) + { + string baseType = _hasUnicode ? "nvarchar" : "varchar"; + int maxAllowed = _hasUnicode ? 4000 : 8000; + + if (_maxLength == 0) + { + return $"{baseType}(1)"; + } + else if (_maxLength > maxAllowed) + { + return $"{baseType}(max)"; + } + else + { + return $"{baseType}({_maxLength})"; + } + } + } +} diff --git a/project/dbatools/Csv/Reader/CsvSchemaInference.cs b/project/dbatools/Csv/Reader/CsvSchemaInference.cs new file mode 100644 index 0000000..89a5481 --- /dev/null +++ b/project/dbatools/Csv/Reader/CsvSchemaInference.cs @@ -0,0 +1,479 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; + +namespace Dataplat.Dbatools.Csv.Reader +{ + /// + /// Provides SQL Server schema inference for CSV files. + /// Analyzes CSV data to determine optimal column types for database import. + /// + public static class CsvSchemaInference + { + /// + /// Default number of rows to sample for schema inference. + /// + public const int DefaultSampleRows = 1000; + + /// + /// Default progress report interval (percentage points). + /// + private const double ProgressReportInterval = 0.01; // 1% + + #region Sample-Based Inference + + /// + /// Infers SQL Server schema by sampling the first N rows of a CSV file. + /// Fast but has a small risk if data patterns change after the sample. + /// + /// Path to the CSV file. + /// CSV reader options (delimiter, encoding, etc.). If null, defaults are used. + /// Number of rows to sample. Default is 1000. + /// List of inferred column definitions. + /// Thrown when path is null. + /// Thrown when the file does not exist. + public static List InferSchemaFromSample(string path, CsvReaderOptions options = null, int sampleRows = DefaultSampleRows) + { + if (path == null) + throw new ArgumentNullException(nameof(path)); + if (!File.Exists(path)) + throw new FileNotFoundException("CSV file not found.", path); + if (sampleRows < 1) + throw new ArgumentOutOfRangeException(nameof(sampleRows), "Sample rows must be at least 1."); + + options = options ?? new CsvReaderOptions(); + + // Create options copy to avoid modifying the caller's options + var inferOptions = options.Clone(); + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(path, inferOptions)) + { + return InferSchemaCore(reader, sampleRows, null, inferOptions.CancellationToken); + } + } + + /// + /// Infers SQL Server schema by sampling the first N rows from a stream. + /// + /// Stream containing CSV data. + /// CSV reader options. + /// Number of rows to sample. + /// Cancellation token. + /// List of inferred column definitions. + public static List InferSchemaFromSample(Stream stream, CsvReaderOptions options = null, int sampleRows = DefaultSampleRows, CancellationToken cancellationToken = default) + { + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + if (sampleRows < 1) + throw new ArgumentOutOfRangeException(nameof(sampleRows), "Sample rows must be at least 1."); + + options = options ?? new CsvReaderOptions(); + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(stream, inferOptions)) + { + return InferSchemaCore(reader, sampleRows, null, cancellationToken); + } + } + + /// + /// Infers SQL Server schema by sampling the first N rows from a TextReader. + /// + /// TextReader containing CSV data. + /// CSV reader options. + /// Number of rows to sample. + /// Cancellation token. + /// List of inferred column definitions. + public static List InferSchemaFromSample(TextReader textReader, CsvReaderOptions options = null, int sampleRows = DefaultSampleRows, CancellationToken cancellationToken = default) + { + if (textReader == null) + throw new ArgumentNullException(nameof(textReader)); + if (sampleRows < 1) + throw new ArgumentOutOfRangeException(nameof(sampleRows), "Sample rows must be at least 1."); + + options = options ?? new CsvReaderOptions(); + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(textReader, inferOptions)) + { + return InferSchemaCore(reader, sampleRows, null, cancellationToken); + } + } + + #endregion + + #region Full Scan Inference + + /// + /// Infers SQL Server schema by scanning the entire CSV file. + /// Slower but guarantees no import failures due to type mismatches. + /// + /// Path to the CSV file. + /// CSV reader options (delimiter, encoding, etc.). If null, defaults are used. + /// Optional callback receiving progress (0.0 to 1.0). + /// Cancellation token. + /// List of inferred column definitions. + /// Thrown when path is null. + /// Thrown when the file does not exist. + public static List InferSchema(string path, CsvReaderOptions options = null, Action progressCallback = null, CancellationToken cancellationToken = default) + { + if (path == null) + throw new ArgumentNullException(nameof(path)); + if (!File.Exists(path)) + throw new FileNotFoundException("CSV file not found.", path); + + options = options ?? new CsvReaderOptions(); + + // Get file size for progress reporting + long fileSize = new FileInfo(path).Length; + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(path, inferOptions)) + { + return InferSchemaCore(reader, int.MaxValue, WrapProgressCallback(progressCallback, fileSize), cancellationToken); + } + } + + /// + /// Infers SQL Server schema by scanning the entire stream. + /// + /// Stream containing CSV data. + /// CSV reader options. + /// Optional callback receiving progress (0.0 to 1.0). + /// Cancellation token. + /// List of inferred column definitions. + public static List InferSchema(Stream stream, CsvReaderOptions options = null, Action progressCallback = null, CancellationToken cancellationToken = default) + { + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + + options = options ?? new CsvReaderOptions(); + + // Try to get stream length for progress + long streamLength = -1; + try + { + if (stream.CanSeek) + { + streamLength = stream.Length; + } + } + catch + { + // Ignore - some streams don't support Length + } + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(stream, inferOptions)) + { + return InferSchemaCore(reader, int.MaxValue, WrapProgressCallback(progressCallback, streamLength), cancellationToken); + } + } + + /// + /// Infers SQL Server schema by scanning the entire TextReader content. + /// Note: Progress callback will not report accurate percentages for TextReader. + /// + /// TextReader containing CSV data. + /// CSV reader options. + /// Optional callback (will not provide accurate progress for TextReader). + /// Cancellation token. + /// List of inferred column definitions. + public static List InferSchema(TextReader textReader, CsvReaderOptions options = null, Action progressCallback = null, CancellationToken cancellationToken = default) + { + if (textReader == null) + throw new ArgumentNullException(nameof(textReader)); + + options = options ?? new CsvReaderOptions(); + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(textReader, inferOptions)) + { + // For TextReader, we can't report accurate progress since we don't know total size + return InferSchemaCore(reader, int.MaxValue, null, cancellationToken); + } + } + + #endregion + + #region Core Implementation + + /// + /// Core implementation for schema inference using CsvDataReader. + /// + private static List InferSchemaCore(CsvDataReader csvReader, int maxRows, Action progressCallback, CancellationToken cancellationToken) + { + var result = new List(); + ColumnTypeAnalyzer[] analyzers = null; + + long rowCount = 0; + long lastProgressReport = 0; + const long progressInterval = 10000; // Report every 10K rows + + while (csvReader.Read() && rowCount < maxRows) + { + cancellationToken.ThrowIfCancellationRequested(); + + // Initialize analyzers on first row (after Read() populates field count) + if (analyzers == null) + { + int fieldCount = csvReader.FieldCount; + analyzers = new ColumnTypeAnalyzer[fieldCount]; + + var readerOptions = GetReaderOptions(csvReader); + for (int i = 0; i < fieldCount; i++) + { + string columnName = csvReader.GetName(i); + analyzers[i] = new ColumnTypeAnalyzer( + columnName, + i, + readerOptions != null ? readerOptions.DateTimeFormats : null, + readerOptions != null ? readerOptions.Culture : System.Globalization.CultureInfo.InvariantCulture); + } + } + + // Analyze each field + for (int i = 0; i < analyzers.Length; i++) + { + string value = csvReader.GetString(i); + analyzers[i].AnalyzeValue(value); + } + + rowCount++; + + // Report progress periodically + if (progressCallback != null && rowCount - lastProgressReport >= progressInterval) + { + progressCallback(rowCount, maxRows < int.MaxValue ? maxRows : -1); + lastProgressReport = rowCount; + } + } + + // Build results + if (analyzers != null) + { + foreach (var analyzer in analyzers) + { + result.Add(analyzer.GetInferredColumn()); + } + } + else if (csvReader.FieldCount > 0) + { + // Headers only (no data rows) - return varchar(1) NULL for each column + for (int i = 0; i < csvReader.FieldCount; i++) + { + result.Add(new InferredColumn + { + ColumnName = csvReader.GetName(i), + Ordinal = i, + SqlDataType = "varchar(1)", + IsNullable = true, + TotalCount = 0, + NonNullCount = 0, + MaxLength = 0 + }); + } + } + + // Final progress report + if (progressCallback != null) + { + progressCallback(rowCount, rowCount); + } + + return result; + } + + /// + /// Gets the options from a CsvDataReader using reflection if necessary. + /// + private static CsvReaderOptions GetReaderOptions(CsvDataReader reader) + { + // Try to access the Options property if it exists + var optionsProperty = typeof(CsvDataReader).GetProperty("Options"); + if (optionsProperty != null) + { + return optionsProperty.GetValue(reader) as CsvReaderOptions; + } + + // Fall back to using a field + var optionsField = typeof(CsvDataReader).GetField("_options", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + if (optionsField != null) + { + return optionsField.GetValue(reader) as CsvReaderOptions; + } + + return null; + } + + /// + /// Wraps a user progress callback with file-size based progress calculation. + /// + private static Action WrapProgressCallback(Action userCallback, long totalSize) + { + if (userCallback == null) + return null; + + double lastReported = -1; + + return (rowsRead, totalRows) => + { + double progress; + if (totalRows > 0) + { + progress = (double)rowsRead / totalRows; + } + else if (totalSize > 0) + { + // Estimate based on rows read (assume average row size) + // This is rough but better than nothing + progress = Math.Min(0.99, rowsRead * 100.0 / totalSize); + } + else + { + // Can't calculate progress + return; + } + + if (progress > 1.0) progress = 1.0; + + // Only report if progress changed significantly + if (progress - lastReported >= ProgressReportInterval || progress >= 1.0) + { + userCallback(progress); + lastReported = progress; + } + }; + } + + #endregion + + #region Utility Methods + + /// + /// Generates a CREATE TABLE statement from inferred columns. + /// + /// The inferred column definitions. + /// The name of the table to create. + /// Optional schema name (default: dbo). + /// A CREATE TABLE SQL statement. + public static string GenerateCreateTableStatement(IEnumerable columns, string tableName, string schemaName = "dbo") + { + if (columns == null) + throw new ArgumentNullException(nameof(columns)); + if (string.IsNullOrWhiteSpace(tableName)) + throw new ArgumentException("Table name is required.", nameof(tableName)); + + var sb = new StringBuilder(); + sb.AppendLine(string.Format("CREATE TABLE [{0}].[{1}]", schemaName, tableName)); + sb.AppendLine("("); + + bool first = true; + foreach (var column in columns) + { + if (!first) + { + sb.AppendLine(","); + } + first = false; + + sb.Append(string.Format(" {0}", column.ToSqlDefinition())); + } + + sb.AppendLine(); + sb.AppendLine(");"); + + return sb.ToString(); + } + + /// + /// Converts inferred columns to a ColumnTypes dictionary for use with CsvReaderOptions. + /// + /// The inferred column definitions. + /// A dictionary mapping column names to .NET types. + public static Dictionary ToColumnTypes(IEnumerable columns) + { + if (columns == null) + throw new ArgumentNullException(nameof(columns)); + + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var column in columns) + { + Type netType = SqlTypeToNetType(column.SqlDataType); + result[column.ColumnName] = netType; + } + + return result; + } + + /// + /// Maps SQL Server data type strings to .NET types. + /// + private static Type SqlTypeToNetType(string sqlType) + { + if (string.IsNullOrEmpty(sqlType)) + return typeof(string); + + // Normalize: remove parentheses and content + string baseType = sqlType.ToLowerInvariant(); + int parenIndex = baseType.IndexOf('('); + if (parenIndex > 0) + { + baseType = baseType.Substring(0, parenIndex); + } + + switch (baseType) + { + case "bit": + return typeof(bool); + case "int": + return typeof(int); + case "bigint": + return typeof(long); + case "smallint": + return typeof(short); + case "tinyint": + return typeof(byte); + case "decimal": + case "numeric": + case "money": + case "smallmoney": + return typeof(decimal); + case "float": + return typeof(double); + case "real": + return typeof(float); + case "datetime": + case "datetime2": + case "date": + case "smalldatetime": + return typeof(DateTime); + case "uniqueidentifier": + return typeof(Guid); + default: + return typeof(string); + } + } + + #endregion + } +} diff --git a/project/dbatools/Csv/Reader/InferredColumn.cs b/project/dbatools/Csv/Reader/InferredColumn.cs new file mode 100644 index 0000000..b03c91c --- /dev/null +++ b/project/dbatools/Csv/Reader/InferredColumn.cs @@ -0,0 +1,86 @@ +namespace Dataplat.Dbatools.Csv.Reader +{ + /// + /// Represents the inferred SQL Server schema for a CSV column. + /// + public sealed class InferredColumn + { + /// + /// Gets or sets the column name from the CSV header. + /// + public string ColumnName { get; set; } + + /// + /// Gets or sets the inferred SQL Server data type. + /// Examples: "int", "bigint", "varchar(47)", "nvarchar(255)", "datetime2", "bit", "uniqueidentifier", "decimal(18,4)" + /// + public string SqlDataType { get; set; } + + /// + /// Gets or sets the maximum length observed for string types. + /// For non-string types, this is 0. + /// + public int MaxLength { get; set; } + + /// + /// Gets or sets whether the column contains null or empty values. + /// When true, the SQL column should allow NULLs. + /// + public bool IsNullable { get; set; } + + /// + /// Gets or sets whether non-ASCII (Unicode) characters were detected. + /// When true, nvarchar should be used instead of varchar. + /// + public bool IsUnicode { get; set; } + + /// + /// Gets or sets the precision for decimal types. + /// Total number of digits (before + after decimal point). + /// + public int Precision { get; set; } + + /// + /// Gets or sets the scale for decimal types. + /// Number of digits after the decimal point. + /// + public int Scale { get; set; } + + /// + /// Gets the zero-based ordinal position of this column. + /// + public int Ordinal { get; internal set; } + + /// + /// Gets or sets the number of distinct non-null values sampled. + /// Useful for estimating cardinality. + /// + public long NonNullCount { get; set; } + + /// + /// Gets or sets the total number of values examined for this column. + /// + public long TotalCount { get; set; } + + /// + /// Returns a string representation of this inferred column. + /// + public override string ToString() + { + string nullability = IsNullable ? " NULL" : " NOT NULL"; + return $"{ColumnName} {SqlDataType}{nullability}"; + } + + /// + /// Gets the full SQL column definition suitable for CREATE TABLE. + /// + /// Whether to quote the column name with square brackets. + /// A SQL column definition string. + public string ToSqlDefinition(bool quoted = true) + { + string name = quoted ? $"[{ColumnName}]" : ColumnName; + string nullability = IsNullable ? "NULL" : "NOT NULL"; + return $"{name} {SqlDataType} {nullability}"; + } + } +} From e3d10c88dae5c773b1f47f4fc925cb1016413a50 Mon Sep 17 00:00:00 2001 From: Chrissy LeMaire Date: Thu, 4 Dec 2025 21:25:21 +0100 Subject: [PATCH 2/2] Document and support strongly typed columns in CSV reader Expanded the README with detailed documentation and examples for defining strongly typed columns, using built-in and custom type converters, and combining with schema inference. Updated CsvSchemaInference methods to require List for improved type safety and consistency. --- project/Dataplat.Dbatools.Csv/README.md | 67 +++++++++++++++++++ .../dbatools/Csv/Reader/CsvSchemaInference.cs | 4 +- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/project/Dataplat.Dbatools.Csv/README.md b/project/Dataplat.Dbatools.Csv/README.md index ca5e226..f881a13 100644 --- a/project/Dataplat.Dbatools.Csv/README.md +++ b/project/Dataplat.Dbatools.Csv/README.md @@ -30,6 +30,7 @@ Install-Package Dataplat.Dbatools.Csv - **Streaming IDataReader** - Works seamlessly with SqlBulkCopy and other ADO.NET consumers - **Schema Inference** - Analyze CSV data to determine optimal SQL Server column types +- **Strongly Typed Columns** - Define column types for automatic conversion with built-in and custom converters - **High Performance** - ~1.5x faster than LumenWorks/CsvHelper with ArrayPool-based memory management - **Parallel Processing** - Optional multi-threaded parsing for large files (25K+ rows/sec) - **String Interning** - Reduce memory for files with repeated values @@ -331,6 +332,72 @@ using var reader = new CsvDataReader("data.csv", options); | `TotalCount` | long | Total rows analyzed | | `NonNullCount` | long | Rows with non-null values | +### Strongly Typed Columns + +Define column types explicitly for automatic conversion during reading: + +```csharp +var options = new CsvReaderOptions +{ + ColumnTypes = new Dictionary + { + ["Id"] = typeof(int), + ["Price"] = typeof(decimal), + ["IsActive"] = typeof(bool), + ["Created"] = typeof(DateTime), + ["UniqueId"] = typeof(Guid) + } +}; + +using var reader = new CsvDataReader("data.csv", options); +while (reader.Read()) +{ + int id = reader.GetInt32(0); // Already converted from string + decimal price = reader.GetDecimal(1); // Culture-aware parsing + bool active = reader.GetBoolean(2); // Handles true/false/yes/no/1/0 + DateTime created = reader.GetDateTime(3); + Guid guid = reader.GetGuid(4); +} +``` + +**Built-in type converters:** `Guid`, `bool`, `DateTime`, `short`, `int`, `long`, `float`, `double`, `decimal`, `byte`, `string` + +**Combine with schema inference:** + +```csharp +// Infer types from CSV data, then use them for reading +var columns = CsvSchemaInference.InferSchemaFromSample("data.csv"); +var typeMap = CsvSchemaInference.ToColumnTypes(columns); + +var options = new CsvReaderOptions { ColumnTypes = typeMap }; +using var reader = new CsvDataReader("data.csv", options); +``` + +**Custom type converters:** + +```csharp +using Dataplat.Dbatools.Csv.TypeConverters; + +// Create a custom converter for enums or custom types +public class StatusConverter : TypeConverterBase +{ + public override bool TryConvert(string value, out OrderStatus result) + { + return Enum.TryParse(value, true, out result); + } +} + +// Register and use +var registry = TypeConverterRegistry.Default; +registry.Register(new StatusConverter()); + +var options = new CsvReaderOptions +{ + TypeConverterRegistry = registry, + ColumnTypes = new Dictionary { ["Status"] = typeof(OrderStatus) } +}; +``` + ### Null vs Empty String Handling CSV files can represent missing data in two ways: an empty field (`,,`) or an explicitly quoted empty string (`,"",...`). The `DistinguishEmptyFromNull` option controls how these are interpreted. diff --git a/project/dbatools/Csv/Reader/CsvSchemaInference.cs b/project/dbatools/Csv/Reader/CsvSchemaInference.cs index 89a5481..aa2a57a 100644 --- a/project/dbatools/Csv/Reader/CsvSchemaInference.cs +++ b/project/dbatools/Csv/Reader/CsvSchemaInference.cs @@ -375,7 +375,7 @@ private static Action WrapProgressCallback(Action userCallba /// The name of the table to create. /// Optional schema name (default: dbo). /// A CREATE TABLE SQL statement. - public static string GenerateCreateTableStatement(IEnumerable columns, string tableName, string schemaName = "dbo") + public static string GenerateCreateTableStatement(List columns, string tableName, string schemaName = "dbo") { if (columns == null) throw new ArgumentNullException(nameof(columns)); @@ -409,7 +409,7 @@ public static string GenerateCreateTableStatement(IEnumerable co /// /// The inferred column definitions. /// A dictionary mapping column names to .NET types. - public static Dictionary ToColumnTypes(IEnumerable columns) + public static Dictionary ToColumnTypes(List columns) { if (columns == null) throw new ArgumentNullException(nameof(columns));