diff --git a/project/Dataplat.Dbatools.Csv/CHANGELOG.md b/project/Dataplat.Dbatools.Csv/CHANGELOG.md index d621992..44f3d61 100644 --- a/project/Dataplat.Dbatools.Csv/CHANGELOG.md +++ b/project/Dataplat.Dbatools.Csv/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.1.10] - 2025-12-04 + +### Added +- **SQL Server schema inference** - New `CsvSchemaInference` class that analyzes CSV data to determine optimal SQL Server column types. Two modes available: + - `InferSchemaFromSample()` - Fast inference from first N rows (default 1000) + - `InferSchema()` - Full file scan with progress callback for zero-risk type detection +- `InferredColumn` class containing column name, SQL data type, max length, nullability, unicode flag, and decimal precision/scale +- Type detection for: `uniqueidentifier`, `bit`, `int`, `bigint`, `decimal(p,s)`, `datetime2`, `varchar(n)`, `nvarchar(n)` +- `GenerateCreateTableStatement()` utility to produce SQL DDL from inferred schema +- `ToColumnTypes()` utility to convert inferred schema to `CsvReaderOptions.ColumnTypes` dictionary +- Early exit optimization: types are eliminated as values fail validation, reducing unnecessary checks +- Progress callback support for full-scan mode (fires every ~1% or 10K rows) + ## [1.1.1] - 2025-12-04 ### Changed diff --git a/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj b/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj index 39b72bb..b0ef36c 100644 --- a/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj +++ b/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj @@ -7,20 +7,20 @@ Dataplat.Dbatools.Csv - 1.1.1 + 1.1.10 Chrissy LeMaire Dataplat Dataplat.Dbatools.Csv - High-performance CSV reader and writer for .NET. Features streaming IDataReader for SqlBulkCopy, automatic compression (GZip, Deflate, Brotli, ZLib), multi-character delimiters, parallel processing, string interning, and robust error handling. 20%+ faster than LumenWorks CsvReader. From the trusted dbatools project. + High-performance CSV reader with native IDataReader for SqlBulkCopy - 6x faster than legacy solutions for database imports. Database-first design with culture-aware parsing, intelligent null handling, and robust support for messy real-world data (duplicate headers, field mismatches). Features automatic compression (GZip, Deflate, Brotli, ZLib), progress reporting with rows/second metrics, and cancellation support. From the trusted dbatools project. Copyright (c) 2025 Chrissy LeMaire csv;parser;reader;writer;datareader;idatareader;sqlbulkcopy;compression;gzip;brotli;dbatools;high-performance;parallel MIT - https://github.com/dataplat/dbatools.library + https://dataplat.dbatools.io/csv https://github.com/dataplat/dbatools.library git main README.md - Initial release with high-performance CSV parsing, parallel processing support, and comprehensive edge case handling. + v1.1.10: SQL Server schema inference - auto-detect column types (int, bigint, decimal, datetime2, bit, uniqueidentifier, varchar/nvarchar). v1.1.5: Updated package metadata and URL. v1.1.1: ~25% performance improvement for all-columns reads. v1.1.0: Added CancellationToken and progress reporting support. true true false diff --git a/project/Dataplat.Dbatools.Csv/README.md b/project/Dataplat.Dbatools.Csv/README.md index 8949851..f881a13 100644 --- a/project/Dataplat.Dbatools.Csv/README.md +++ b/project/Dataplat.Dbatools.Csv/README.md @@ -8,6 +8,7 @@ **What makes this library unique:** - **Native IDataReader** - Stream directly to SqlBulkCopy with zero intermediate allocations +- **Schema Inference** - Auto-detect SQL Server column types (int, bigint, decimal, datetime2, bit, uniqueidentifier, varchar/nvarchar) - **Built-in compression** - GZip, Brotli, Deflate, ZLib with decompression bomb protection - **Real-world data handling** - Lenient parsing, smart quotes, duplicate headers, field count mismatches - **Faster than LumenWorks & CsvHelper** - ~1.5x faster with modern .NET (Span, ArrayPool) @@ -28,6 +29,8 @@ Install-Package Dataplat.Dbatools.Csv ## Features - **Streaming IDataReader** - Works seamlessly with SqlBulkCopy and other ADO.NET consumers +- **Schema Inference** - Analyze CSV data to determine optimal SQL Server column types +- **Strongly Typed Columns** - Define column types for automatic conversion with built-in and custom converters - **High Performance** - ~1.5x faster than LumenWorks/CsvHelper with ArrayPool-based memory management - **Parallel Processing** - Optional multi-threaded parsing for large files (25K+ rows/sec) - **String Interning** - Reduce memory for files with repeated values @@ -271,6 +274,130 @@ while (reader.Read()) } ``` +### Schema Inference + +Automatically detect optimal SQL Server column types from CSV data. No more `nvarchar(MAX)` for everything: + +```csharp +using Dataplat.Dbatools.Csv.Reader; + +// Fast: Sample first 1000 rows (tiny risk if data changes after sample) +var columns = CsvSchemaInference.InferSchemaFromSample("data.csv"); + +// Safe: Scan entire file with progress reporting (zero risk of type mismatches) +var columns = CsvSchemaInference.InferSchema("data.csv", null, progress => { + Console.WriteLine($"Progress: {progress:P0}"); +}); + +// Examine inferred types +foreach (var col in columns) +{ + Console.WriteLine($"{col.ColumnName}: {col.SqlDataType} {(col.IsNullable ? "NULL" : "NOT NULL")}"); +} +// Output: +// Id: int NOT NULL +// Name: nvarchar(100) NULL +// Price: decimal(10,2) NOT NULL +// Created: datetime2 NULL + +// Generate CREATE TABLE statement +string sql = CsvSchemaInference.GenerateCreateTableStatement(columns, "Products", "dbo"); +// CREATE TABLE [dbo].[Products] ( +// [Id] int NOT NULL, +// [Name] nvarchar(100) NULL, +// [Price] decimal(10,2) NOT NULL, +// [Created] datetime2 NULL +// ); + +// Use inferred types with CsvDataReader +var typeMap = CsvSchemaInference.ToColumnTypes(columns); +var options = new CsvReaderOptions { ColumnTypes = typeMap }; +using var reader = new CsvDataReader("data.csv", options); +``` + +**Detected types:** `uniqueidentifier`, `bit`, `int`, `bigint`, `decimal(p,s)`, `datetime2`, `varchar(n)`, `nvarchar(n)` (when Unicode is detected) + +**InferredColumn properties:** + +| Property | Type | Description | +|----------|------|-------------| +| `ColumnName` | string | Column header name | +| `SqlDataType` | string | SQL Server data type (e.g., `int`, `decimal(10,2)`, `nvarchar(50)`) | +| `IsNullable` | bool | True if any NULL/empty values were found | +| `IsUnicode` | bool | True if non-ASCII characters detected | +| `MaxLength` | int | Maximum string length observed | +| `Precision` | int | Decimal precision (total digits) | +| `Scale` | int | Decimal scale (digits after decimal point) | +| `Ordinal` | int | Column position (0-based) | +| `TotalCount` | long | Total rows analyzed | +| `NonNullCount` | long | Rows with non-null values | + +### Strongly Typed Columns + +Define column types explicitly for automatic conversion during reading: + +```csharp +var options = new CsvReaderOptions +{ + ColumnTypes = new Dictionary + { + ["Id"] = typeof(int), + ["Price"] = typeof(decimal), + ["IsActive"] = typeof(bool), + ["Created"] = typeof(DateTime), + ["UniqueId"] = typeof(Guid) + } +}; + +using var reader = new CsvDataReader("data.csv", options); +while (reader.Read()) +{ + int id = reader.GetInt32(0); // Already converted from string + decimal price = reader.GetDecimal(1); // Culture-aware parsing + bool active = reader.GetBoolean(2); // Handles true/false/yes/no/1/0 + DateTime created = reader.GetDateTime(3); + Guid guid = reader.GetGuid(4); +} +``` + +**Built-in type converters:** `Guid`, `bool`, `DateTime`, `short`, `int`, `long`, `float`, `double`, `decimal`, `byte`, `string` + +**Combine with schema inference:** + +```csharp +// Infer types from CSV data, then use them for reading +var columns = CsvSchemaInference.InferSchemaFromSample("data.csv"); +var typeMap = CsvSchemaInference.ToColumnTypes(columns); + +var options = new CsvReaderOptions { ColumnTypes = typeMap }; +using var reader = new CsvDataReader("data.csv", options); +``` + +**Custom type converters:** + +```csharp +using Dataplat.Dbatools.Csv.TypeConverters; + +// Create a custom converter for enums or custom types +public class StatusConverter : TypeConverterBase +{ + public override bool TryConvert(string value, out OrderStatus result) + { + return Enum.TryParse(value, true, out result); + } +} + +// Register and use +var registry = TypeConverterRegistry.Default; +registry.Register(new StatusConverter()); + +var options = new CsvReaderOptions +{ + TypeConverterRegistry = registry, + ColumnTypes = new Dictionary { ["Status"] = typeof(OrderStatus) } +}; +``` + ### Null vs Empty String Handling CSV files can represent missing data in two ways: an empty field (`,,`) or an explicitly quoted empty string (`,"",...`). The `DistinguishEmptyFromNull` option controls how these are interpreted. diff --git a/project/dbatools.Tests/Csv/CsvSchemaInferenceTest.cs b/project/dbatools.Tests/Csv/CsvSchemaInferenceTest.cs new file mode 100644 index 0000000..4b7f4aa --- /dev/null +++ b/project/dbatools.Tests/Csv/CsvSchemaInferenceTest.cs @@ -0,0 +1,681 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Dataplat.Dbatools.Csv.Reader; + +namespace Dataplat.Dbatools.Csv.Tests +{ + [TestClass] + public class CsvSchemaInferenceTest + { + private string _tempDir; + + [TestInitialize] + public void Setup() + { + _tempDir = Path.Combine(Path.GetTempPath(), "CsvSchemaInferenceTests_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempDir); + } + + [TestCleanup] + public void Cleanup() + { + if (Directory.Exists(_tempDir)) + { + try { Directory.Delete(_tempDir, true); } catch { } + } + } + + #region File-Based Tests + + [TestMethod] + public void TestInferSchema_RealFile_MixedTypes() + { + string csvPath = Path.Combine(_tempDir, "mixed_types.csv"); + File.WriteAllText(csvPath, @"Id,Name,Price,Quantity,IsActive,Created,UniqueId +1,Widget A,19.99,100,true,2024-01-15,550e8400-e29b-41d4-a716-446655440000 +2,Widget B,29.50,50,false,2024-02-20,6ba7b810-9dad-11d1-80b4-00c04fd430c8 +3,Gadget C,99.00,25,yes,2024-03-25,f47ac10b-58cc-4372-a567-0e02b2c3d479 +4,Thing D,5.99,1000,no,2024-04-30,7c9e6679-7425-40de-944b-e07fc1f90ae7 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(7, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); // Id + Assert.IsTrue(columns[1].SqlDataType.StartsWith("varchar(")); // Name + Assert.IsTrue(columns[2].SqlDataType.StartsWith("decimal(")); // Price + Assert.AreEqual("int", columns[3].SqlDataType); // Quantity + Assert.AreEqual("bit", columns[4].SqlDataType); // IsActive + Assert.AreEqual("datetime2", columns[5].SqlDataType); // Created + Assert.AreEqual("uniqueidentifier", columns[6].SqlDataType); // UniqueId + } + + [TestMethod] + public void TestInferSchema_RealFile_LargeIntegers() + { + string csvPath = Path.Combine(_tempDir, "large_ints.csv"); + var sb = new StringBuilder(); + sb.AppendLine("SmallInt,RegularInt,BigInt,TooBig"); + sb.AppendLine("100,2000000000,9000000000000000000,99999999999999999999"); + sb.AppendLine("200,1500000000,8000000000000000000,88888888888888888888"); + File.WriteAllText(csvPath, sb.ToString()); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual("int", columns[0].SqlDataType); // SmallInt fits in int + Assert.AreEqual("int", columns[1].SqlDataType); // RegularInt fits in int + Assert.AreEqual("bigint", columns[2].SqlDataType); // BigInt needs bigint + Assert.IsTrue(columns[3].SqlDataType.StartsWith("varchar(") || + columns[3].SqlDataType.StartsWith("decimal(")); // TooBig overflows + } + + [TestMethod] + public void TestInferSchema_RealFile_DecimalPrecision() + { + string csvPath = Path.Combine(_tempDir, "decimals.csv"); + File.WriteAllText(csvPath, @"Price,Tax,Total,Tiny +19.99,1.50,21.49,0.001 +199.99,15.00,214.99,0.002 +1999.99,150.00,2149.99,0.003 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + // All should be decimal with appropriate precision + Assert.IsTrue(columns[0].SqlDataType.Contains("decimal")); + Assert.IsTrue(columns[1].SqlDataType.Contains("decimal")); + Assert.IsTrue(columns[2].SqlDataType.Contains("decimal")); + Assert.IsTrue(columns[3].SqlDataType.Contains("decimal")); + Assert.AreEqual(3, columns[3].Scale); // Tiny has 3 decimal places + } + + [TestMethod] + public void TestInferSchema_RealFile_NegativeNumbers() + { + string csvPath = Path.Combine(_tempDir, "negatives.csv"); + File.WriteAllText(csvPath, @"Temperature,Balance,Change +-10,1000.50,-5.25 +25,-500.00,10.00 +-40,-1234.56,-0.01 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual("int", columns[0].SqlDataType); // Temperature - integers + Assert.IsTrue(columns[1].SqlDataType.Contains("decimal")); // Balance + Assert.IsTrue(columns[2].SqlDataType.Contains("decimal")); // Change + } + + [TestMethod] + public void TestInferSchema_RealFile_UnicodeStrings() + { + string csvPath = Path.Combine(_tempDir, "unicode.csv"); + File.WriteAllText(csvPath, @"Name,City,Description +José García,São Paulo,Développeur senior +田中太郎,東京,ソフトウェアエンジニア +Müller,München,Geschäftsführer +", Encoding.UTF8); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.IsTrue(columns[0].SqlDataType.StartsWith("nvarchar(")); + Assert.IsTrue(columns[0].IsUnicode); + Assert.IsTrue(columns[1].SqlDataType.StartsWith("nvarchar(")); + Assert.IsTrue(columns[2].SqlDataType.StartsWith("nvarchar(")); + } + + [TestMethod] + public void TestInferSchema_RealFile_DateFormats() + { + string csvPath = Path.Combine(_tempDir, "dates.csv"); + File.WriteAllText(csvPath, @"ISO,US,WithTime +2024-01-15,01/15/2024,2024-01-15 14:30:00 +2024-02-20,02/20/2024,2024-02-20 09:15:30 +2024-03-25,03/25/2024,2024-03-25 18:45:00 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual("datetime2", columns[0].SqlDataType); + Assert.AreEqual("datetime2", columns[1].SqlDataType); + Assert.AreEqual("datetime2", columns[2].SqlDataType); + } + + [TestMethod] + public void TestInferSchema_RealFile_BooleanVariants() + { + string csvPath = Path.Combine(_tempDir, "booleans.csv"); + File.WriteAllText(csvPath, @"TrueFalse,YesNo,OnOff,TF,YN +true,yes,on,t,y +false,no,off,f,n +TRUE,YES,ON,T,Y +FALSE,NO,OFF,F,N +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + foreach (var col in columns) + { + Assert.AreEqual("bit", col.SqlDataType, $"Column {col.ColumnName} should be bit"); + } + } + + [TestMethod] + public void TestInferSchema_RealFile_NullableColumns() + { + string csvPath = Path.Combine(_tempDir, "nullable.csv"); + File.WriteAllText(csvPath, @"Id,Name,OptionalValue +1,John,100 +2,Jane, +3,,200 +4,Bob, +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.IsFalse(columns[0].IsNullable); // Id has all values + Assert.IsTrue(columns[1].IsNullable); // Name has empty + Assert.IsTrue(columns[2].IsNullable); // OptionalValue has empty + } + + [TestMethod] + public void TestInferSchema_RealFile_VeryLongStrings() + { + string csvPath = Path.Combine(_tempDir, "longstrings.csv"); + string longString = new string('x', 5000); + string veryLongString = new string('y', 10000); + File.WriteAllText(csvPath, $"Short,Long,VeryLong\nabc,{longString},{veryLongString}\n"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual("varchar(3)", columns[0].SqlDataType); + Assert.AreEqual("varchar(5000)", columns[1].SqlDataType); + Assert.AreEqual("varchar(max)", columns[2].SqlDataType); // > 8000 + } + + #endregion + + #region Full Scan Tests + + [TestMethod] + public void TestInferSchema_FullScan_10000Rows() + { + string csvPath = Path.Combine(_tempDir, "large.csv"); + using (var writer = new StreamWriter(csvPath)) + { + writer.WriteLine("Id,Value,Category"); + for (int i = 0; i < 10000; i++) + { + // Mix of values to test type detection + string category = i % 10 == 0 ? "A" : (i % 10 == 1 ? "B" : "C"); + writer.WriteLine($"{i},{i * 1.5m:F2},{category}"); + } + } + + var progressValues = new List(); + var columns = CsvSchemaInference.InferSchema(csvPath, null, p => progressValues.Add(p)); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + Assert.IsTrue(columns[1].SqlDataType.Contains("decimal")); + Assert.IsTrue(columns[2].SqlDataType.StartsWith("varchar(")); + + // Progress should have been reported + Assert.IsTrue(progressValues.Count > 0); + Assert.AreEqual(1.0, progressValues.Last(), 0.01); + + // Verify row counts + Assert.AreEqual(10000, columns[0].TotalCount); + } + + [TestMethod] + public void TestInferSchema_FullScan_WithCancellation() + { + string csvPath = Path.Combine(_tempDir, "cancellable.csv"); + using (var writer = new StreamWriter(csvPath)) + { + writer.WriteLine("Id,Value"); + for (int i = 0; i < 1000; i++) + { + writer.WriteLine($"{i},{i * 10}"); + } + } + + // Test 1: Pre-cancelled token should throw immediately + var preCancelledCts = new CancellationTokenSource(); + preCancelledCts.Cancel(); + + try + { + CsvSchemaInference.InferSchema(csvPath, null, null, preCancelledCts.Token); + Assert.Fail("Should have thrown OperationCanceledException for pre-cancelled token"); + } + catch (OperationCanceledException) + { + // Expected + } + + // Test 2: Stream-based inference with cancellation + var cts = new CancellationTokenSource(); + + try + { + // Create data in memory + var sb = new StringBuilder(); + sb.AppendLine("Id,Value"); + for (int i = 0; i < 10000; i++) + { + sb.AppendLine($"{i},{i * 10}"); + } + + using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString()))) + { + // Cancel after very short time to trigger during read + cts.CancelAfter(1); + + // This may or may not throw depending on timing - just verify it handles gracefully + var columns = CsvSchemaInference.InferSchemaFromSample(stream, null, 10000, cts.Token); + + // If we got here, data was small enough to complete before cancellation + // That's acceptable - cancellation is best-effort + } + } + catch (OperationCanceledException) + { + // Expected if cancellation kicked in + } + } + + #endregion + + #region Sample vs Full Scan Comparison + + [TestMethod] + public void TestInferSchema_SampleVsFullScan_ConsistentResults() + { + string csvPath = Path.Combine(_tempDir, "consistent.csv"); + // Use consistent value ranges so sample and full scan produce same type classifications + using (var writer = new StreamWriter(csvPath)) + { + writer.WriteLine("Id,Price,Name"); + for (int i = 0; i < 5000; i++) + { + // Keep all values in same range (1-100, price ~20) + writer.WriteLine($"{(i % 100) + 1},{19.99m + (i % 10) * 0.01m:F2},Product{(i % 10)}"); + } + } + + var sampleColumns = CsvSchemaInference.InferSchemaFromSample(csvPath, null, 100); + var fullColumns = CsvSchemaInference.InferSchema(csvPath); + + // Base type categories should match (int vs decimal vs string), precision may vary + Assert.AreEqual("int", sampleColumns[0].SqlDataType); + Assert.AreEqual("int", fullColumns[0].SqlDataType); + Assert.IsTrue(sampleColumns[1].SqlDataType.Contains("decimal")); + Assert.IsTrue(fullColumns[1].SqlDataType.Contains("decimal")); + Assert.IsTrue(sampleColumns[2].SqlDataType.StartsWith("varchar(")); + Assert.IsTrue(fullColumns[2].SqlDataType.StartsWith("varchar(")); + } + + #endregion + + #region Compressed File Tests + + [TestMethod] + public void TestInferSchema_GzipCompressed() + { + string csvPath = Path.Combine(_tempDir, "data.csv.gz"); + string csvContent = @"Id,Name,Value +1,Test,100 +2,Demo,200 +3,Sample,300 +"; + using (var fs = File.Create(csvPath)) + using (var gz = new GZipStream(fs, CompressionMode.Compress)) + using (var writer = new StreamWriter(gz)) + { + writer.Write(csvContent); + } + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + Assert.IsTrue(columns[1].SqlDataType.StartsWith("varchar(")); + Assert.AreEqual("int", columns[2].SqlDataType); + } + + #endregion + + #region Custom Options Tests + + [TestMethod] + public void TestInferSchema_CustomDelimiter() + { + string csvPath = Path.Combine(_tempDir, "semicolon.csv"); + File.WriteAllText(csvPath, @"Id;Name;Value +1;John;100 +2;Jane;200 +"); + + var options = new CsvReaderOptions { Delimiter = ";" }; + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath, options); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("Id", columns[0].ColumnName); + Assert.AreEqual("Name", columns[1].ColumnName); + Assert.AreEqual("Value", columns[2].ColumnName); + } + + [TestMethod] + public void TestInferSchema_TabDelimited() + { + string csvPath = Path.Combine(_tempDir, "tabs.tsv"); + File.WriteAllText(csvPath, "Id\tName\tValue\n1\tJohn\t100\n2\tJane\t200\n"); + + var options = new CsvReaderOptions { Delimiter = "\t" }; + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath, options); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + } + + [TestMethod] + public void TestInferSchema_CustomDateFormat() + { + string csvPath = Path.Combine(_tempDir, "customdate.csv"); + File.WriteAllText(csvPath, @"Id,Date +1,25-Dec-2024 +2,15-Jan-2025 +3,01-Feb-2025 +"); + + var options = new CsvReaderOptions + { + DateTimeFormats = new[] { "dd-MMM-yyyy" } + }; + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath, options); + + Assert.AreEqual("datetime2", columns[1].SqlDataType); + } + + [TestMethod] + public void TestInferSchema_NoHeaderRow() + { + string csvPath = Path.Combine(_tempDir, "noheader.csv"); + File.WriteAllText(csvPath, @"1,John,100 +2,Jane,200 +3,Bob,300 +"); + + var options = new CsvReaderOptions { HasHeaderRow = false }; + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath, options); + + Assert.AreEqual(3, columns.Count); + // Column names are auto-generated by CsvDataReader (0-based: Column0, Column1, Column2) + Assert.AreEqual("Column0", columns[0].ColumnName); + Assert.AreEqual("Column1", columns[1].ColumnName); + Assert.AreEqual("Column2", columns[2].ColumnName); + } + + #endregion + + #region Edge Cases + + [TestMethod] + public void TestInferSchema_EmptyFile() + { + string csvPath = Path.Combine(_tempDir, "empty.csv"); + File.WriteAllText(csvPath, ""); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(0, columns.Count); + } + + [TestMethod] + public void TestInferSchema_HeaderOnly() + { + string csvPath = Path.Combine(_tempDir, "headeronly.csv"); + File.WriteAllText(csvPath, "Id,Name,Value\n"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("varchar(1)", columns[0].SqlDataType); + Assert.IsTrue(columns[0].IsNullable); + } + + [TestMethod] + public void TestInferSchema_SingleRow() + { + string csvPath = Path.Combine(_tempDir, "singlerow.csv"); + File.WriteAllText(csvPath, "Id,Name,Value\n1,Test,100\n"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + Assert.AreEqual(1, columns[0].TotalCount); + } + + [TestMethod] + public void TestInferSchema_ScientificNotation() + { + string csvPath = Path.Combine(_tempDir, "scientific.csv"); + File.WriteAllText(csvPath, @"Value,BigValue +1.5e2,1.0E10 +2.5e2,2.0E10 +3.5e2,3.0E10 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + // Scientific notation should be handled + Assert.IsTrue(columns[0].SqlDataType.Contains("decimal") || + columns[0].SqlDataType.StartsWith("varchar(")); + } + + [TestMethod] + public void TestInferSchema_MixedTypesInColumn_FallsBackToVarchar() + { + string csvPath = Path.Combine(_tempDir, "mixed.csv"); + File.WriteAllText(csvPath, @"Value +100 +abc +200 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.IsTrue(columns[0].SqlDataType.StartsWith("varchar(")); + } + + [TestMethod] + public void TestInferSchema_QuotedFields() + { + string csvPath = Path.Combine(_tempDir, "quoted.csv"); + // RFC 4180: quotes inside quoted fields are escaped by doubling them + var sb = new StringBuilder(); + sb.AppendLine("Id,Name,Description"); + sb.AppendLine("1,\"John Smith\",\"A \"\"quoted\"\" value\""); + sb.AppendLine("2,\"Jane Doe\",\"Another, with comma\""); + File.WriteAllText(csvPath, sb.ToString()); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(3, columns.Count); + Assert.IsTrue(columns[2].MaxLength > 10); // Should capture full quoted content + } + + [TestMethod] + public void TestInferSchema_LeadingZeros_TreatedAsString() + { + string csvPath = Path.Combine(_tempDir, "leadingzeros.csv"); + File.WriteAllText(csvPath, @"ZipCode,Phone +01234,0123456789 +02345,0234567890 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + // Leading zeros should preserve as integer since parsing ignores them + // but this tests that we handle them gracefully + Assert.IsNotNull(columns[0].SqlDataType); + Assert.IsNotNull(columns[1].SqlDataType); + } + + #endregion + + #region Utility Method Tests + + [TestMethod] + public void TestGenerateCreateTableStatement_ComplexTable() + { + string csvPath = Path.Combine(_tempDir, "complex.csv"); + File.WriteAllText(csvPath, @"Id,Name,Price,IsActive,Created,UniqueId +1,Widget,19.99,true,2024-01-15,550e8400-e29b-41d4-a716-446655440000 +2,,29.50,false,2024-02-20,6ba7b810-9dad-11d1-80b4-00c04fd430c8 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + string sql = CsvSchemaInference.GenerateCreateTableStatement(columns, "Products", "sales"); + + Assert.IsTrue(sql.Contains("CREATE TABLE [sales].[Products]")); + Assert.IsTrue(sql.Contains("[Id] int NOT NULL")); + Assert.IsTrue(sql.Contains("[Name]") && sql.Contains("NULL")); // Name is nullable + Assert.IsTrue(sql.Contains("[Price] decimal")); + Assert.IsTrue(sql.Contains("[IsActive] bit")); + Assert.IsTrue(sql.Contains("[Created] datetime2")); + Assert.IsTrue(sql.Contains("[UniqueId] uniqueidentifier")); + } + + [TestMethod] + public void TestToColumnTypes_Mapping() + { + string csvPath = Path.Combine(_tempDir, "types.csv"); + File.WriteAllText(csvPath, @"IntCol,DecCol,BoolCol,DateCol,GuidCol,StrCol +1,1.5,true,2024-01-01,550e8400-e29b-41d4-a716-446655440000,text +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + var typeMap = CsvSchemaInference.ToColumnTypes(columns); + + Assert.AreEqual(typeof(int), typeMap["IntCol"]); + Assert.AreEqual(typeof(decimal), typeMap["DecCol"]); + Assert.AreEqual(typeof(bool), typeMap["BoolCol"]); + Assert.AreEqual(typeof(DateTime), typeMap["DateCol"]); + Assert.AreEqual(typeof(Guid), typeMap["GuidCol"]); + Assert.AreEqual(typeof(string), typeMap["StrCol"]); + } + + [TestMethod] + public void TestInferredColumn_Properties() + { + string csvPath = Path.Combine(_tempDir, "props.csv"); + File.WriteAllText(csvPath, @"Name,Value +John,100 +Jane,200 +Bob,300 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + // Check Name column + Assert.AreEqual("Name", columns[0].ColumnName); + Assert.AreEqual(0, columns[0].Ordinal); + Assert.AreEqual(4, columns[0].MaxLength); // "John" is longest + Assert.IsFalse(columns[0].IsNullable); + Assert.IsFalse(columns[0].IsUnicode); + Assert.AreEqual(3, columns[0].TotalCount); + Assert.AreEqual(3, columns[0].NonNullCount); + } + + #endregion + + #region Stream-Based Tests + + [TestMethod] + public void TestInferSchema_FromStream() + { + string csv = "Id,Name,Value\n1,John,100\n2,Jane,200\n"; + using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(csv))) + { + var columns = CsvSchemaInference.InferSchemaFromSample(stream); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + } + } + + [TestMethod] + public void TestInferSchema_FromTextReader() + { + string csv = "Id,Name,Value\n1,John,100\n2,Jane,200\n"; + using (var reader = new StringReader(csv)) + { + var columns = CsvSchemaInference.InferSchemaFromSample(reader); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + } + } + + #endregion + + #region Real-World Scenario Tests + + [TestMethod] + public void TestInferSchema_SalesData() + { + string csvPath = Path.Combine(_tempDir, "sales.csv"); + File.WriteAllText(csvPath, @"OrderId,CustomerId,ProductName,Quantity,UnitPrice,Discount,OrderDate,ShipCountry +10248,VINET,Queso Cabrales,12,14.00,0.00,1996-07-04,France +10249,TOMSP,Tofu,9,18.60,0.00,1996-07-05,Germany +10250,HANAR,Sir Rodney's Scones,40,8.00,0.05,1996-07-08,Brazil +10251,VICTE,Manjimup Dried Apples,35,42.40,0.15,1996-07-08,France +10252,SUPRD,Filo Mix,48,5.60,0.10,1996-07-09,Belgium +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(8, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); // OrderId + Assert.IsTrue(columns[1].SqlDataType.StartsWith("varchar(")); // CustomerId + Assert.IsTrue(columns[2].SqlDataType.StartsWith("varchar(")); // ProductName + Assert.AreEqual("int", columns[3].SqlDataType); // Quantity + Assert.IsTrue(columns[4].SqlDataType.Contains("decimal")); // UnitPrice + Assert.IsTrue(columns[5].SqlDataType.Contains("decimal")); // Discount + Assert.AreEqual("datetime2", columns[6].SqlDataType); // OrderDate + Assert.IsTrue(columns[7].SqlDataType.StartsWith("varchar(")); // ShipCountry + } + + [TestMethod] + public void TestInferSchema_EmployeeData() + { + string csvPath = Path.Combine(_tempDir, "employees.csv"); + File.WriteAllText(csvPath, @"EmployeeId,FirstName,LastName,Email,HireDate,Salary,IsManager,DepartmentCode +E001,John,Smith,john.smith@company.com,2020-03-15,75000.00,true,IT +E002,Jane,Doe,jane.doe@company.com,2019-07-22,85000.00,true,HR +E003,Bob,Johnson,bob.j@company.com,2021-01-10,65000.00,false,IT +E004,Alice,Williams,alice.w@company.com,2018-11-05,95000.00,true,FIN +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(8, columns.Count); + Assert.IsTrue(columns[0].SqlDataType.StartsWith("varchar(")); // EmployeeId (has letter prefix) + Assert.AreEqual("datetime2", columns[4].SqlDataType); // HireDate + Assert.IsTrue(columns[5].SqlDataType.Contains("decimal")); // Salary + Assert.AreEqual("bit", columns[6].SqlDataType); // IsManager + } + + #endregion + } +} diff --git a/project/dbatools/Csv/Reader/ColumnTypeAnalyzer.cs b/project/dbatools/Csv/Reader/ColumnTypeAnalyzer.cs new file mode 100644 index 0000000..c1d8e17 --- /dev/null +++ b/project/dbatools/Csv/Reader/ColumnTypeAnalyzer.cs @@ -0,0 +1,438 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.RegularExpressions; + +namespace Dataplat.Dbatools.Csv.Reader +{ + /// + /// Analyzes values for a single column to determine the optimal SQL Server data type. + /// Uses incremental analysis with early exit when types are eliminated. + /// + internal sealed class ColumnTypeAnalyzer + { + // Type flags - tracks which types are still possible + [Flags] + private enum PossibleTypes + { + None = 0, + Guid = 1 << 0, + Boolean = 1 << 1, + Int = 1 << 2, + BigInt = 1 << 3, + Decimal = 1 << 4, + DateTime = 1 << 5, + String = 1 << 6, // Always possible as fallback + All = Guid | Boolean | Int | BigInt | Decimal | DateTime | String + } + + private static readonly HashSet BooleanTrueValues = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "true", "yes", "1", "on", "y", "t" + }; + + private static readonly HashSet BooleanFalseValues = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "false", "no", "0", "off", "n", "f" + }; + + // Standard DateTime formats to try + private static readonly string[] StandardDateTimeFormats = new[] + { + "yyyy-MM-dd HH:mm:ss.fff", + "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd", + "yyyy/MM/dd HH:mm:ss", + "yyyy/MM/dd", + "MM/dd/yyyy HH:mm:ss", + "MM/dd/yyyy", + "dd/MM/yyyy HH:mm:ss", + "dd/MM/yyyy", + "dd-MM-yyyy HH:mm:ss", + "dd-MM-yyyy", + "M/d/yyyy HH:mm:ss", + "M/d/yyyy", + "yyyyMMdd", + "yyyyMMddHHmmss", + "yyyy-MM-ddTHH:mm:ss", + "yyyy-MM-ddTHH:mm:ss.fff", + "yyyy-MM-ddTHH:mm:ssZ", + "yyyy-MM-ddTHH:mm:ss.fffZ", + }; + + private readonly string _columnName; + private readonly int _ordinal; + private readonly string[] _customDateTimeFormats; + private readonly CultureInfo _culture; + + private PossibleTypes _possibleTypes = PossibleTypes.All; + private long _totalCount; + private long _nullCount; + private int _maxLength; + private bool _hasUnicode; + + // Decimal tracking + private int _maxPrecision; // Max total digits + private int _maxScale; // Max digits after decimal point + private int _maxIntegerDigits; // Max digits before decimal point + + /// + /// Creates a new column type analyzer. + /// + /// The column name from the CSV header. + /// The zero-based column position. + /// Optional custom DateTime formats to try first. + /// The culture for parsing numbers and dates. + public ColumnTypeAnalyzer(string columnName, int ordinal, string[] customDateTimeFormats, CultureInfo culture) + { + _columnName = columnName; + _ordinal = ordinal; + _customDateTimeFormats = customDateTimeFormats; + _culture = culture ?? CultureInfo.InvariantCulture; + } + + /// + /// Analyzes a single value and updates type statistics. + /// + /// The string value to analyze. + public void AnalyzeValue(string value) + { + _totalCount++; + + // Handle null/empty + if (string.IsNullOrEmpty(value) || string.IsNullOrWhiteSpace(value)) + { + _nullCount++; + return; + } + + string trimmed = value.Trim(); + if (trimmed.Length == 0) + { + _nullCount++; + return; + } + + // Track max length for string types + if (trimmed.Length > _maxLength) + { + _maxLength = trimmed.Length; + } + + // Check for Unicode characters (non-ASCII) + if (!_hasUnicode) + { + foreach (char c in trimmed) + { + if (c > 127) + { + _hasUnicode = true; + break; + } + } + } + + // Try each type in priority order, eliminating as they fail + // Early exit: if a type is already eliminated, skip checking it + + // 1. GUID check (most specific) + if ((_possibleTypes & PossibleTypes.Guid) != 0) + { + if (!Guid.TryParse(trimmed, out _)) + { + _possibleTypes &= ~PossibleTypes.Guid; + } + } + + // 2. Boolean check + if ((_possibleTypes & PossibleTypes.Boolean) != 0) + { + if (!BooleanTrueValues.Contains(trimmed) && !BooleanFalseValues.Contains(trimmed)) + { + _possibleTypes &= ~PossibleTypes.Boolean; + } + } + + // 3. Integer checks (int then bigint) + if ((_possibleTypes & PossibleTypes.Int) != 0) + { + if (!int.TryParse(trimmed, NumberStyles.Integer, _culture, out _)) + { + _possibleTypes &= ~PossibleTypes.Int; + } + } + + if ((_possibleTypes & PossibleTypes.BigInt) != 0) + { + if (!long.TryParse(trimmed, NumberStyles.Integer, _culture, out _)) + { + _possibleTypes &= ~PossibleTypes.BigInt; + } + } + + // 4. Decimal check - track precision and scale + if ((_possibleTypes & PossibleTypes.Decimal) != 0) + { + if (decimal.TryParse(trimmed, NumberStyles.Number, _culture, out decimal decVal)) + { + // Calculate precision and scale + AnalyzeDecimalPrecision(trimmed, decVal); + } + else + { + _possibleTypes &= ~PossibleTypes.Decimal; + } + } + + // 5. DateTime check + if ((_possibleTypes & PossibleTypes.DateTime) != 0) + { + if (!TryParseDateTime(trimmed)) + { + _possibleTypes &= ~PossibleTypes.DateTime; + } + } + } + + /// + /// Analyzes decimal precision and scale from a parsed value. + /// + private void AnalyzeDecimalPrecision(string original, decimal value) + { + // Use the string representation to count actual digits + // This handles scientific notation and trailing zeros correctly + + string normalized = original.Trim(); + + // Remove sign + if (normalized.StartsWith("-") || normalized.StartsWith("+")) + { + normalized = normalized.Substring(1); + } + + // Handle scientific notation - fall back to string analysis of the decimal + if (normalized.IndexOf('e') >= 0 || normalized.IndexOf('E') >= 0) + { + // Use decimal's string representation for scientific notation + normalized = Math.Abs(value).ToString(CultureInfo.InvariantCulture); + } + + // Remove thousands separators + normalized = normalized.Replace(",", "").Replace(" ", ""); + + // Find decimal point + int decimalIndex = normalized.IndexOf('.'); + if (decimalIndex < 0) + { + // Use culture-specific decimal separator + string decSep = _culture.NumberFormat.NumberDecimalSeparator; + decimalIndex = normalized.IndexOf(decSep, StringComparison.Ordinal); + } + + int integerDigits; + int fractionalDigits; + + if (decimalIndex >= 0) + { + // Count integer part digits (excluding leading zeros for values < 1) + string intPart = normalized.Substring(0, decimalIndex); + integerDigits = CountSignificantDigits(intPart, true); + + // Count fractional digits (including trailing zeros as they indicate precision) + string fracPart = normalized.Substring(decimalIndex + 1); + fractionalDigits = fracPart.Length; + } + else + { + integerDigits = CountSignificantDigits(normalized, true); + fractionalDigits = 0; + } + + // Track maximums + if (integerDigits > _maxIntegerDigits) + { + _maxIntegerDigits = integerDigits; + } + if (fractionalDigits > _maxScale) + { + _maxScale = fractionalDigits; + } + + int totalPrecision = integerDigits + fractionalDigits; + if (totalPrecision > _maxPrecision) + { + _maxPrecision = totalPrecision; + } + } + + /// + /// Counts significant digits in a numeric string. + /// + private static int CountSignificantDigits(string value, bool isIntegerPart) + { + int count = 0; + bool foundNonZero = false; + + foreach (char c in value) + { + if (char.IsDigit(c)) + { + if (isIntegerPart) + { + // For integer part, count after first non-zero (or all if it's just "0") + if (c != '0') + { + foundNonZero = true; + } + if (foundNonZero || value.Length == 1) + { + count++; + } + } + else + { + // For fractional part, count all digits + count++; + } + } + } + + return count > 0 ? count : 1; // At least 1 digit + } + + /// + /// Attempts to parse a value as DateTime using custom and standard formats. + /// + private bool TryParseDateTime(string value) + { + DateTimeStyles styles = DateTimeStyles.AllowWhiteSpaces; + + // Try custom formats first + if (_customDateTimeFormats != null && _customDateTimeFormats.Length > 0) + { + if (DateTime.TryParseExact(value, _customDateTimeFormats, _culture, styles, out _)) + { + return true; + } + } + + // Try standard formats + if (DateTime.TryParseExact(value, StandardDateTimeFormats, _culture, styles, out _)) + { + return true; + } + + // Try general parsing as fallback + return DateTime.TryParse(value, _culture, styles, out _); + } + + /// + /// Returns the inferred column based on all analyzed values. + /// + public InferredColumn GetInferredColumn() + { + var column = new InferredColumn + { + ColumnName = _columnName, + Ordinal = _ordinal, + TotalCount = _totalCount, + NonNullCount = _totalCount - _nullCount, + IsNullable = _nullCount > 0, + IsUnicode = _hasUnicode, + MaxLength = _maxLength + }; + + // If all values were null/empty + if (_totalCount == _nullCount) + { + column.SqlDataType = "varchar(1)"; + column.IsNullable = true; + return column; + } + + // Determine type in priority order + // Priority: GUID > Int > BigInt > Decimal > DateTime > Boolean > String + // Note: Int/BigInt are checked before Boolean because "1" and "0" are valid for both, + // and integer types are more restrictive (if we saw "2", boolean is eliminated but int remains) + + if ((_possibleTypes & PossibleTypes.Guid) != 0) + { + column.SqlDataType = "uniqueidentifier"; + } + else if ((_possibleTypes & PossibleTypes.Int) != 0) + { + column.SqlDataType = "int"; + } + else if ((_possibleTypes & PossibleTypes.BigInt) != 0) + { + column.SqlDataType = "bigint"; + } + else if ((_possibleTypes & PossibleTypes.Decimal) != 0) + { + // Calculate SQL decimal precision and scale + // SQL Server decimal: precision 1-38, scale 0-precision + int precision = _maxIntegerDigits + _maxScale; + int scale = _maxScale; + + // Ensure valid SQL Server decimal bounds + if (precision < 1) precision = 1; + if (precision > 38) precision = 38; + if (scale > precision) scale = precision; + if (scale < 0) scale = 0; + + // If it's effectively an integer in decimal form + if (scale == 0 && precision <= 10 && (_possibleTypes & PossibleTypes.Int) != 0) + { + column.SqlDataType = "int"; + } + else if (scale == 0 && precision <= 19 && (_possibleTypes & PossibleTypes.BigInt) != 0) + { + column.SqlDataType = "bigint"; + } + else + { + column.SqlDataType = $"decimal({precision},{scale})"; + column.Precision = precision; + column.Scale = scale; + } + } + else if ((_possibleTypes & PossibleTypes.Boolean) != 0) + { + column.SqlDataType = "bit"; + } + else if ((_possibleTypes & PossibleTypes.DateTime) != 0) + { + column.SqlDataType = "datetime2"; + } + else + { + // Fall back to string type + column.SqlDataType = GetStringType(column); + } + + return column; + } + + /// + /// Determines the appropriate string type (varchar/nvarchar with length). + /// + private string GetStringType(InferredColumn column) + { + string baseType = _hasUnicode ? "nvarchar" : "varchar"; + int maxAllowed = _hasUnicode ? 4000 : 8000; + + if (_maxLength == 0) + { + return $"{baseType}(1)"; + } + else if (_maxLength > maxAllowed) + { + return $"{baseType}(max)"; + } + else + { + return $"{baseType}({_maxLength})"; + } + } + } +} diff --git a/project/dbatools/Csv/Reader/CsvSchemaInference.cs b/project/dbatools/Csv/Reader/CsvSchemaInference.cs new file mode 100644 index 0000000..aa2a57a --- /dev/null +++ b/project/dbatools/Csv/Reader/CsvSchemaInference.cs @@ -0,0 +1,479 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; + +namespace Dataplat.Dbatools.Csv.Reader +{ + /// + /// Provides SQL Server schema inference for CSV files. + /// Analyzes CSV data to determine optimal column types for database import. + /// + public static class CsvSchemaInference + { + /// + /// Default number of rows to sample for schema inference. + /// + public const int DefaultSampleRows = 1000; + + /// + /// Default progress report interval (percentage points). + /// + private const double ProgressReportInterval = 0.01; // 1% + + #region Sample-Based Inference + + /// + /// Infers SQL Server schema by sampling the first N rows of a CSV file. + /// Fast but has a small risk if data patterns change after the sample. + /// + /// Path to the CSV file. + /// CSV reader options (delimiter, encoding, etc.). If null, defaults are used. + /// Number of rows to sample. Default is 1000. + /// List of inferred column definitions. + /// Thrown when path is null. + /// Thrown when the file does not exist. + public static List InferSchemaFromSample(string path, CsvReaderOptions options = null, int sampleRows = DefaultSampleRows) + { + if (path == null) + throw new ArgumentNullException(nameof(path)); + if (!File.Exists(path)) + throw new FileNotFoundException("CSV file not found.", path); + if (sampleRows < 1) + throw new ArgumentOutOfRangeException(nameof(sampleRows), "Sample rows must be at least 1."); + + options = options ?? new CsvReaderOptions(); + + // Create options copy to avoid modifying the caller's options + var inferOptions = options.Clone(); + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(path, inferOptions)) + { + return InferSchemaCore(reader, sampleRows, null, inferOptions.CancellationToken); + } + } + + /// + /// Infers SQL Server schema by sampling the first N rows from a stream. + /// + /// Stream containing CSV data. + /// CSV reader options. + /// Number of rows to sample. + /// Cancellation token. + /// List of inferred column definitions. + public static List InferSchemaFromSample(Stream stream, CsvReaderOptions options = null, int sampleRows = DefaultSampleRows, CancellationToken cancellationToken = default) + { + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + if (sampleRows < 1) + throw new ArgumentOutOfRangeException(nameof(sampleRows), "Sample rows must be at least 1."); + + options = options ?? new CsvReaderOptions(); + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(stream, inferOptions)) + { + return InferSchemaCore(reader, sampleRows, null, cancellationToken); + } + } + + /// + /// Infers SQL Server schema by sampling the first N rows from a TextReader. + /// + /// TextReader containing CSV data. + /// CSV reader options. + /// Number of rows to sample. + /// Cancellation token. + /// List of inferred column definitions. + public static List InferSchemaFromSample(TextReader textReader, CsvReaderOptions options = null, int sampleRows = DefaultSampleRows, CancellationToken cancellationToken = default) + { + if (textReader == null) + throw new ArgumentNullException(nameof(textReader)); + if (sampleRows < 1) + throw new ArgumentOutOfRangeException(nameof(sampleRows), "Sample rows must be at least 1."); + + options = options ?? new CsvReaderOptions(); + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(textReader, inferOptions)) + { + return InferSchemaCore(reader, sampleRows, null, cancellationToken); + } + } + + #endregion + + #region Full Scan Inference + + /// + /// Infers SQL Server schema by scanning the entire CSV file. + /// Slower but guarantees no import failures due to type mismatches. + /// + /// Path to the CSV file. + /// CSV reader options (delimiter, encoding, etc.). If null, defaults are used. + /// Optional callback receiving progress (0.0 to 1.0). + /// Cancellation token. + /// List of inferred column definitions. + /// Thrown when path is null. + /// Thrown when the file does not exist. + public static List InferSchema(string path, CsvReaderOptions options = null, Action progressCallback = null, CancellationToken cancellationToken = default) + { + if (path == null) + throw new ArgumentNullException(nameof(path)); + if (!File.Exists(path)) + throw new FileNotFoundException("CSV file not found.", path); + + options = options ?? new CsvReaderOptions(); + + // Get file size for progress reporting + long fileSize = new FileInfo(path).Length; + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(path, inferOptions)) + { + return InferSchemaCore(reader, int.MaxValue, WrapProgressCallback(progressCallback, fileSize), cancellationToken); + } + } + + /// + /// Infers SQL Server schema by scanning the entire stream. + /// + /// Stream containing CSV data. + /// CSV reader options. + /// Optional callback receiving progress (0.0 to 1.0). + /// Cancellation token. + /// List of inferred column definitions. + public static List InferSchema(Stream stream, CsvReaderOptions options = null, Action progressCallback = null, CancellationToken cancellationToken = default) + { + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + + options = options ?? new CsvReaderOptions(); + + // Try to get stream length for progress + long streamLength = -1; + try + { + if (stream.CanSeek) + { + streamLength = stream.Length; + } + } + catch + { + // Ignore - some streams don't support Length + } + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(stream, inferOptions)) + { + return InferSchemaCore(reader, int.MaxValue, WrapProgressCallback(progressCallback, streamLength), cancellationToken); + } + } + + /// + /// Infers SQL Server schema by scanning the entire TextReader content. + /// Note: Progress callback will not report accurate percentages for TextReader. + /// + /// TextReader containing CSV data. + /// CSV reader options. + /// Optional callback (will not provide accurate progress for TextReader). + /// Cancellation token. + /// List of inferred column definitions. + public static List InferSchema(TextReader textReader, CsvReaderOptions options = null, Action progressCallback = null, CancellationToken cancellationToken = default) + { + if (textReader == null) + throw new ArgumentNullException(nameof(textReader)); + + options = options ?? new CsvReaderOptions(); + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(textReader, inferOptions)) + { + // For TextReader, we can't report accurate progress since we don't know total size + return InferSchemaCore(reader, int.MaxValue, null, cancellationToken); + } + } + + #endregion + + #region Core Implementation + + /// + /// Core implementation for schema inference using CsvDataReader. + /// + private static List InferSchemaCore(CsvDataReader csvReader, int maxRows, Action progressCallback, CancellationToken cancellationToken) + { + var result = new List(); + ColumnTypeAnalyzer[] analyzers = null; + + long rowCount = 0; + long lastProgressReport = 0; + const long progressInterval = 10000; // Report every 10K rows + + while (csvReader.Read() && rowCount < maxRows) + { + cancellationToken.ThrowIfCancellationRequested(); + + // Initialize analyzers on first row (after Read() populates field count) + if (analyzers == null) + { + int fieldCount = csvReader.FieldCount; + analyzers = new ColumnTypeAnalyzer[fieldCount]; + + var readerOptions = GetReaderOptions(csvReader); + for (int i = 0; i < fieldCount; i++) + { + string columnName = csvReader.GetName(i); + analyzers[i] = new ColumnTypeAnalyzer( + columnName, + i, + readerOptions != null ? readerOptions.DateTimeFormats : null, + readerOptions != null ? readerOptions.Culture : System.Globalization.CultureInfo.InvariantCulture); + } + } + + // Analyze each field + for (int i = 0; i < analyzers.Length; i++) + { + string value = csvReader.GetString(i); + analyzers[i].AnalyzeValue(value); + } + + rowCount++; + + // Report progress periodically + if (progressCallback != null && rowCount - lastProgressReport >= progressInterval) + { + progressCallback(rowCount, maxRows < int.MaxValue ? maxRows : -1); + lastProgressReport = rowCount; + } + } + + // Build results + if (analyzers != null) + { + foreach (var analyzer in analyzers) + { + result.Add(analyzer.GetInferredColumn()); + } + } + else if (csvReader.FieldCount > 0) + { + // Headers only (no data rows) - return varchar(1) NULL for each column + for (int i = 0; i < csvReader.FieldCount; i++) + { + result.Add(new InferredColumn + { + ColumnName = csvReader.GetName(i), + Ordinal = i, + SqlDataType = "varchar(1)", + IsNullable = true, + TotalCount = 0, + NonNullCount = 0, + MaxLength = 0 + }); + } + } + + // Final progress report + if (progressCallback != null) + { + progressCallback(rowCount, rowCount); + } + + return result; + } + + /// + /// Gets the options from a CsvDataReader using reflection if necessary. + /// + private static CsvReaderOptions GetReaderOptions(CsvDataReader reader) + { + // Try to access the Options property if it exists + var optionsProperty = typeof(CsvDataReader).GetProperty("Options"); + if (optionsProperty != null) + { + return optionsProperty.GetValue(reader) as CsvReaderOptions; + } + + // Fall back to using a field + var optionsField = typeof(CsvDataReader).GetField("_options", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + if (optionsField != null) + { + return optionsField.GetValue(reader) as CsvReaderOptions; + } + + return null; + } + + /// + /// Wraps a user progress callback with file-size based progress calculation. + /// + private static Action WrapProgressCallback(Action userCallback, long totalSize) + { + if (userCallback == null) + return null; + + double lastReported = -1; + + return (rowsRead, totalRows) => + { + double progress; + if (totalRows > 0) + { + progress = (double)rowsRead / totalRows; + } + else if (totalSize > 0) + { + // Estimate based on rows read (assume average row size) + // This is rough but better than nothing + progress = Math.Min(0.99, rowsRead * 100.0 / totalSize); + } + else + { + // Can't calculate progress + return; + } + + if (progress > 1.0) progress = 1.0; + + // Only report if progress changed significantly + if (progress - lastReported >= ProgressReportInterval || progress >= 1.0) + { + userCallback(progress); + lastReported = progress; + } + }; + } + + #endregion + + #region Utility Methods + + /// + /// Generates a CREATE TABLE statement from inferred columns. + /// + /// The inferred column definitions. + /// The name of the table to create. + /// Optional schema name (default: dbo). + /// A CREATE TABLE SQL statement. + public static string GenerateCreateTableStatement(List columns, string tableName, string schemaName = "dbo") + { + if (columns == null) + throw new ArgumentNullException(nameof(columns)); + if (string.IsNullOrWhiteSpace(tableName)) + throw new ArgumentException("Table name is required.", nameof(tableName)); + + var sb = new StringBuilder(); + sb.AppendLine(string.Format("CREATE TABLE [{0}].[{1}]", schemaName, tableName)); + sb.AppendLine("("); + + bool first = true; + foreach (var column in columns) + { + if (!first) + { + sb.AppendLine(","); + } + first = false; + + sb.Append(string.Format(" {0}", column.ToSqlDefinition())); + } + + sb.AppendLine(); + sb.AppendLine(");"); + + return sb.ToString(); + } + + /// + /// Converts inferred columns to a ColumnTypes dictionary for use with CsvReaderOptions. + /// + /// The inferred column definitions. + /// A dictionary mapping column names to .NET types. + public static Dictionary ToColumnTypes(List columns) + { + if (columns == null) + throw new ArgumentNullException(nameof(columns)); + + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var column in columns) + { + Type netType = SqlTypeToNetType(column.SqlDataType); + result[column.ColumnName] = netType; + } + + return result; + } + + /// + /// Maps SQL Server data type strings to .NET types. + /// + private static Type SqlTypeToNetType(string sqlType) + { + if (string.IsNullOrEmpty(sqlType)) + return typeof(string); + + // Normalize: remove parentheses and content + string baseType = sqlType.ToLowerInvariant(); + int parenIndex = baseType.IndexOf('('); + if (parenIndex > 0) + { + baseType = baseType.Substring(0, parenIndex); + } + + switch (baseType) + { + case "bit": + return typeof(bool); + case "int": + return typeof(int); + case "bigint": + return typeof(long); + case "smallint": + return typeof(short); + case "tinyint": + return typeof(byte); + case "decimal": + case "numeric": + case "money": + case "smallmoney": + return typeof(decimal); + case "float": + return typeof(double); + case "real": + return typeof(float); + case "datetime": + case "datetime2": + case "date": + case "smalldatetime": + return typeof(DateTime); + case "uniqueidentifier": + return typeof(Guid); + default: + return typeof(string); + } + } + + #endregion + } +} diff --git a/project/dbatools/Csv/Reader/InferredColumn.cs b/project/dbatools/Csv/Reader/InferredColumn.cs new file mode 100644 index 0000000..b03c91c --- /dev/null +++ b/project/dbatools/Csv/Reader/InferredColumn.cs @@ -0,0 +1,86 @@ +namespace Dataplat.Dbatools.Csv.Reader +{ + /// + /// Represents the inferred SQL Server schema for a CSV column. + /// + public sealed class InferredColumn + { + /// + /// Gets or sets the column name from the CSV header. + /// + public string ColumnName { get; set; } + + /// + /// Gets or sets the inferred SQL Server data type. + /// Examples: "int", "bigint", "varchar(47)", "nvarchar(255)", "datetime2", "bit", "uniqueidentifier", "decimal(18,4)" + /// + public string SqlDataType { get; set; } + + /// + /// Gets or sets the maximum length observed for string types. + /// For non-string types, this is 0. + /// + public int MaxLength { get; set; } + + /// + /// Gets or sets whether the column contains null or empty values. + /// When true, the SQL column should allow NULLs. + /// + public bool IsNullable { get; set; } + + /// + /// Gets or sets whether non-ASCII (Unicode) characters were detected. + /// When true, nvarchar should be used instead of varchar. + /// + public bool IsUnicode { get; set; } + + /// + /// Gets or sets the precision for decimal types. + /// Total number of digits (before + after decimal point). + /// + public int Precision { get; set; } + + /// + /// Gets or sets the scale for decimal types. + /// Number of digits after the decimal point. + /// + public int Scale { get; set; } + + /// + /// Gets the zero-based ordinal position of this column. + /// + public int Ordinal { get; internal set; } + + /// + /// Gets or sets the number of distinct non-null values sampled. + /// Useful for estimating cardinality. + /// + public long NonNullCount { get; set; } + + /// + /// Gets or sets the total number of values examined for this column. + /// + public long TotalCount { get; set; } + + /// + /// Returns a string representation of this inferred column. + /// + public override string ToString() + { + string nullability = IsNullable ? " NULL" : " NOT NULL"; + return $"{ColumnName} {SqlDataType}{nullability}"; + } + + /// + /// Gets the full SQL column definition suitable for CREATE TABLE. + /// + /// Whether to quote the column name with square brackets. + /// A SQL column definition string. + public string ToSqlDefinition(bool quoted = true) + { + string name = quoted ? $"[{ColumnName}]" : ColumnName; + string nullability = IsNullable ? "NULL" : "NOT NULL"; + return $"{name} {SqlDataType} {nullability}"; + } + } +}