Skip to content

allows explicitkey id property to bulk insert #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mennansevim wants to merge 2 commits into
base: master
Choose a base branch
from
Open

allows explicitkey id property to bulk insert #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
39db1c9
allows explicitkey id property to bulk insert
May 20, 2020
fda033d
apply explicitkey control to all methods
May 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions src/Dapper.Bulk.Shared/PropertiesCache.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,17 @@ public static List<PropertyInfo> KeyPropertiesCache(Type type)
return keyProperties;
}

public static List<PropertyInfo> ExplicitKeyPropertiesCache(Type type)
{
var keyProperties = KeyPropertiesCache(type);
var explicitKeys = keyProperties
.Where<PropertyInfo>(p => ((IEnumerable<object>)p.GetCustomAttributes(true))
.Any<object>((Func<object, bool>)(a => a.GetType().Name == "ExplicitKeyAttribute")))
.ToList<PropertyInfo>();

return explicitKeys;
}

public static List<PropertyInfo> ComputedPropertiesCache(Type type)
{
if (ComputedProperties.TryGetValue(type.TypeHandle, out var cachedProps))
Expand Down
107 changes: 73 additions & 34 deletions src/Dapper.Bulk/DapperBulk.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,11 @@ public static class DapperBulk
/// <param name="transaction">The transaction to run under, null (the default) if none</param>
/// <param name="batchSize">Number of bulk items inserted together, 0 (the default) if all</param>
/// <param name="bulkCopyTimeout">Number of seconds before bulk command execution timeout, 30 (the default)</param>
public static void BulkInsert<T>(this SqlConnection connection, IEnumerable<T> data, SqlTransaction transaction = null, int batchSize = 0, int bulkCopyTimeout = 30)
public static void BulkInsert<T>(this SqlConnection connection, IEnumerable<T> data,
SqlTransaction transaction = null, int batchSize = 0, int bulkCopyTimeout = 30)
{
var type = typeof(T);
BulkInsert(connection,type,data.Cast<object>(),transaction,batchSize,bulkCopyTimeout);
BulkInsert(connection, type, data.Cast<object>(), transaction, batchSize, bulkCopyTimeout);
}

/// <summary>
Expand All @@ -41,19 +42,25 @@ public static void BulkInsert<T>(this SqlConnection connection, IEnumerable<T> d
/// <param name="transaction">The transaction to run under, null (the default) if none</param>
/// <param name="batchSize">Number of bulk items inserted together, 0 (the default) if all</param>
/// <param name="bulkCopyTimeout">Number of seconds before bulk command execution timeout, 30 (the default)</param>
public static void BulkInsert(this SqlConnection connection, Type type, IEnumerable<object> data, SqlTransaction transaction = null, int batchSize = 0, int bulkCopyTimeout = 30)
{
public static void BulkInsert(this SqlConnection connection, Type type, IEnumerable<object> data,
SqlTransaction transaction = null, int batchSize = 0, int bulkCopyTimeout = 30)
{
var tableName = TableMapper.GetTableName(type);
var allProperties = PropertiesCache.TypePropertiesCache(type);
var keyProperties = PropertiesCache.KeyPropertiesCache(type);
var explicitKeyProperties = PropertiesCache.ExplicitKeyPropertiesCache(type);
var computedProperties = PropertiesCache.ComputedPropertiesCache(type);
var columns = PropertiesCache.GetColumnNamesCache(type);

var allPropertiesExceptKeyAndComputed = allProperties.Except(keyProperties.Union(computedProperties)).ToList();
var allPropertiesExceptKeyAndComputedString = GetColumnsStringSqlServer(allPropertiesExceptKeyAndComputed, columns);
var allPropertiesExceptKeyAndComputed =
GetAllPropertiesExceptKeyAndComputed(allProperties, keyProperties, explicitKeyProperties, computedProperties);
var allPropertiesExceptKeyAndComputedString =
GetColumnsStringSqlServer(allPropertiesExceptKeyAndComputed, columns);
var tempToBeInserted = $"#TempInsert_{tableName}".Replace(".", string.Empty);

connection.Execute($@"SELECT TOP 0 {allPropertiesExceptKeyAndComputedString} INTO {tempToBeInserted} FROM {FormatTableName(tableName)} target WITH(NOLOCK);", null, transaction);
connection.Execute(
$@"SELECT TOP 0 {allPropertiesExceptKeyAndComputedString} INTO {tempToBeInserted} FROM {FormatTableName(tableName)} target WITH(NOLOCK);",
null, transaction);

using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.Default, transaction))
{
Expand All @@ -70,6 +77,13 @@ public static void BulkInsert(this SqlConnection connection, Type type, IEnumera
DROP TABLE {tempToBeInserted};", null, transaction);
}

private static List<PropertyInfo> GetAllPropertiesExceptKeyAndComputed(List<PropertyInfo> allProperties, List<PropertyInfo> keyProperties, List<PropertyInfo> explicitKeyProperties, List<PropertyInfo> computedProperties)
{
return explicitKeyProperties.Count > 0 ?
allProperties :
allProperties.Except(keyProperties.Union(computedProperties)).ToList();
}

/// <summary>
/// Inserts entities into table <typeparamref name="T"/>s (by default) returns inserted entities.
/// </summary>
Expand All @@ -80,12 +94,14 @@ public static void BulkInsert(this SqlConnection connection, Type type, IEnumera
/// <param name="batchSize">Number of bulk items inserted together, 0 (the default) if all</param>
/// <param name="bulkCopyTimeout">Number of seconds before bulk command execution timeout, 30 (the default)</param>
/// <returns>Inserted entities</returns>
public static IEnumerable<T> BulkInsertAndSelect<T>(this SqlConnection connection, IEnumerable<T> data, SqlTransaction transaction = null, int batchSize = 0, int bulkCopyTimeout = 30)
public static IEnumerable<T> BulkInsertAndSelect<T>(this SqlConnection connection, IEnumerable<T> data,
SqlTransaction transaction = null, int batchSize = 0, int bulkCopyTimeout = 30)
{
var type = typeof(T);
var tableName = TableMapper.GetTableName(type);
var allProperties = PropertiesCache.TypePropertiesCache(type);
var keyProperties = PropertiesCache.KeyPropertiesCache(type);
var explicitKeyProperties = PropertiesCache.ExplicitKeyPropertiesCache(type);
var computedProperties = PropertiesCache.ComputedPropertiesCache(type);
var columns = PropertiesCache.GetColumnNamesCache(type);

Expand All @@ -96,17 +112,21 @@ public static IEnumerable<T> BulkInsertAndSelect<T>(this SqlConnection connectio
return dataList;
}

var allPropertiesExceptKeyAndComputed = allProperties.Except(keyProperties.Union(computedProperties)).ToList();
var allPropertiesExceptKeyAndComputed =
GetAllPropertiesExceptKeyAndComputed(allProperties, keyProperties, explicitKeyProperties, computedProperties);

var keyPropertiesString = GetColumnsStringSqlServer(keyProperties,columns);
var keyPropertiesString = GetColumnsStringSqlServer(keyProperties, columns);
var keyPropertiesInsertedString = GetColumnsStringSqlServer(keyProperties, columns, "inserted.");
var allPropertiesExceptKeyAndComputedString = GetColumnsStringSqlServer(allPropertiesExceptKeyAndComputed, columns);
var allPropertiesExceptKeyAndComputedString =
GetColumnsStringSqlServer(allPropertiesExceptKeyAndComputed, columns);
var allPropertiesString = GetColumnsStringSqlServer(allProperties, columns, "target.");

var tempToBeInserted = $"#TempInsert_{tableName}".Replace(".", string.Empty);
var tempInsertedWithIdentity = $"@TempInserted_{tableName}".Replace(".", string.Empty);

connection.Execute($"SELECT TOP 0 {allPropertiesExceptKeyAndComputedString} INTO {tempToBeInserted} FROM {FormatTableName(tableName)} target WITH(NOLOCK);", null, transaction);
connection.Execute(
$"SELECT TOP 0 {allPropertiesExceptKeyAndComputedString} INTO {tempToBeInserted} FROM {FormatTableName(tableName)} target WITH(NOLOCK);",
null, transaction);

using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.Default, transaction))
{
Expand All @@ -116,8 +136,8 @@ public static IEnumerable<T> BulkInsertAndSelect<T>(this SqlConnection connectio
bulkCopy.WriteToServer(ToDataTable(data, allPropertiesExceptKeyAndComputed).CreateDataReader());
}

var table = string.Join(", ", keyProperties.Select(k => $"[{k.Name }] bigint"));
var joinOn = string.Join(" AND ", keyProperties.Select(k => $"target.[{k.Name }] = ins.[{k.Name }]"));
var table = string.Join(", ", keyProperties.Select(k => $"[{k.Name}] bigint"));
var joinOn = string.Join(" AND ", keyProperties.Select(k => $"target.[{k.Name}] = ins.[{k.Name}]"));
return connection.Query<T>($@"
DECLARE {tempInsertedWithIdentity} TABLE ({table})
INSERT INTO {FormatTableName(tableName)}({allPropertiesExceptKeyAndComputedString})
Expand All @@ -139,27 +159,35 @@ public static IEnumerable<T> BulkInsertAndSelect<T>(this SqlConnection connectio
/// <param name="transaction">The transaction to run under, null (the default) if none</param>
/// <param name="batchSize">Number of bulk items inserted together, 0 (the default) if all</param>
/// <param name="bulkCopyTimeout">Number of seconds before bulk command execution timeout, 30 (the default)</param>
public static async Task BulkInsertAsync<T>(this SqlConnection connection, IEnumerable<T> data, SqlTransaction transaction = null, int batchSize = 0, int bulkCopyTimeout = 30)
public static async Task BulkInsertAsync<T>(this SqlConnection connection, IEnumerable<T> data,
SqlTransaction transaction = null, int batchSize = 0, int bulkCopyTimeout = 30)
{
var type = typeof(T);
var tableName = TableMapper.GetTableName(type);
var allProperties = PropertiesCache.TypePropertiesCache(type);
var keyProperties = PropertiesCache.KeyPropertiesCache(type);
var explicitKeyProperties = PropertiesCache.ExplicitKeyPropertiesCache(type);
var computedProperties = PropertiesCache.ComputedPropertiesCache(type);
var columns = PropertiesCache.GetColumnNamesCache(type);

var allPropertiesExceptKeyAndComputed = allProperties.Except(keyProperties.Union(computedProperties)).ToList();
var allPropertiesExceptKeyAndComputedString = GetColumnsStringSqlServer(allPropertiesExceptKeyAndComputed,columns);
var allPropertiesExceptKeyAndComputed =
GetAllPropertiesExceptKeyAndComputed(allProperties, keyProperties, explicitKeyProperties, computedProperties);

var allPropertiesExceptKeyAndComputedString =
GetColumnsStringSqlServer(allPropertiesExceptKeyAndComputed, columns);
var tempToBeInserted = $"#TempInsert_{tableName}".Replace(".", string.Empty);

await connection.ExecuteAsync($@"SELECT TOP 0 {allPropertiesExceptKeyAndComputedString} INTO {tempToBeInserted} FROM {FormatTableName(tableName)} target WITH(NOLOCK);", null, transaction);
await connection.ExecuteAsync(
$@"SELECT TOP 0 {allPropertiesExceptKeyAndComputedString} INTO {tempToBeInserted} FROM {FormatTableName(tableName)} target WITH(NOLOCK);",
null, transaction);

using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.Default, transaction))
{
bulkCopy.BulkCopyTimeout = bulkCopyTimeout;
bulkCopy.BatchSize = batchSize;
bulkCopy.DestinationTableName = tempToBeInserted;
await bulkCopy.WriteToServerAsync(ToDataTable(data, allPropertiesExceptKeyAndComputed).CreateDataReader());
await bulkCopy.WriteToServerAsync(ToDataTable(data, allPropertiesExceptKeyAndComputed)
.CreateDataReader());
}

await connection.ExecuteAsync($@"
Expand All @@ -179,12 +207,14 @@ await connection.ExecuteAsync($@"
/// <param name="batchSize">Number of bulk items inserted together, 0 (the default) if all</param>
/// <param name="bulkCopyTimeout">Number of seconds before bulk command execution timeout, 30 (the default)</param>
/// <returns>Inserted entities</returns>
public static async Task<IEnumerable<T>> BulkInsertAndSelectAsync<T>(this SqlConnection connection, IEnumerable<T> data, SqlTransaction transaction = null, int batchSize = 0, int bulkCopyTimeout = 30)
public static async Task<IEnumerable<T>> BulkInsertAndSelectAsync<T>(this SqlConnection connection,
IEnumerable<T> data, SqlTransaction transaction = null, int batchSize = 0, int bulkCopyTimeout = 30)
{
var type = typeof(T);
var tableName = TableMapper.GetTableName(type);
var allProperties = PropertiesCache.TypePropertiesCache(type);
var keyProperties = PropertiesCache.KeyPropertiesCache(type);
var explicitKeyProperties = PropertiesCache.ExplicitKeyPropertiesCache(type);
var computedProperties = PropertiesCache.ComputedPropertiesCache(type);
var columns = PropertiesCache.GetColumnNamesCache(type);

Expand All @@ -195,28 +225,33 @@ public static async Task<IEnumerable<T>> BulkInsertAndSelectAsync<T>(this SqlCon
return dataList;
}

var allPropertiesExceptKeyAndComputed = allProperties.Except(keyProperties.Union(computedProperties)).ToList();
var allPropertiesExceptKeyAndComputed =
GetAllPropertiesExceptKeyAndComputed(allProperties, keyProperties, explicitKeyProperties, computedProperties);

var keyPropertiesString = GetColumnsStringSqlServer(keyProperties,columns);
var keyPropertiesInsertedString = GetColumnsStringSqlServer(keyProperties,columns, "inserted.");
var allPropertiesExceptKeyAndComputedString = GetColumnsStringSqlServer(allPropertiesExceptKeyAndComputed,columns);
var keyPropertiesString = GetColumnsStringSqlServer(keyProperties, columns);
var keyPropertiesInsertedString = GetColumnsStringSqlServer(keyProperties, columns, "inserted.");
var allPropertiesExceptKeyAndComputedString =
GetColumnsStringSqlServer(allPropertiesExceptKeyAndComputed, columns);
var allPropertiesString = GetColumnsStringSqlServer(allProperties, columns, "target.");

var tempToBeInserted = $"#TempInsert_{tableName}".Replace(".", string.Empty);
var tempInsertedWithIdentity = $"@TempInserted_{tableName}".Replace(".", string.Empty);

await connection.ExecuteAsync($@"SELECT TOP 0 {allPropertiesExceptKeyAndComputedString} INTO {tempToBeInserted} FROM {FormatTableName(tableName)} target WITH(NOLOCK);", null, transaction);
await connection.ExecuteAsync(
$@"SELECT TOP 0 {allPropertiesExceptKeyAndComputedString} INTO {tempToBeInserted} FROM {FormatTableName(tableName)} target WITH(NOLOCK);",
null, transaction);

using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.Default, transaction))
{
bulkCopy.BulkCopyTimeout = bulkCopyTimeout;
bulkCopy.BatchSize = batchSize;
bulkCopy.DestinationTableName = tempToBeInserted;
await bulkCopy.WriteToServerAsync(ToDataTable(data, allPropertiesExceptKeyAndComputed).CreateDataReader());
await bulkCopy.WriteToServerAsync(ToDataTable(data, allPropertiesExceptKeyAndComputed)
.CreateDataReader());
}

var table = string.Join(", ", keyProperties.Select(k => $"[{k.Name }] bigint"));
var joinOn = string.Join(" AND ", keyProperties.Select(k => $"target.[{k.Name }] = ins.[{k.Name }]"));
var table = string.Join(", ", keyProperties.Select(k => $"[{k.Name}] bigint"));
var joinOn = string.Join(" AND ", keyProperties.Select(k => $"target.[{k.Name}] = ins.[{k.Name}]"));
return await connection.QueryAsync<T>($@"
DECLARE {tempInsertedWithIdentity} TABLE ({table})
INSERT INTO {FormatTableName(tableName)}({allPropertiesExceptKeyAndComputedString})
Expand All @@ -229,16 +264,19 @@ public static async Task<IEnumerable<T>> BulkInsertAndSelectAsync<T>(this SqlCon
DROP TABLE {tempToBeInserted};", null, transaction);
}

private static string GetColumnsStringSqlServer(IEnumerable<PropertyInfo> properties, IReadOnlyDictionary<string, string> columnNames, string tablePrefix = null)
private static string GetColumnsStringSqlServer(IEnumerable<PropertyInfo> properties,
IReadOnlyDictionary<string, string> columnNames, string tablePrefix = null)
{
if (tablePrefix == "target.")
{
return string.Join(", ", properties.Select(property => $"{tablePrefix}[{columnNames[property.Name]}] as [{property.Name}] "));
return string.Join(", ",
properties.Select(property =>
$"{tablePrefix}[{columnNames[property.Name]}] as [{property.Name}] "));
}

return string.Join(", ", properties.Select(property => $"{tablePrefix}[{columnNames[property.Name]}] "));
}

private static DataTable ToDataTable<T>(IEnumerable<T> data, IList<PropertyInfo> properties)
{
var typeCasts = new Type[properties.Count];
Expand All @@ -258,8 +296,9 @@ private static DataTable ToDataTable<T>(IEnumerable<T> data, IList<PropertyInfo>
for (var i = 0; i < properties.Count; i++)
{
// Nullable types are not supported.
var propertyNonNullType = Nullable.GetUnderlyingType(properties[i].PropertyType) ?? properties[i].PropertyType;
dataTable.Columns.Add(properties[i].Name, typeCasts[i] == null ? propertyNonNullType : typeCasts[i]);
var propertyNonNullType =
Nullable.GetUnderlyingType(properties[i].PropertyType) ?? properties[i].PropertyType;
dataTable.Columns.Add(properties[i].Name, typeCasts[i] == null ? propertyNonNullType : typeCasts[i]);
}

foreach (var item in data)
Expand Down Expand Up @@ -294,4 +333,4 @@ internal static string FormatTableName(string table)
return $"[{parts[0]}].[{parts[1]}]";
}
}
}
}