Skip to content

Commit 8d6642a

Browse files
Copilotllali
Copilot
and
llali
committed
Implement vector index support - add grammar, AST classes, and script generators
Co-authored-by: llali <17187348+llali@users.noreply.github.com>
1 parent 43a5080 commit 8d6642a

15 files changed

Lines changed: 366 additions & 0 deletions

‎SqlScriptDom/Parser/TSql/Ast.xml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4607,6 +4607,13 @@
46074607
<Member Name="ForJsonPaths" Type="StringLiteral" Collection="true" Summary="The JSON paths specified in the FOR clause. Optional may have zero elements."/>
46084608
<InheritedMember Name="IndexOptions" ContainerClass="IndexStatement" />
46094609
</Class>
4610+
<Class Name="CreateVectorIndexStatement" Base="IndexStatement" Summary="Represents the create vector index statement.">
4611+
<InheritedMember Name="Name" ContainerClass="IndexStatement" />
4612+
<InheritedMember Name="OnName" ContainerClass="IndexStatement" />
4613+
<Member Name="VectorColumn" Type="Identifier" Summary="The vector column for the index."/>
4614+
<InheritedMember Name="IndexOptions" ContainerClass="IndexStatement" />
4615+
<Member Name="OnFileGroupOrPartitionScheme" Type="FileGroupOrPartitionScheme" Summary="The filegroup or partition scheme. Might be null."/>
4616+
</Class>
46104617
<Class Name="WindowFrameClause" Summary="Represents the specification of window bounds for windowing aggregates.">
46114618
<Member Name="Top" Type="WindowDelimiter" Summary="Top boundary of the window."/>
46124619
<Member Name="Bottom" Type="WindowDelimiter" Summary="Bottom boundary of the window. Optional may be null."/>
@@ -4639,6 +4646,14 @@
46394646
<Member Name="Expression" Type="ScalarExpression" Summary="Can be an integer in the range 0 - 10080."/>
46404647
<Member Name="TimeUnit" Type="CompressionDelayTimeUnit" GenerateUpdatePositionInfoCall="false" Summary="The optional unit of measure for the compression delay (it can be MINUTE or MINUTES)."/>
46414648
</Class>
4649+
<Class Name="VectorMetricIndexOption" Base="IndexOption" Summary="Represents the vector metric index option.">
4650+
<InheritedClass Name="IndexOption" />
4651+
<Member Name="MetricType" Type="VectorMetricType" GenerateUpdatePositionInfoCall="false" Summary="The vector metric type (cosine, dot, euclidean)."/>
4652+
</Class>
4653+
<Class Name="VectorTypeIndexOption" Base="IndexOption" Summary="Represents the vector type index option.">
4654+
<InheritedClass Name="IndexOption" />
4655+
<Member Name="VectorType" Type="VectorIndexType" GenerateUpdatePositionInfoCall="false" Summary="The vector index type (DiskANN)."/>
4656+
</Class>
46424657
<Class Name="ExternalLibraryStatement" Abstract="true" Base="TSqlStatement" Summary="Represents common parts in create and alter external library statements.">
46434658
<Member Name="Name" Type="Identifier" Summary="The name of the external library."/>
46444659
<Member Name="Language" Type="StringLiteral" Summary="The language of this external library."/>

‎SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,7 @@ internal static class CodeGenerationSupporter
225225
internal const string CompressionDelay = "COMPRESSION_DELAY";
226226
internal const string CompressAllRowGroups = "COMPRESS_ALL_ROW_GROUPS";
227227
internal const string Concat = "CONCAT";
228+
internal const string Cosine = "COSINE";
228229
internal const string Configuration = "CONFIGURATION";
229230
internal const string ConnectionOptions = "CONNECTION_OPTIONS";
230231
internal const string Contained = "CONTAINED";
@@ -299,6 +300,8 @@ internal static class CodeGenerationSupporter
299300
internal const string Dependents = "DEPENDENTS";
300301
internal const string Description = "DESCRIPTION";
301302
internal const string DesiredState = "DESIRED_STATE";
303+
internal const string DiskANN = "DISKANN";
304+
internal const string Dot = "DOT";
302305
internal const string Delay = "DELAY";
303306
internal const string DelayedDurability = "DELAYED_DURABILITY";
304307
internal const string DelimitedText = "DELIMITEDTEXT";
@@ -347,6 +350,7 @@ internal static class CodeGenerationSupporter
347350
internal const string EnvironmentVariables = "ENVIRONMENT_VARIABLES";
348351
internal const string Equal = "=";
349352
internal const string Error = "ERROR";
353+
internal const string Euclidean = "EUCLIDEAN";
350354
internal const string ErrorBrokerConversations = "ERROR_BROKER_CONVERSATIONS";
351355
internal const string ErrorDataSource = "ERRORFILE_DATA_SOURCE";
352356
internal const string ErrorFile = "ERRORFILE";
@@ -510,6 +514,7 @@ internal static class CodeGenerationSupporter
510514
internal const string Json = "JSON";
511515
internal const string JsonArray = "JSON_ARRAY";
512516
internal const string JsonObject = "JSON_OBJECT";
517+
internal const string Vector = "VECTOR";
513518
internal const string Keep = "KEEP";
514519
internal const string KeepDefaults = "KEEPDEFAULTS";
515520
internal const string KeepFixed = "KEEPFIXED";
@@ -612,6 +617,7 @@ internal static class CodeGenerationSupporter
612617
internal const string MessageForwarding = "MESSAGE_FORWARDING";
613618
internal const string MessageForwardSize = "MESSAGE_FORWARD_SIZE";
614619
internal const string MigrationState = "MIGRATION_STATE";
620+
internal const string Metric = "METRIC";
615621
internal const string Min = "MIN";
616622
internal const string MinGrantPercent = "MIN_GRANT_PERCENT";
617623
internal const string MinCpuPercent = "MIN_CPU_PERCENT";

‎SqlScriptDom/Parser/TSql/IndexOptionHelper.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ private IndexOptionHelper()
4343

4444
AddOptionMapping(IndexOptionKind.XmlCompression, CodeGenerationSupporter.XmlCompression, SqlVersionFlags.TSql160AndAbove);
4545

46+
AddOptionMapping(IndexOptionKind.VectorMetric, CodeGenerationSupporter.Metric, SqlVersionFlags.TSql170AndAbove);
47+
AddOptionMapping(IndexOptionKind.VectorType, CodeGenerationSupporter.Type, SqlVersionFlags.TSql170AndAbove);
48+
4649
}
4750

4851
internal static readonly IndexOptionHelper Instance = new IndexOptionHelper();

‎SqlScriptDom/Parser/TSql/IndexOptionKind.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ public enum IndexOptionKind
3838
WaitAtLowPriority = 21,
3939
OptimizeForSequentialKey = 22,
4040
XmlCompression = 23,
41+
VectorMetric = 24,
42+
VectorType = 25,
4143
}
4244

4345
#pragma warning restore 1591

‎SqlScriptDom/Parser/TSql/TSql170.g‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -886,6 +886,9 @@ create2005Statements returns [TSqlStatement vResult = null]
886886
|
887887
{NextTokenMatches(CodeGenerationSupporter.Json)}?
888888
vResult=createJsonIndexStatement[null, null]
889+
|
890+
{NextTokenMatches(CodeGenerationSupporter.Vector)}?
891+
vResult=createVectorIndexStatement[null, null]
889892
|
890893
{NextTokenMatches(CodeGenerationSupporter.Contract)}?
891894
vResult=createContractStatement
@@ -16848,6 +16851,7 @@ createIndexStatement returns [TSqlStatement vResult = null]
1684816851
vResult=createRelationalIndexStatement[tUnique, isClustered]
1684916852
| vResult=createColumnStoreIndexStatement[tUnique, isClustered]
1685016853
| vResult=createJsonIndexStatement[tUnique, isClustered]
16854+
| vResult=createVectorIndexStatement[tUnique, isClustered]
1685116855
)
1685216856
)
1685316857
|
@@ -17036,6 +17040,50 @@ createJsonIndexStatement [IToken tUnique, bool? isClustered] returns [CreateJson
1703617040
)?
1703717041
;
1703817042

17043+
createVectorIndexStatement [IToken tUnique, bool? isClustered] returns [CreateVectorIndexStatement vResult = FragmentFactory.CreateFragment<CreateVectorIndexStatement>()]
17044+
{
17045+
Identifier vIdentifier;
17046+
SchemaObjectName vSchemaObjectName;
17047+
Identifier vVectorColumn;
17048+
FileGroupOrPartitionScheme vFileGroupOrPartitionScheme;
17049+
17050+
if (tUnique != null)
17051+
{
17052+
ThrowIncorrectSyntaxErrorException(tUnique);
17053+
}
17054+
if (isClustered.HasValue)
17055+
{
17056+
ThrowIncorrectSyntaxErrorException(LT(1));
17057+
}
17058+
}
17059+
: tVector:Identifier tIndex:Index vIdentifier=identifier
17060+
{
17061+
Match(tVector, CodeGenerationSupporter.Vector);
17062+
vResult.Name = vIdentifier;
17063+
}
17064+
tOn:On vSchemaObjectName=schemaObjectThreePartName
17065+
{
17066+
vResult.OnName = vSchemaObjectName;
17067+
}
17068+
LeftParenthesis vVectorColumn=identifier tRParen:RightParenthesis
17069+
{
17070+
vResult.VectorColumn = vVectorColumn;
17071+
UpdateTokenInfo(vResult, tRParen);
17072+
}
17073+
(
17074+
// Greedy due to conflict with withCommonTableExpressionsAndXmlNamespaces
17075+
options {greedy = true; } :
17076+
With
17077+
indexOptionList[IndexAffectingStatement.CreateIndex, vResult.IndexOptions, vResult]
17078+
)?
17079+
(
17080+
On vFileGroupOrPartitionScheme=filegroupOrPartitionScheme
17081+
{
17082+
vResult.OnFileGroupOrPartitionScheme = vFileGroupOrPartitionScheme;
17083+
}
17084+
)?
17085+
;
17086+
1703917087
indexKeyColumnList[CreateIndexStatement vParent]
1704017088
{
1704117089
ColumnWithSortOrder vColumnWithSortOrder;
@@ -17680,6 +17728,12 @@ indexOption returns [IndexOption vResult = null]
1768017728
|
1768117729
{NextTokenMatches(CodeGenerationSupporter.WaitAtLowPriority)}?
1768217730
vResult=waitAtLowPriorityOption
17731+
|
17732+
{NextTokenMatches(CodeGenerationSupporter.Metric)}?
17733+
vResult=vectorMetricOption
17734+
|
17735+
{NextTokenMatches(CodeGenerationSupporter.Type)}?
17736+
vResult=vectorTypeOption
1768317737
|
1768417738
vResult=indexStateOption
1768517739
;
@@ -27669,6 +27723,26 @@ xmlCompressionOption returns [XmlCompressionOption vResult = FragmentFactory.Cre
2766927723
)?
2767027724
;
2767127725

27726+
vectorMetricOption returns [VectorMetricIndexOption vResult = FragmentFactory.CreateFragment<VectorMetricIndexOption>()]
27727+
: tMetric:Identifier EqualsSign tMetricValue:StringLiteral
27728+
{
27729+
Match(tMetric, CodeGenerationSupporter.Metric);
27730+
vResult.OptionKind = IndexOptionKind.VectorMetric;
27731+
vResult.MetricType = VectorMetricTypeHelper.Instance.ParseOption(tMetricValue);
27732+
UpdateTokenInfo(vResult, tMetricValue);
27733+
}
27734+
;
27735+
27736+
vectorTypeOption returns [VectorTypeIndexOption vResult = FragmentFactory.CreateFragment<VectorTypeIndexOption>()]
27737+
: tType:Identifier EqualsSign tTypeValue:StringLiteral
27738+
{
27739+
Match(tType, CodeGenerationSupporter.Type);
27740+
vResult.OptionKind = IndexOptionKind.VectorType;
27741+
vResult.VectorType = VectorIndexTypeHelper.Instance.ParseOption(tTypeValue);
27742+
UpdateTokenInfo(vResult, tTypeValue);
27743+
}
27744+
;
27745+
2767227746
compressionPartitionRange returns [CompressionPartitionRange vResult = FragmentFactory.CreateFragment<CompressionPartitionRange>()]
2767327747
{
2767427748
ScalarExpression vExpression;

‎SqlScriptDom/Parser/TSql/VectorIndexType.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
//------------------------------------------------------------------------------
2+
// <copyright file="VectorIndexType.cs" company="Microsoft">
3+
// Copyright (c) Microsoft Corporation. All rights reserved.
4+
// </copyright>
5+
//------------------------------------------------------------------------------
6+
7+
8+
9+
namespace Microsoft.SqlServer.TransactSql.ScriptDom
10+
{
11+
#pragma warning disable 1591
12+
13+
/// <summary>
14+
/// The possible values for vector index type
15+
/// </summary>
16+
public enum VectorIndexType
17+
{
18+
DiskANN = 0
19+
}
20+
21+
#pragma warning restore 1591
22+
}
23+

‎SqlScriptDom/Parser/TSql/VectorIndexTypeHelper.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
//------------------------------------------------------------------------------
2+
// <copyright file="VectorIndexTypeHelper.cs" company="Microsoft">
3+
// Copyright (c) Microsoft Corporation. All rights reserved.
4+
// </copyright>
5+
//------------------------------------------------------------------------------
6+
7+
8+
9+
namespace Microsoft.SqlServer.TransactSql.ScriptDom
10+
{
11+
12+
internal class VectorIndexTypeHelper : OptionsHelper<VectorIndexType>
13+
{
14+
private VectorIndexTypeHelper()
15+
{
16+
AddOptionMapping(VectorIndexType.DiskANN, CodeGenerationSupporter.DiskANN);
17+
}
18+
19+
public static readonly VectorIndexTypeHelper Instance = new VectorIndexTypeHelper();
20+
}
21+
}

‎SqlScriptDom/Parser/TSql/VectorMetricType.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
//------------------------------------------------------------------------------
2+
// <copyright file="VectorMetricType.cs" company="Microsoft">
3+
// Copyright (c) Microsoft Corporation. All rights reserved.
4+
// </copyright>
5+
//------------------------------------------------------------------------------
6+
7+
8+
9+
namespace Microsoft.SqlServer.TransactSql.ScriptDom
10+
{
11+
#pragma warning disable 1591
12+
13+
/// <summary>
14+
/// The possible values for vector index metric
15+
/// </summary>
16+
public enum VectorMetricType
17+
{
18+
Cosine = 0,
19+
Dot = 1,
20+
Euclidean = 2
21+
}
22+
23+
#pragma warning restore 1591
24+
}
25+

‎SqlScriptDom/Parser/TSql/VectorMetricTypeHelper.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
//------------------------------------------------------------------------------
2+
// <copyright file="VectorMetricTypeHelper.cs" company="Microsoft">
3+
// Copyright (c) Microsoft Corporation. All rights reserved.
4+
// </copyright>
5+
//------------------------------------------------------------------------------
6+
7+
8+
9+
namespace Microsoft.SqlServer.TransactSql.ScriptDom
10+
{
11+
12+
internal class VectorMetricTypeHelper : OptionsHelper<VectorMetricType>
13+
{
14+
private VectorMetricTypeHelper()
15+
{
16+
AddOptionMapping(VectorMetricType.Cosine, CodeGenerationSupporter.Cosine);
17+
AddOptionMapping(VectorMetricType.Dot, CodeGenerationSupporter.Dot);
18+
AddOptionMapping(VectorMetricType.Euclidean, CodeGenerationSupporter.Euclidean);
19+
}
20+
21+
public static readonly VectorMetricTypeHelper Instance = new VectorMetricTypeHelper();
22+
}
23+
}

‎SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.CreateVectorIndexStatement.cs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
//------------------------------------------------------------------------------
2+
// <copyright file="SqlScriptGeneratorVisitor.CreateVectorIndexStatement.cs" company="Microsoft">
3+
// Copyright (c) Microsoft Corporation. All rights reserved.
4+
// </copyright>
5+
//------------------------------------------------------------------------------
6+
using System.Collections.Generic;
7+
using Microsoft.SqlServer.TransactSql.ScriptDom;
8+
9+
namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator
10+
{
11+
partial class SqlScriptGeneratorVisitor
12+
{
13+
public override void ExplicitVisit(CreateVectorIndexStatement node)
14+
{
15+
GenerateKeyword(TSqlTokenType.Create);
16+
17+
GenerateSpaceAndIdentifier(CodeGenerationSupporter.Vector);
18+
19+
GenerateSpaceAndKeyword(TSqlTokenType.Index);
20+
21+
// name
22+
GenerateSpaceAndFragmentIfNotNull(node.Name);
23+
24+
NewLineAndIndent();
25+
GenerateKeyword(TSqlTokenType.On);
26+
GenerateSpaceAndFragmentIfNotNull(node.OnName);
27+
28+
// Vector column
29+
if (node.VectorColumn != null)
30+
{
31+
GenerateSpace();
32+
GenerateSymbol(TSqlTokenType.LeftParenthesis);
33+
GenerateFragmentIfNotNull(node.VectorColumn);
34+
GenerateSymbol(TSqlTokenType.RightParenthesis);
35+
}
36+
37+
GenerateIndexOptions(node.IndexOptions);
38+
39+
if (node.OnFileGroupOrPartitionScheme != null)
40+
{
41+
NewLineAndIndent();
42+
GenerateKeyword(TSqlTokenType.On);
43+
44+
GenerateSpaceAndFragmentIfNotNull(node.OnFileGroupOrPartitionScheme);
45+
}
46+
}
47+
}
48+
}

0 commit comments

Comments
 (0)