Azure-DataEngineering/Azure DataFactory/DataFactory Repo/dataflow/DataflowPipeline_for_DimOnlineService.json at main · TheDataArtisanDev/Azure-DataEngineering · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
{
	"name": "DataflowPipeline_for_DimOnlineService",
	"properties": {
		"type": "MappingDataFlow",
		"typeProperties": {
			"sources": [
				{
					"dataset": {
						"referenceName": "DS_AzureDB_tOnlineService",
						"type": "DatasetReference"
					},
					"name": "StreamTOnlineServiceInput"
				}
			],
			"sinks": [
				{
					"dataset": {
						"referenceName": "LS_Azure_DB_DATAMART_DimOnlineService",
						"type": "DatasetReference"
					},
					"name": "StreamtOnlineServiceOutput"
				}
			],
			"transformations": [
				{
					"name": "StreamDerivedColumn"
				}
			],
			"scriptLines": [
				"source(output(",
				"          ServiceId as integer,",
				"          ServiceCode as string,",
				"          ServiceName as string,",
				"          CreatedDate as timestamp,",
				"          ModifiedDate as timestamp",
				"     ),",
				"     allowSchemaDrift: true,",
				"     validateSchema: false,",
				"     isolationLevel: 'READ_UNCOMMITTED',",
				"     format: 'table') ~> StreamTOnlineServiceInput",
				"StreamTOnlineServiceInput derive(CreatedDate = currentUTC(),",
				"          ModifiedDate = currentUTC()) ~> StreamDerivedColumn",
				"StreamDerivedColumn sink(allowSchemaDrift: true,",
				"     validateSchema: false,",
				"     input(",
				"          ServiceKey as integer,",
				"          ServiceCode as string,",
				"          ServiceName as string,",
				"          CreatedDate as timestamp,",
				"          ModifiedDate as timestamp",
				"     ),",
				"     deletable:false,",
				"     insertable:true,",
				"     updateable:false,",
				"     upsertable:false,",
				"     format: 'table',",
				"     preSQLs:['truncate table [dbo].[DimOnlineService];'],",
				"     skipDuplicateMapInputs: true,",
				"     skipDuplicateMapOutputs: true,",
				"     errorHandlingOption: 'stopOnFirstError',",
				"     mapColumn(",
				"          ServiceCode,",
				"          ServiceName,",
				"          CreatedDate,",
				"          ModifiedDate",
				"     )) ~> StreamtOnlineServiceOutput"
			]
		}
	}
}