Azure-DataEngineering/Azure DataFactory/DataFactory Repo/dataflow/DataflowPipeline_for_FactOnlinePurchase.json at main · TheDataArtisanDev/Azure-DataEngineering · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
{
	"name": "DataflowPipeline_for_FactOnlinePurchase",
	"properties": {
		"type": "MappingDataFlow",
		"typeProperties": {
			"sources": [
				{
					"dataset": {
						"referenceName": "DS_AzureDB_tOnlinePurchase",
						"type": "DatasetReference"
					},
					"name": "source1"
				},
				{
					"dataset": {
						"referenceName": "LS_Azure_DB_DATAMART_DimOnlineService",
						"type": "DatasetReference"
					},
					"name": "source2"
				},
				{
					"dataset": {
						"referenceName": "LS_Azure_DB_DATAMART_DimMovie",
						"type": "DatasetReference"
					},
					"name": "source3"
				}
			],
			"sinks": [
				{
					"dataset": {
						"referenceName": "DS_AzureDB_FactOnlinePurchase",
						"type": "DatasetReference"
					},
					"name": "sink1"
				}
			],
			"transformations": [
				{
					"name": "join1"
				},
				{
					"name": "join2"
				},
				{
					"name": "derivedColumn1"
				}
			],
			"scriptLines": [
				"source(output(",
				"          Id as integer,",
				"          ServiceId as integer,",
				"          MovieId as integer,",
				"          UserId as integer,",
				"          Price as decimal(4,1),",
				"          TransactionId as string,",
				"          TransactionDate as timestamp,",
				"          CreatedDate as timestamp,",
				"          ModifiedDate as timestamp,",
				"          SrcFileName as string",
				"     ),",
				"     allowSchemaDrift: true,",
				"     validateSchema: false,",
				"     isolationLevel: 'READ_UNCOMMITTED',",
				"     format: 'table') ~> source1",
				"source(output(",
				"          ServiceKey as integer,",
				"          ServiceCode as string,",
				"          ServiceName as string,",
				"          CreatedDate as timestamp,",
				"          ModifiedDate as timestamp",
				"     ),",
				"     allowSchemaDrift: true,",
				"     validateSchema: false,",
				"     isolationLevel: 'READ_UNCOMMITTED',",
				"     format: 'table') ~> source2",
				"source(output(",
				"          MovieKey as integer,",
				"          MovieIdNK as integer,",
				"          Title as string,",
				"          ReleaseDate as date,",
				"          Runtime as integer,",
				"          MovieStatus as string,",
				"          CreatedDate as timestamp,",
				"          ModifiedDate as timestamp",
				"     ),",
				"     allowSchemaDrift: true,",
				"     validateSchema: false,",
				"     isolationLevel: 'READ_UNCOMMITTED',",
				"     format: 'table') ~> source3",
				"source1, source2 join(ServiceId == ServiceKey,",
				"     joinType:'inner',",
				"     matchType:'exact',",
				"     ignoreSpaces: false,",
				"     broadcast: 'auto')~> join1",
				"source3, join1 join(MovieIdNK == MovieId,",
				"     joinType:'inner',",
				"     matchType:'exact',",
				"     ignoreSpaces: false,",
				"     broadcast: 'auto')~> join2",
				"join2 derive(TransactionDate = toString(TransactionDate, 'yyyyMMdd')) ~> derivedColumn1",
				"derivedColumn1 sink(allowSchemaDrift: true,",
				"     validateSchema: false,",
				"     input(",
				"          DateKey as integer,",
				"          OnlineServiceKey as integer,",
				"          MovieKey as integer,",
				"          Price as decimal(4,1),",
				"          TransactionId as string",
				"     ),",
				"     deletable:false,",
				"     insertable:true,",
				"     updateable:false,",
				"     upsertable:false,",
				"     format: 'table',",
				"     preSQLs:['truncate table [dbo].[FactOnlinePurchase]'],",
				"     skipDuplicateMapInputs: true,",
				"     skipDuplicateMapOutputs: true,",
				"     errorHandlingOption: 'stopOnFirstError',",
				"     mapColumn(",
				"          DateKey = TransactionDate,",
				"          OnlineServiceKey = ServiceKey,",
				"          MovieKey,",
				"          Price,",
				"          TransactionId",
				"     )) ~> sink1"
			]
		}
	}
}