Skip to content

Commit

Permalink
Updated ADF Walkthrough Samples
Browse files Browse the repository at this point in the history 
Update JSON files to the latest format.
  • Loading branch information
Wee Hyong Tok committed Jul 21, 2015
1 parent 2a087d2 commit 3814571
Show file tree
Hide file tree
Showing 17 changed files with 720 additions and 473 deletions.
13 changes: 7 additions & 6 deletions Samples/ADFWalkthrough/LinkedServices/AzureSqlLinkedService.json
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
{
"name": "AzureSqlLinkedService",
"properties":
{
"type": "AzureSqlLinkedService",
"connectionString": "Server=tcp:<azure sql server>.database.windows.net,1433;Database=MarketingCampaigns;User ID=<user name>;Password=<password>;Trusted_Connection=False;Encrypt=True;Connection Timeout=30"
"name": "AzureSqlLinkedService",
"properties": {
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": "Server=tcp:<azure sql server>.database.windows.net,1433;Database=MarketingCampaigns;User ID=<user name>;Password=<password>;Trusted_Connection=False;Encrypt=True;Connection Timeout=30"
}
}
}
}
17 changes: 9 additions & 8 deletions Samples/ADFWalkthrough/LinkedServices/HDInsightLinkedService.json
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
{
"name": "HDInsightLinkedService",
"properties":
{
"type": "HDInsightOnDemandLinkedService",
"clusterSize": "4",
"jobsContainer": "adfjobs",
"timeToLive": "00:05:00",
"linkedServiceName": "HDInsightStorageLinkedService"
"name": "HDInsightLinkedService",
"properties": {
"type": "HDInsightOnDemand",
"typeProperties": {
"clusterSize": "4",
"jobsContainer": "adfjobs",
"timeToLive": "00:05:00",
"linkedServiceName": "HDInsightStorageLinkedService"
}
}
}
11 changes: 6 additions & 5 deletions Samples/ADFWalkthrough/LinkedServices/HDInsightStorageLinkedService.json
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
{
"name": "HDInsightStorageLinkedService",
"properties":
{
"type": "AzureStorageLinkedService",
"connectionString": "DefaultEndpointsProtocol=https;AccountName=<account name>;AccountKey=<account key>"
"name": "HDInsightStorageLinkedService",
"properties": {
"type": "AzureStorage",
"typeProperties": {
"connectionString": "DefaultEndpointsProtocol=https;AccountName=<account name>;AccountKey=<account key>"
}
}
}
13 changes: 7 additions & 6 deletions Samples/ADFWalkthrough/LinkedServices/StorageLinkedService.json
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
{
"name": "StorageLinkedService",
"properties":
{
"type": "AzureStorageLinkedService",
"connectionString": "DefaultEndpointsProtocol=https;AccountName=<account name>;AccountKey=<Account key>"
"name": "StorageLinkedService",
"properties": {
"type": "AzureStorage",
"typeProperties": {
"connectionString": "DefaultEndpointsProtocol=https;AccountName=<account name>;AccountKey=<Account key>"
}
}
}
}
76 changes: 39 additions & 37 deletions Samples/ADFWalkthrough/OnPremises/EgressDataToOnPremPipeline.json
View file
Original file line number Diff line number Diff line change
@@ -1,38 +1,40 @@
{
"name": "EgressDataToOnPremPipeline",
"properties":
{
"description" : "Egress data to on-premises SQL Server (v1.0)",
"activities":
[
{
"name": "EgressDataToSqlServer",
"description": "Push Regional Effectiveness Campaign data to On-Premises SQL Server",
"type": "CopyActivity",
"inputs": [ {"name": "MarketingCampaignEffectivenessBlobTable"} ],
"outputs": [ {"name": "MarketingCampaignEffectivenessOnPremSQLTable"} ],
"transformation":
{
"source":
{
"type": "BlobSource"
},
"sink":
{
"type": "SqlSink",
"SqlWriterTableType": "MarketingCampaignEffectivenessType",
"SqlWriterStoredProcedureName": "spEgressOverwriteMarketingCampaignEffectiveness"
}
},
"Policy":
{
"concurrency": 1,
"executionPriorityOrder": "NewestFirst",
"style": "StartOfInterval",
"retry": 0,
"timeout": "01:00:00"
}
}
]
}
}
"name": "EgressDataToOnPremPipeline",
"properties": {
"description": "Egress data to on-premises SQL Server (v1.0)",
"activities": [
{
"name": "EgressDataToSqlServer",
"description": "Push Regional Effectiveness Campaign data to On-Premises SQL Server",
"type": "Copy",
"inputs": [
{
"name": "MarketingCampaignEffectivenessBlobTable"
}
],
"outputs": [
{
"name": "MarketingCampaignEffectivenessOnPremSQLTable"
}
],
"typeProperties": {
"source": {
"type": "BlobSource"
},
"sink": {
"type": "SqlSink",
"SqlWriterTableType": "MarketingCampaignEffectivenessType",
"SqlWriterStoredProcedureName": "spEgressOverwriteMarketingCampaignEffectiveness"
}
},
"Policy": {
"concurrency": 1,
"executionPriorityOrder": "NewestFirst",
"style": "StartOfInterval",
"retry": 0,
"timeout": "01:00:00"
}
}
]
}
}
85 changes: 57 additions & 28 deletions Samples/ADFWalkthrough/OnPremises/MarketingCampaignEffectivenessOnPremSQLTable.json
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,60 @@
{
"name": "MarketingCampaignEffectivenessOnPremSQLTable",
"properties":
{
"structure":
[
{ "name": "ProfileID", "type": "String"},
{ "name": "SessionStart", "type": "String"},
{ "name": "Duration", "type": "Int"},
{ "name": "State", "type": "String"},
{ "name": "SrcIPAddress", "type": "String"},
{ "name": "GameType", "type": "String"},
{ "name": "Multiplayer", "type": "String"},
{ "name": "EndRank", "type": "Int"},
{ "name": "WeaponsUsed", "type": "Int"},
{ "name": "UsersInteractedWith", "type": "String"},
{ "name": "Impressions", "type": "String"}
],
"location":
{
"type": "OnPremisesSqlServerTableLocation",
"tableName": "MarketingCampaignEffectiveness",
"linkedServiceName": "OnPremSqlServerLinkedService"
},
"availability":
{
"frequency": "Day",
"interval": 1
}
"name": "MarketingCampaignEffectivenessOnPremSQLTable",
"properties": {
"structure": [
{
"name": "ProfileID",
"type": "String"
},
{
"name": "SessionStart",
"type": "String"
},
{
"name": "Duration",
"type": "Int"
},
{
"name": "State",
"type": "String"
},
{
"name": "SrcIPAddress",
"type": "String"
},
{
"name": "GameType",
"type": "String"
},
{
"name": "Multiplayer",
"type": "String"
},
{
"name": "EndRank",
"type": "Int"
},
{
"name": "WeaponsUsed",
"type": "Int"
},
{
"name": "UsersInteractedWith",
"type": "String"
},
{
"name": "Impressions",
"type": "String"
}
],
"type": "SqlServerTable",
"linkedServiceName": "OnPremSqlServerLinkedService",
"typeProperties": {
"tableName": "MarketingCampaignEffectiveness"
},
"availability": {
"frequency": "Day",
"interval": 1
}
}
}
13 changes: 7 additions & 6 deletions Samples/ADFWalkthrough/OnPremises/OnPremSqlServerLinkedService.json
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
{
"Name": "OnPremSqlServerLinkedService",
"Properties":
{
"type": "OnPremisesSqlLinkedService",
"ConnectionString": "Data Source=<servername>;Initial Catalog=MarketingCampaigns;Integrated Security=False;User ID=<username>;Password=<password>;",
"gatewayName": "<gatewayname>"
"properties": {
"type": "OnPremisesSqlServer",
"typeProperties": {
"ConnectionString": "Data Source=<servername>;Initial Catalog=MarketingCampaigns;Integrated Security=False;User ID=<username>;Password=<password>;",
"gatewayName": "<gatewayname>"
}
}
}
}
135 changes: 72 additions & 63 deletions Samples/ADFWalkthrough/Pipelines/AnalyzeMarketingCampaignPipeline.json
View file
Original file line number Diff line number Diff line change
@@ -1,65 +1,74 @@
{
"name": "AnalyzeMarketingCampaignPipeline",
"properties":
{
"description" : "(v1.0) Join the Regional Campaign data and with Enriched Gamer Fact Data and push to Azure SQLD Database",
"activities":
[
{
"name": "JoinData",
"description": "Join Regional Campaign data with Enriched Gamer Fact Data",
"type": "HDInsightActivity",
"inputs": [ {"name": "EnrichedGameEventsTable"}, {"name": "RefMarketingCampaignTable"} ],
"outputs": [ {"name": "MarketingCampaignEffectivenessBlobTable"} ],
"linkedServiceName": "HDInsightLinkedService",
"transformation":
{
"type": "Hive",
"extendedProperties":
{
"EventsInput": "$$Text.Format('wasb://adfwalkthrough@<storageaccountname>.blob.core.windows.net/logs/enrichedgameevents/yearno={0:yyyy}/monthno={0:%M}/dayno={0:%d}/', SliceStart)",
"CampaignInput": "wasb://adfwalkthrough@<storageaccountname>.blob.core.windows.net/refdata/refmarketingcampaign/",
"CampaignOutput": "$$Text.Format('wasb://adfwalkthrough@<storageaccountname>.blob.core.windows.net/marketingcampaigneffectiveness/yearno={0:yyyy}/monthno={0:%M}/dayno={0:%d}/', SliceStart)"
},
"scriptpath": "adfwalkthrough\\scripts\\transformdata.hql",
"scriptLinkedService": "StorageLinkedService"
},
"policy":
{
"concurrency": 1,
"executionPriorityOrder": "NewestFirst",
"retry": 1,
"timeout": "01:00:00"
}
},
{
"name": "EgressDataAzure",
"description": "Push Regional Effectiveness Campaign data to Sql Azure",
"type": "CopyActivity",
"inputs": [ {"name": "MarketingCampaignEffectivenessBlobTable"} ],
"outputs": [ {"name": "MarketingCampaignEffectivenessSQLTable"} ],
"transformation":
{
"source":
{
"type": "BlobSource"
},
"sink":
{
"type": "SqlSink",
"SqlWriterTableType": "MarketingCampaignEffectivenessType",
"SqlWriterStoredProcedureName": "spEgressOverwriteMarketingCampaignEffectiveness"
}
},
"Policy":
{
"concurrency": 1,
"executionPriorityOrder": "NewestFirst",
"style": "StartOfInterval",
"retry": 0,
"timeout": "01:00:00"
}
}
]
"name": "AnalyzeMarketingCampaignPipeline",
"properties": {
"description": "(v1.0) Join the Regional Campaign data and with Enriched Gamer Fact Data and push to Azure SQLD Database",
"activities": [
{
"name": "JoinData",
"description": "Join Regional Campaign data with Enriched Gamer Fact Data",
"inputs": [
{
"name": "EnrichedGameEventsTable"
},
{
"name": "RefMarketingCampaignTable"
}
],
"outputs": [
{
"name": "MarketingCampaignEffectivenessBlobTable"
}
],
"linkedServiceName": "HDInsightLinkedService",
"type": "HDInsightHive",
"typeProperties": {
"scriptpath": "adfwalkthrough\\scripts\\transformdata.hql",
"scriptLinkedService": "StorageLinkedService",
"defines": {
"EventsInput": "$$Text.Format('wasb://adfwalkthrough@<storageaccountname>.blob.core.windows.net/logs/enrichedgameevents/yearno={0:yyyy}/monthno={0:%M}/dayno={0:%d}/', SliceStart)",
"CampaignInput": "wasb://adfwalkthrough@<storageaccountname>.blob.core.windows.net/refdata/refmarketingcampaign/",
"CampaignOutput": "$$Text.Format('wasb://adfwalkthrough@<storageaccountname>.blob.core.windows.net/marketingcampaigneffectiveness/yearno={0:yyyy}/monthno={0:%M}/dayno={0:%d}/', SliceStart)"
}
},
"policy": {
"concurrency": 1,
"executionPriorityOrder": "NewestFirst",
"retry": 1,
"timeout": "01:00:00"
}
},
{
"name": "EgressDataAzure",
"description": "Push Regional Effectiveness Campaign data to Sql Azure",
"type": "Copy",
"inputs": [
{
"name": "MarketingCampaignEffectivenessBlobTable"
}
],
"outputs": [
{
"name": "MarketingCampaignEffectivenessSQLTable"
}
],
"typeProperties": {
"source": {
"type": "BlobSource"
},
"sink": {
"type": "SqlSink",
"SqlWriterTableType": "MarketingCampaignEffectivenessType",
"SqlWriterStoredProcedureName": "spEgressOverwriteMarketingCampaignEffectiveness"
}
},
"Policy": {
"concurrency": 1,
"executionPriorityOrder": "NewestFirst",
"style": "StartOfInterval",
"retry": 0,
"timeout": "01:00:00"
}
}
}
]
}
}
Loading

0 comments on commit 3814571

Please sign in to comment.