From 86084c528eb0f697415a7eab42f9eac655e5dd88 Mon Sep 17 00:00:00 2001 From: Bill Gibson Date: Mon, 8 May 2017 18:50:04 -0700 Subject: [PATCH] Scriptsmigrationfromold (#10) * Create placeholder * Baseline Replacement of Learning Modules folder from folder in -OLD repo * Complete prior commit * First pass updates to scripts to accommodate changes to new WTP application Updated Get-TenantKey to use MD5 and moved to CatalogAndDatabaseManagement Renamed AppVersionSpecific to WtpConfig and updated all uses Renamed Venues -> Venue in all SQL scripts Updated uses of venue types in provisioning scripts to use new values Changed email length validation for 50 chars Removed all tutorials Renamed EDJ user guide to remove 'private' * First check in of new ticket generator * Work in progress on new ticket generator * Updates to fix issues with Auto Provision and add the 'curved' version of the ticket generator * Update Demo-TicketGenerator script to enable use of the new 'curved' ticket generator * Updates to fix bugs in auto provisioning * Updates to fix provisioning bugs after migration * Minor updates to demo- scripts, user config, and ticket generator and sales curves * Refactor rename-database command to include sourcedatabasename and servername parameters * Updates to catalog module for restore tenant scenarios 1. Added No echo to scripts to prevent printing out subscription info twice 2. Replaced Remove-ExtendedTenantMetadataFromCatalog with Remove-ExtendedTenant 3. The default databases deployed still do not come with sp_deleteEvent installed * Adding updates to TicketGenerator2 to be consistent with -OLD * Removing AutoProvision from the release branch * Revert "Removing AutoProvision from the release branch" This reverts commit 1697ee5b95d22d420d9abf4aafc91d2f59dbdc47. * Updated operational analytics scripts to point to latest ticket generator * Also updated operational analytics scripts to use updated server name prefix ('tenants1' not 'customers1') * Updated database project to include sp_DeleteEvent * Updated to remove launch of Admin app whch is not available in this release * Removing reference to Databases table from Remove-ExtendedTenant * refactoring to make removal of extended database data occur in catalog sync only * Adding to previous commit * Adding updates to replicated copies of CatalogAndDatabaseManagement to AutoProvisioning folders * Fixed bug that mismatched server used in case of golden server; fixed source of bacpac used in deploy tenant * Add files via upload * Update Demo-TenantAnalyticsDB.ps1 * Create foo.txt * Add files via upload * Updates to ticket generator to fix bugs and make consistent with OLD version * Changing New-TenantBatch to use template from new release URL * Add support for Go statements in Invoke-SqlAzure command * Revert "Add support for Go statements in Invoke-SqlAzure command" This reverts commit ef6f17cf875c2650d2e2fb57768b045e5598fa30. * Addition of Search database info to WtpConfig * Addition of search head database scenario based on EQ * Corrections to venue types in SQL scripts in ref data update and in adhoc analytics db population * Updating Search-related scripts * Adding Server name to the data projected into the search database * Feature jobs private preview warning (#9) * Adding error message prompting users to download updated PowerShell SDK for elastic jobs APIs * Add error checking to check if subscription is white-listed for Elastic jobs preview * Adding TenantAnalyticsDW demo script files * Fixing extra tab and LF that breaks script * Updating behavior of InvokeSqlCmdOnTenantDatabases to include BaseTenantDB * Removing specific script from Demo-InvokeSqlCmdOnTenantDatabases * Fix latent bug in Get-Catalog * Changed LoadGenerator to run continuously, looking for new dbs to add to the current load * Tweaks to progress bar display behavior * Removing New-Tenant file, function is in CatalogAndDatabaseManagement * Suporting PostalCode input on single and batch tenants * Increasing progress bar to show 10 minutes rows (60 * 10 seconds) * Updated adhoc analytics queries to match those in -OLD * Updated comments in Tenant Analytics scripts * Changes to comments. Removes unused variable assignment * Update to readme with new linky thing. removed references to features not supported in this release. * Updated SQL files with TicketFacts and venue* 'global' views * Removing search module * updating postal codes used by ticket generator --- .../Common/CatalogAndDatabaseManagement.psm1 | 27 +- .../Adhoc Analytics/Demo-AdhocAnalytics.ps1 | 2 +- .../Demo-AdhocAnalyticsQueries.sql | 86 ++-- .../Deploy-AdhocAnalyticsDB.ps1 | 22 +- .../Copy-TenantDataToDataWarehouse.ps1 | 198 ++++++++ .../Demo-TenantAnalyticsDW.ps1 | 84 ++++ .../Demo-TenantAnalyticsDWQueries.sql | 10 + .../Deploy-TenantAnalyticsDW.ps1 | 50 ++ .../Tenant Analytics DW/foo.txt | 1 - .../Demo-TenantAnalyticsDB.ps1 | 108 +++-- .../Deploy-TenantAnalyticsDB-CS.ps1 | 92 ++++ .../TicketPurchasesfromAllTenants.sql | 152 +++--- .../TicketPurchasesfromAllTenantsCS.sql | 77 +++ .../Common/CatalogAndDatabaseManagement.psm1 | 18 +- .../Common/CatalogAndDatabaseManagement.psm1 | 18 +- .../Catalog Sync WebJob/Run.ps1 | 13 + .../Auto Provision/New-TenantRequest.ps1 | 6 +- .../Common/CatalogAndDatabaseManagement.psm1 | 18 +- .../Demo-ProvisionAndCatalog.ps1 | 7 +- .../Provision and Catalog/New-Tenant.ps1 | 84 ---- .../Provision and Catalog/New-TenantBatch.ps1 | 7 +- .../Schema Management/Deploy-JobAccount.ps1 | 39 +- .../Schema Management/DeployReferenceData.sql | 24 +- .../Demo-InvokeSqlCmdOnTenantDatabases.ps1 | 25 +- .../Utilities/Demo-LoadGenerator.ps1 | 14 +- .../Utilities/FictitiousNames.csv | 1 - .../Invoke-SqlCmdOnTenantDatabases.ps1 | 68 +-- Learning Modules/Utilities/LoadGenerator.ps1 | 458 ++++++++++-------- .../Utilities/SeattleZonedPostalCodes.csv | 28 +- .../Utilities/TicketGenerator2.ps1 | 91 ++-- Learning Modules/WtpConfig.psm1 | 43 +- README.md | 18 +- .../WingtipTenantDB/WingtipTenantDB.sqlproj | 1 + .../dbo/StoredProcedures/sp_DeleteEvent.sql | 22 + .../WingtipTenantDB/dbo/Views/TicketFacts.sql | 16 + .../WingtipTenantDB/dbo/Views/VenueEvents.sql | 2 + .../dbo/Views/VenueTicketPurchases.sql | 2 + .../dbo/Views/VenueTickets.sql | 2 + .../dbo/Views/VenuesGlobal.sql | 2 + 39 files changed, 1246 insertions(+), 690 deletions(-) create mode 100644 Learning Modules/Operational Analytics/Tenant Analytics DW/Copy-TenantDataToDataWarehouse.ps1 create mode 100644 Learning Modules/Operational Analytics/Tenant Analytics DW/Demo-TenantAnalyticsDW.ps1 create mode 100644 Learning Modules/Operational Analytics/Tenant Analytics DW/Demo-TenantAnalyticsDWQueries.sql create mode 100644 Learning Modules/Operational Analytics/Tenant Analytics DW/Deploy-TenantAnalyticsDW.ps1 delete mode 100644 Learning Modules/Operational Analytics/Tenant Analytics DW/foo.txt create mode 100644 Learning Modules/Operational Analytics/Tenant Analytics/Deploy-TenantAnalyticsDB-CS.ps1 create mode 100644 Learning Modules/Operational Analytics/Tenant Analytics/TicketPurchasesfromAllTenantsCS.sql delete mode 100644 Learning Modules/Provision and Catalog/New-Tenant.ps1 create mode 100644 WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/StoredProcedures/sp_DeleteEvent.sql create mode 100644 WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/TicketFacts.sql create mode 100644 WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenueEvents.sql create mode 100644 WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenueTicketPurchases.sql create mode 100644 WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenueTickets.sql create mode 100644 WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenuesGlobal.sql diff --git a/Learning Modules/Common/CatalogAndDatabaseManagement.psm1 b/Learning Modules/Common/CatalogAndDatabaseManagement.psm1 index 7657d1e..04efc72 100644 --- a/Learning Modules/Common/CatalogAndDatabaseManagement.psm1 +++ b/Learning Modules/Common/CatalogAndDatabaseManagement.psm1 @@ -173,7 +173,7 @@ function Get-Catalog if (!$shardmapManager) { - throw "Failed to initialize shard map manager from '$(config.CatalogDatabaseName)' database. Ensure catalog is initialized by opening the Events app and try again." + throw "Failed to initialize shard map manager from '$($config.CatalogDatabaseName)' database. Ensure catalog is initialized by opening the Events app and try again." } # Initialize shard map @@ -994,7 +994,11 @@ function New-Tenant [string]$PoolName, [Parameter(Mandatory=$false)] - [string]$VenueType + [string]$VenueType, + + [Parameter(Mandatory=$false)] + [string]$PostalCode = "98052" + ) $WtpUser = $WtpUser.ToLower() @@ -1024,6 +1028,7 @@ function New-Tenant -ElasticPoolName $PoolName ` -TenantName $TenantName ` -VenueType $VenueType ` + -PostalCode $PostalCode ` -WtpUser $WtpUser # Register the tenant and database in the catalog @@ -1216,7 +1221,6 @@ function Remove-CatalogInfoFromTenantDatabase -Password $adminPassword ` -ServerInstance ($TenantDatabase.ServerName + ".database.windows.net") ` -Database $TenantDatabase.DatabaseName ` - -ConnectionTimeout 30 ` -Query $commandText ` } @@ -1231,13 +1235,16 @@ function Remove-ExtendedDatabase [parameter(Mandatory=$true)] [object]$Catalog, + [parameter(Mandatory=$true)] + [string]$ServerName, + [parameter(Mandatory=$true)] [string]$DatabaseName ) $commandText = " DELETE FROM Databases - WHERE DatabaseName = $DatabaseName;" + WHERE ServerName = '$ServerName' AND DatabaseName = '$DatabaseName';" Invoke-SqlAzureWithRetry ` -ServerInstance $Catalog.FullyQualifiedServerName ` @@ -1245,8 +1252,6 @@ function Remove-ExtendedDatabase -Password $config.CatalogAdminPassword ` -Database $Catalog.Database.DatabaseName ` -Query $commandText ` - -ConnectionTimeout 30 ` - -QueryTimeout 30 ` } @@ -1277,9 +1282,7 @@ function Remove-ExtendedElasticPool{ -Database $Catalog.Database.DatabaseName ` -Query $commandText ` -UserName $config.CatalogAdminUserName ` - -Password $config.CatalogAdminPassword ` - -ConnectionTimeout 30 ` - -QueryTimeout 15 + -Password $config.CatalogAdminPassword } @@ -1313,7 +1316,7 @@ function Remove-ExtendedServer <# .SYNOPSIS - Removes extended tenant and associated database meta data entries from catalog + Removes extended tenant entry from catalog #> function Remove-ExtendedTenant { @@ -1341,9 +1344,7 @@ function Remove-ExtendedTenant # Delete the tenant name from the Tenants table $commandText = " DELETE FROM Tenants - WHERE TenantId = $rawkeyHexString; - DELETE FROM Databases - WHERE ServerName = '$ServerName' AND DatabaseName = '$DatabaseName';" + WHERE TenantId = $rawkeyHexString;" Invoke-SqlAzureWithRetry ` -ServerInstance $Catalog.FullyQualifiedServerName ` diff --git a/Learning Modules/Operational Analytics/Adhoc Analytics/Demo-AdhocAnalytics.ps1 b/Learning Modules/Operational Analytics/Adhoc Analytics/Demo-AdhocAnalytics.ps1 index 9246360..fc75bf5 100644 --- a/Learning Modules/Operational Analytics/Adhoc Analytics/Demo-AdhocAnalytics.ps1 +++ b/Learning Modules/Operational Analytics/Adhoc Analytics/Demo-AdhocAnalytics.ps1 @@ -31,7 +31,7 @@ if ($DemoScenario -eq 1) { Write-Output "Running ticket generator ..." - & $PSScriptRoot\..\..\Utilities\TicketGenerator.ps1 ` + & $PSScriptRoot\..\..\Utilities\TicketGenerator2.ps1 ` -WtpResourceGroupName $wtpUser.ResourceGroupName ` -WtpUser $wtpUser.Name exit diff --git a/Learning Modules/Operational Analytics/Adhoc Analytics/Demo-AdhocAnalyticsQueries.sql b/Learning Modules/Operational Analytics/Adhoc Analytics/Demo-AdhocAnalyticsQueries.sql index 79448e4..d24dbf5 100644 --- a/Learning Modules/Operational Analytics/Adhoc Analytics/Demo-AdhocAnalyticsQueries.sql +++ b/Learning Modules/Operational Analytics/Adhoc Analytics/Demo-AdhocAnalyticsQueries.sql @@ -5,81 +5,67 @@ select * from sys.external_tables; GO -- ******************************************************* --- SOME ADHOC QUERIES. Any others you would like to try? +-- SAMPLE QUERIES -- ******************************************************* --- What venues are registered on the Wingtip platform right now? +-- Which venues are currently registered on the Wingtip platform? SELECT VenueName, VenueType -FROM dbo.Venue +FROM dbo.VenuesGlobal GO -- What are the most popular venue types? SELECT VenueType, - Count(TicketPurchaseId) AS PurchasedTicketCount -FROM dbo.Venue - INNER JOIN dbo.TicketPurchases ON TicketPurchaseId > 0 + Count(TicketId) AS PurchasedTicketCount +FROM dbo.VenuesGlobal + INNER JOIN dbo.VenueTickets ON VenuesGlobal.VenueId = VenueTickets.VenueId GROUP BY VenueType ORDER BY PurchasedTicketCount DESC GO --- Which day had the most tickets sold? -SELECT CAST(PurchaseDate AS DATE) AS TicketPurchaseDate, - Count(TicketPurchaseId) AS TicketCount -FROM TicketPurchases -GROUP BY PurchaseDate -ORDER BY TicketCount DESC, TicketPurchaseDate ASC +-- On which day were the most tickets sold? +SELECT CAST(PurchaseDate AS DATE) AS TicketPurchaseDate, + Count(TicketId) AS TicketCount +FROM VenueTicketPurchases + INNER JOIN VenueTickets ON (VenueTickets.TicketPurchaseId = VenueTicketPurchases.TicketPurchaseId AND VenueTickets.VenueId = VenueTicketPurchases.VenueId) +GROUP BY (CAST(PurchaseDate AS DATE)) +ORDER BY TicketCount DESC, TicketPurchaseDate ASC GO --- What was the highest revenue event per each venue? +-- Which event had the highest revenue at each venue? EXEC sp_execute_remote N'WtpTenantDBs', N'SELECT TOP (1) VenueName, EventName, - SUM(PurchaseTotal) AS PurchaseTotal - FROM Venue - INNER JOIN Events ON Events.EventId > 0 - INNER JOIN Tickets ON Tickets.EventId = Events.EventId - INNER JOIN TicketPurchases ON TicketPurchases.TicketPurchaseId = Tickets.TicketPurchaseId - GROUP BY VenueName, EventName - ' + Subtitle AS Performers, + COUNT(TicketId) AS TicketsSold, + CONVERT(VARCHAR(30), SUM(PurchaseTotal), 1) AS PurchaseTotal + FROM VenueEvents + INNER JOIN VenueTickets ON VenueTickets.EventId = VenueEvents.EventId + INNER JOIN VenueTicketPurchases ON VenueTicketPurchases.TicketPurchaseId = VenueTickets.TicketPurchaseId + INNER JOIN VenuesGlobal ON VenueEvents.VenueId = VenuesGlobal.VenueId + GROUP BY VenueName, EventName, Subtitle + ORDER BY PurchaseTotal DESC' GO -- What are the top 10 grossing events across all venues on the WTP platform - ---Create temp table to hold the highest revenue events from each venue -DROP TABLE IF EXISTS #tmpMaxRevenue -CREATE TABLE #tmpMaxRevenue (VenueName nvarchar(50), EventName nvarchar(50), PurchaseTotal money, ShardName nvarchar(200)) -INSERT INTO #tmpMaxRevenue (VenueName, EventName, PurchaseTotal, ShardName) -EXEC sp_execute_remote - N'WtpTenantDBs', - N'SELECT TOP (10) - (SELECT TOP (1) VenueName FROM Venue) AS VenueName, - EventName, - SUM(PurchaseTotal) AS PurchaseTotal - FROM Events - INNER JOIN Tickets ON Tickets.EventId = Events.EventId - INNER JOIN TicketPurchases ON TicketPurchases.TicketPurchaseId = Tickets.TicketPurchaseId - GROUP BY EventName - ORDER BY PurchaseTotal DESC' - --- Select highest grossing events. Reference event revenue temp table from above SELECT TOP (10) VenueName, - EventName, - PurchaseTotal -FROM #tmpMaxRevenue -GROUP BY VenueName, PurchaseTotal, EventName -ORDER BY PurchaseTotal DESC, EventName - -GO - - - - - + EventName, + Subtitle AS EventPerformers, + CAST(VenueEvents.Date AS DATE) AS EventDate, + COUNT(TicketId) AS TicketPurchaseCount, + CONVERT(VARCHAR(30), SUM(PurchaseTotal), 1) AS EventRevenue +FROM VenueEvents + INNER JOIN VenueTickets ON (VenueTickets.EventId = VenueEvents.EventId AND VenueTickets.VenueId = VenueEvents.VenueId) + INNER JOIN VenueTicketPurchases ON (VenueTicketPurchases.TicketPurchaseId = VenueTickets.TicketPurchaseId AND VenueTicketPurchases.VenueId = VenueEvents.VenueId) + INNER JOIN VenuesGlobal ON VenueEvents.VenueId = VenuesGlobal.VenueId +GROUP BY VenueName, Subtitle, EventName, (CAST(VenueEvents.Date AS DATE)) +ORDER BY SUM(PurchaseTotal) DESC + +GO \ No newline at end of file diff --git a/Learning Modules/Operational Analytics/Adhoc Analytics/Deploy-AdhocAnalyticsDB.ps1 b/Learning Modules/Operational Analytics/Adhoc Analytics/Deploy-AdhocAnalyticsDB.ps1 index dddc2b5..572f19a 100644 --- a/Learning Modules/Operational Analytics/Adhoc Analytics/Deploy-AdhocAnalyticsDB.ps1 +++ b/Learning Modules/Operational Analytics/Adhoc Analytics/Deploy-AdhocAnalyticsDB.ps1 @@ -155,7 +155,7 @@ $commandText = " CREATE EXTERNAL TABLE [dbo].[Venue] ( [VenueName] NVARCHAR (50) NOT NULL, [VenueType] CHAR (30) NOT NULL, - [AdminEmail] NCHAR (30) NOT NULL, + [AdminEmail] VARCHAR (50) NOT NULL, [AdminPassword] NCHAR (30) NULL, [PostalCode] CHAR (10) NULL, [CountryCode] CHAR (3) NOT NULL @@ -187,16 +187,16 @@ $commandText = " INSERT INTO [dbo].[VenueTypes] ([VenueType],[VenueTypeName],[EventTypeName],[EventTypeShortName],[EventTypeShortNamePlural],[Language]) VALUES - ('MultiPurposeVenue','Multi Purpose Venue','Event', 'Event','Events','en-us'), - ('ClassicalConcertHall','Classical Concert Hall','Classical Concert','Concert','Concerts','en-us'), - ('JazzClub','Jazz Club','Jazz Session','Session','Sessions','en-us'), - ('JudoClub','Judo Club','Judo Tournament','Tournament','Tournaments','en-us'), - ('SoccerClub','Soccer Club','Soccer Match', 'Match','Matches','en-us'), - ('MotorRacing','Motor Racing','Car Race', 'Race','Races','en-us'), - ('DanceStudio', 'Dance Studio', 'Performance', 'Performance', 'Performances','en-us'), - ('BluesClub', 'Blues Club', 'Blues Session', 'Session','Sessions','en-us' ), - ('RockMusicVenue','Rock Music Venue','Rock Concert','Concert', 'Concerts','en-us'), - ('Opera','Opera','Opera','Opera','Operas','en-us'); + ('multipurpose','Multi-Purpose','Event', 'Event','Events','en-us'), + ('classicalmusic','Classical Music ','Classical Concert','Concert','Concerts','en-us'), + ('jazz','Jazz','Jazz Session','Session','Sessions','en-us'), + ('judo','Judo','Judo Tournament','Tournament','Tournaments','en-us'), + ('soccer','Soccer','Soccer Match', 'Match','Matches','en-us'), + ('motorracing','Motor Racing','Car Race', 'Race','Races','en-us'), + ('dance', 'Dance', 'Performance', 'Performance', 'Performances','en-us'), + ('blues', 'Blues', 'Blues Session', 'Session','Sessions','en-us' ), + ('rockmusic','Rock Music','Rock Concert','Concert', 'Concerts','en-us'), + ('opera','Opera','Opera','Opera','Operas','en-us'); GO PRINT N'Update complete.'; diff --git a/Learning Modules/Operational Analytics/Tenant Analytics DW/Copy-TenantDataToDataWarehouse.ps1 b/Learning Modules/Operational Analytics/Tenant Analytics DW/Copy-TenantDataToDataWarehouse.ps1 new file mode 100644 index 0000000..c6825e5 --- /dev/null +++ b/Learning Modules/Operational Analytics/Tenant Analytics DW/Copy-TenantDataToDataWarehouse.ps1 @@ -0,0 +1,198 @@ +<# +.SYNOPSIS + Extracts ticket sales data from a tenant database to an analysis database or data warehouse + +.DESCRIPTION + Creates an Elastic Job that extracts ticket sales data from a tenant database and + outputs it to an analysis database or data warehouse +#> +param( + [Parameter(Mandatory=$true)] + [string]$WtpResourceGroupName, + + [Parameter(Mandatory=$true)] + [string]$WtpUser, + + [Parameter(Mandatory=$true)] + [string]$JobExecutionCredentialName, + + [Parameter(Mandatory=$true)] + [string]$OutputServer, + + [Parameter(Mandatory=$true)] + [string]$OutputDataWarehouse, + + [Parameter(Mandatory=$true)] + [string]$OutputServerCredentialName, + + [Parameter(Mandatory=$false)] + [string]$OutputTableName = "AllTicketPurchasesFromAllTenants" +) + +Import-Module $PSScriptRoot\..\..\Common\SubscriptionManagement -Force +Import-Module $PSScriptRoot\..\..\WtpConfig -Force + + +# Get Azure credentials if not already logged on, Use -Force to select a different subscription +Initialize-Subscription + +$config = Get-Configuration + +# Get server that contains all tenant databases +$tenantServer = $config.TenantServerNameStem + $WtpUser + ".database.windows.net" + +# Get 'operations' server that contains catalog database, elastic job database, and other tenant management databases +#$opsServer = + +$opsServer = $config.CatalogServerNameStem + $WtpUser + ".database.windows.net" + +$jobName = "Extract all tenants ticket purchases to DW" +$jobDescription = "Retrieve ticket sales data from all Wingtip tenants" + +$commandText = " + DECLARE @jobIdentifier uniqueidentifier; + + -- Create a target group + EXEC [jobs].sp_add_target_group @target_group_name = 'TenantGroupDW'; + + -- Add all tenant servers to target group + EXEC [jobs].sp_add_target_group_member + @target_group_name = 'TenantGroupDW', + @membership_type = 'Include', + @target_type = 'SqlServer', + @refresh_credential_name='$JobExecutionCredentialName', + @server_name='$tenantServer'; + + -- Create elastic job definition + EXEC jobs.sp_add_job + @job_name='$jobName', + @description='$jobDescription', + @enabled=1, + @schedule_interval_type='Once', + @job_id= @jobIdentifier OUTPUT; + + -- Add job step to retrieve all tenant ticket purchases + EXEC jobs.sp_add_jobstep + @job_name='$jobName', + @command=N' + WITH Venues_CTE (VenueId, VenueName, VenueType, VenuePostalCode, VenueCapacity, X) + AS + (SELECT TOP 1 Convert(int, HASHBYTES(''md5'',VenueName)) AS VenueId, VenueName, VenueType, PostalCode AS VenuePostalCode, + (SELECT SUM ([SeatRows]*[SeatsPerRow]) FROM [dbo].[Sections]) AS VenueCapacity, + 1 AS X FROM Venues) + SELECT v.VenueId, v.VenueName, v.VenueType,v.VenuePostalCode, v.VenueCapacity, tp.TicketPurchaseId, tp.PurchaseDate, tp.PurchaseTotal, c.CustomerId, c.PostalCode as CustomerPostalCode, c.CountryCode, e.EventId, e.EventName, e.Subtitle as EventSubtitle, e.Date as EventDate FROM + Venues_CTE as v + INNER JOIN TicketPurchases AS tp ON v.X = 1 + INNER JOIN Tickets AS t ON t.TicketPurchaseId = tp.TicketPurchaseId + INNER JOIN Events AS e ON t.EventId = e.EventId + INNER JOIN Customers AS c ON tp.CustomerId = c.CustomerId', + @retry_attempts=2, + @credential_name='$JobExecutionCredentialName', + @target_group_name='TenantGroupDW', + @output_type='SqlDatabase', + @output_credential_name='$JobExecutionCredentialName', + @output_server_name='$OutputServer', + @output_database_name='$OutputDataWarehouse', + @output_table_name='$OutputTableName'; + + PRINT N'Elastic job submitted';" + + Invoke-Sqlcmd ` + -ServerInstance $opsServer ` + -Username $config.CatalogAdminUserName ` + -Password $config.CatalogAdminPassword ` + -Database $config.JobAccountDatabaseName ` + -Query $commandText ` + -ConnectionTimeout 30 ` + -QueryTimeout 30 ` + -EncryptConnection + +Write-output "Copying tenant ticket purchases to '$outputDataWarehouse' ..." + +# Check for status of job +$copyComplete = $false +$completedTenants = @(); + +while(!$copyComplete) +{ + # Poll for status of copy job + $jobStatusQuery = " + SELECT [is_active], [lifecycle] FROM [jobs].[job_executions] + WHERE [job_name] = '$jobName' and [step_id] IS NULL" + + $jobStatus = Invoke-Sqlcmd ` + -ServerInstance $opsServer ` + -Username $config.CatalogAdminUserName ` + -Password $config.CatalogAdminPassword ` + -Database $config.JobAccountDatabaseName ` + -Query $jobStatusQuery ` + -ConnectionTimeout 30 ` + -QueryTimeout 30 ` + -EncryptConnection + + if ($jobStatus.lifecycle -eq "Succeeded" -or $jobStatus.lifecycle -eq "Failed") + { + $copyComplete = $true + } + + # Get details on which tenants' purchases have been copied + $tenantStatusQuery = " + SELECT [target_database_name], [is_active], [lifecycle], [last_message] FROM [jobs].[job_executions] + WHERE [job_name] = '$jobName' and [step_id] IS NOT NULL and [target_database_name] IS NOT NULL" + + $tenantStatus = Invoke-Sqlcmd ` + -ServerInstance $opsServer ` + -Username $config.CatalogAdminUserName ` + -Password $config.CatalogAdminPassword ` + -Database $config.JobAccountDatabaseName ` + -Query $tenantStatusQuery ` + -ConnectionTimeout 30 ` + -QueryTimeout 30 ` + -EncryptConnection + + # Print status of tenants' purchases copy job + foreach($tenant in $tenantStatus) + { + if (($tenant.lifecycle -eq "Succeeded") -and ($completedTenants -notcontains $tenant.target_database_name)) + { + Write-Output "Tenant '$($tenant.target_database_name)' ticket purchases copy complete." + $completedTenants += $tenant.target_database_name + } + elseif ($tenant.lifecycle -eq "Failed") + { + Write-Output "Copy ticket purchases job failed for tenant '$($tenant.target_database_name)' with error: $($tenant.last_message) " + } + } + + if($copyComplete) + { + Write-Output "----Copy tenant ticket purchases job complete ----" + } + else + { + if ($tenantStatus.Length) + { + $inProgressTenants = $tenantStatus.Length - $completedTenants.Length + Write-Output "----Processing $inProgressTenants tenant(s)----" + Start-Sleep -s 5 + } + } + +} + +# Cleanup completed job +Write-output "Deleting completed job from job database ..." + +$commandText = " + EXEC [jobs].[sp_delete_job] '$jobName' + EXEC [jobs].[sp_delete_target_group] 'TenantGroupDW' " + + Invoke-Sqlcmd ` + -ServerInstance $opsServer ` + -Username $config.CatalogAdminUserName ` + -Password $config.CatalogAdminPassword ` + -Database $config.JobAccountDatabaseName ` + -Query $commandText ` + -ConnectionTimeout 30 ` + -QueryTimeout 30 ` + -EncryptConnection diff --git a/Learning Modules/Operational Analytics/Tenant Analytics DW/Demo-TenantAnalyticsDW.ps1 b/Learning Modules/Operational Analytics/Tenant Analytics DW/Demo-TenantAnalyticsDW.ps1 new file mode 100644 index 0000000..76f59b6 --- /dev/null +++ b/Learning Modules/Operational Analytics/Tenant Analytics DW/Demo-TenantAnalyticsDW.ps1 @@ -0,0 +1,84 @@ +# Helper script for demonstrating tenant analytics using a SQL Data Warehouse + +Import-Module "$PSScriptRoot\..\..\Common\SubscriptionManagement" -Force +Import-Module "$PSScriptRoot\..\..\UserConfig" -Force +Import-Module "$PSScriptRoot\..\..\WtpConfig" -Force + +# Get Azure credentials if not already logged on, Use -Force to select a different subscription +Initialize-Subscription -NoEcho + +# Get the resource group and user names used when the WTP application was deployed from UserConfig.psm1. +$wtpUser = Get-UserConfig +$config = Get-Configuration + +# Enter the fully-qualified name of an existing analytics server and data warehouse if they are already created +$AnalyticsServerName = "" +$AnalyticsDataWarehouseName = "" + +$DemoScenario = 0 +<# Select the demo scenario that will be run. It is recommended you run the scenarios below in order. + Demo Scenario + 0 None + 1 Purchase tickets for events at all venues + 2 Deploy tenant analytics data warehouse + 3 Deploy job account database to manage the data extract jobs + 4 Create and run job to extract tenant data to the data warehouse for analysis +#> + +## ------------------------------------------------------------------------------------------------ + +### Default state - enter a valid demo scenaro +if ($DemoScenario -eq 0) +{ + Write-Output "Please modify the demo script to select a scenario to run." + exit +} + +### Purchase new tickets +if ($DemoScenario -eq 1) +{ + Write-Output "Running ticket generator ..." + + & $PSScriptRoot\..\..\Utilities\TicketGenerator2.ps1 ` + -WtpResourceGroupName $wtpUser.ResourceGroupName ` + -WtpUser $wtpUser.Name + exit +} + +### Provision a Data Warehouse for tenant analytics results +if ($DemoScenario -eq 2) +{ + & $PSScriptRoot\Deploy-TenantAnalyticsDW.ps1 ` + -WtpResourceGroupName $wtpUser.ResourceGroupName ` + -WtpUser $wtpUser.Name + exit +} + +### Deploy job account database to manage the data extract jobs +if ($DemoScenario -eq 3) +{ + & $PSSciptRoot\..\..\Schema Management\Deploy-JobAccount.ps1 ` + -WtpResourceGroupName $wtpUser.ResourceGroupName ` + -WtpUser $wtpUser.Name + exit +} + +### Create and run job to extract tenant data to the data warehouse for analysis +if ($DemoScenario -eq 4) +{ + # Retrieve Wingtip default analytics server and data warehouse if no existing analytics server/datawarehouse have been provided + if ($AnalyticsServerName -eq "" -or $AnalyticsDataWarehouseName -eq "") + { + $AnalyticsServerName = $config.catalogServerNameStem + $AnalyticsDataWarehouseName = $config.TenantAnalyticsDWDatabaseName + } + + & $PSScriptRoot\Copy-TenantDataToDataWarehouse.ps1 ` + -WtpResourceGroupName $wtpUser.ResourceGroupName ` + -WtpUser $wtpUser.Name ` + -JobExecutionCredentialName $config.JobAccountCredentialName ` + -OutputServer $AnalyticsServerName ` + -OutputDataWarehouse $AnalyticsDataWarehouseName ` + -OutputServerCredentialName $config.JobAccountCredentialName + exit +} diff --git a/Learning Modules/Operational Analytics/Tenant Analytics DW/Demo-TenantAnalyticsDWQueries.sql b/Learning Modules/Operational Analytics/Tenant Analytics DW/Demo-TenantAnalyticsDWQueries.sql new file mode 100644 index 0000000..b751735 --- /dev/null +++ b/Learning Modules/Operational Analytics/Tenant Analytics DW/Demo-TenantAnalyticsDWQueries.sql @@ -0,0 +1,10 @@ +-- ******************************************************* +-- SIMPLE ADHOC QUERY. Any others you would like to try? +-- ******************************************************* + +-- Get a summary of all ticket sales from all tenants +SELECT VenueName, SUM(PurchaseTotal) as TotalSales +FROM dbo.AllTicketsPurchasesfromAllTenants +GROUP BY VenueName +ORDER BY TotalSales DESC; +GO \ No newline at end of file diff --git a/Learning Modules/Operational Analytics/Tenant Analytics DW/Deploy-TenantAnalyticsDW.ps1 b/Learning Modules/Operational Analytics/Tenant Analytics DW/Deploy-TenantAnalyticsDW.ps1 new file mode 100644 index 0000000..60be48e --- /dev/null +++ b/Learning Modules/Operational Analytics/Tenant Analytics DW/Deploy-TenantAnalyticsDW.ps1 @@ -0,0 +1,50 @@ +<# +.SYNOPSIS + Creates an Operational Analytics DW database for tenant query data + +.DESCRIPTION + Creates the operational tenant analytics DW database for result sets queries from Elastic jobs. Database is created in the resource group + created when the WTP application was deployed. + +#> +param( + [Parameter(Mandatory=$true)] + [string]$WtpResourceGroupName, + + [Parameter(Mandatory=$true)] + [string]$WtpUser +) + +Import-Module $PSScriptRoot\..\..\Common\SubscriptionManagement -Force + +# Get Azure credentials if not already logged on, Use -Force to select a different subscription +Initialize-Subscription + +Import-Module $PSScriptRoot\..\..\AppVersionSpecific -Force + +$config = Get-Configuration + +$catalogServerName = $($config.CatalogServerNameStem) + $WtpUser +$databaseName = $config.TenantAnalyticsDWDatabaseName + +# Check if Analytics DW database has already been created +$TenantAnalyticsDWDatabaseName = Get-AzureRmSqlDatabase ` + -ResourceGroupName $WtpResourceGroupName ` + -ServerName $catalogServerName ` + -DatabaseName $databaseName ` + -ErrorAction SilentlyContinue + +if($TenantAnalyticsDatabaseName) +{ + Write-Output "Tenant Analytics DW database '$databaseName' already exists." + exit +} + +Write-output "Initializing the DW database '$databaseName'..." + +# Create the tenant analytics DW database +New-AzureRmSqlDatabase ` + -ResourceGroupName $WtpResourceGroupName ` + -ServerName $catalogServerName ` + -DatabaseName $databaseName ` + -RequestedServiceObjectiveName "DW400" diff --git a/Learning Modules/Operational Analytics/Tenant Analytics DW/foo.txt b/Learning Modules/Operational Analytics/Tenant Analytics DW/foo.txt deleted file mode 100644 index 8b13789..0000000 --- a/Learning Modules/Operational Analytics/Tenant Analytics DW/foo.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/Learning Modules/Operational Analytics/Tenant Analytics/Demo-TenantAnalyticsDB.ps1 b/Learning Modules/Operational Analytics/Tenant Analytics/Demo-TenantAnalyticsDB.ps1 index 1011091..5c2c5be 100644 --- a/Learning Modules/Operational Analytics/Tenant Analytics/Demo-TenantAnalyticsDB.ps1 +++ b/Learning Modules/Operational Analytics/Tenant Analytics/Demo-TenantAnalyticsDB.ps1 @@ -1,49 +1,59 @@ -# Helper script for demonstrating tenant analytics scripts -deploys the Tenant Analytics database to the catalog server - -Import-Module "$PSScriptRoot\..\..\Common\SubscriptionManagement" -Force -Import-Module "$PSScriptRoot\..\..\UserConfig" -Force - -# Get Azure credentials if not already logged on, Use -Force to select a different subscription -Initialize-Subscription -NoEcho - -# Get the resource group and user names used when the WTP application was deployed from UserConfig.psm1. -$wtpUser = Get-UserConfig - -$DemoScenario = 0 -<# Select the demo scenario that will be run. It is recommended you run the scenarios below in order. - Demo Scenario - 0 None - 1 Purchase tickets for events at all venues - 2 Deploy operational analytics database -#> - -## ------------------------------------------------------------------------------------------------ - -### Default state - enter a valid demo scenaro -if ($DemoScenario -eq 0) -{ - Write-Output "Please modify the demo script to select a scenario to run." - exit -} - -### Purchase new tickets -if ($DemoScenario -eq 1) -{ - Write-Output "Running ticket generator ..." - - & $PSScriptRoot\..\..\Utilities\TicketGenerator.ps1 ` - -WtpResourceGroupName $wtpUser.ResourceGroupName ` - -WtpUser $wtpUser.Name - exit -} - -### Provision a database for operational analytics results -if ($DemoScenario -eq 2) -{ - & $PSScriptRoot\Deploy-TenantAnalyticsDB.ps1 ` - -WtpResourceGroupName $wtpUser.ResourceGroupName ` - -WtpUser $wtpUser.Name - exit -} - -Write-Output "Invalid scenario selected" \ No newline at end of file +# Helper script for demonstrating tenant analytics scripts + +Import-Module "$PSScriptRoot\..\..\Common\SubscriptionManagement" -Force +Import-Module "$PSScriptRoot\..\..\UserConfig" -Force + +# Get Azure credentials if not already logged on, Use -Force to select a different subscription +Initialize-Subscription -NoEcho + +# Get the resource group and user names used when the WTP application was deployed from UserConfig.psm1. +$wtpUser = Get-UserConfig + +$DemoScenario = 0 +<# Select the demo scenario that will be run. It is recommended you run the scenarios below in order. + Demo Scenario + 0 None + 1 Purchase tickets for events at all venues + 2 Deploy tenant analytics database + 3 Deploy tenant analytics columnstore database (creates a Premium P1 database) +#> + +## ------------------------------------------------------------------------------------------------ + +### Default state - enter a valid demo scenaro +if ($DemoScenario -eq 0) +{ + Write-Output "Please modify the demo script to select a scenario to run." + exit +} + +### Purchase new tickets +if ($DemoScenario -eq 1) +{ + Write-Output "Running ticket generator ..." + + & $PSScriptRoot\..\..\Utilities\TicketGenerator.ps1 ` + -WtpResourceGroupName $wtpUser.ResourceGroupName ` + -WtpUser $wtpUser.Name + exit +} + +### Provision a database for operational analytics results +if ($DemoScenario -eq 2) +{ + & $PSScriptRoot\Deploy-TenantAnalyticsDB.ps1 ` + -WtpResourceGroupName $wtpUser.ResourceGroupName ` + -WtpUser $wtpUser.Name + exit +} + +### Provision a columnstore database for operational tenant analytics results +if ($DemoScenario -eq 3) +{ + & $PSScriptRoot\Deploy-TenantAnalyticsDB-CS.ps1 ` + -WtpResourceGroupName $wtpUser.ResourceGroupName ` + -WtpUser $wtpUser.Name + exit +} + +Write-Output "Invalid scenario selected" diff --git a/Learning Modules/Operational Analytics/Tenant Analytics/Deploy-TenantAnalyticsDB-CS.ps1 b/Learning Modules/Operational Analytics/Tenant Analytics/Deploy-TenantAnalyticsDB-CS.ps1 new file mode 100644 index 0000000..da8c25f --- /dev/null +++ b/Learning Modules/Operational Analytics/Tenant Analytics/Deploy-TenantAnalyticsDB-CS.ps1 @@ -0,0 +1,92 @@ +<# +.SYNOPSIS + Creates a tenant analytics database with a columnstore table for ticket analytics data. + +.DESCRIPTION + Creates the tenant analytics database for result sets queries from Elastic jobs. + Database is created in the resource group created when the WTP application was deployed. + +#> +param( + [Parameter(Mandatory=$true)] + [string]$WtpResourceGroupName, + + [Parameter(Mandatory=$true)] + [string]$WtpUser +) + +Import-Module $PSScriptRoot\..\..\Common\SubscriptionManagement -Force +Import-Module $PSScriptRoot\..\..\WtpConfig -Force + +# Get Azure credentials if not already logged on, Use -Force to select a different subscription +Initialize-Subscription + +$config = Get-Configuration + +$catalogServerName = $($config.CatalogServerNameStem) + $WtpUser +$databaseName = $config.TenantAnalyticsCSDatabaseName +$fullyQualfiedCatalogServerName = $catalogServerName + ".database.windows.net" + +# Check if Analytics database has already been created +$TenantAnalyticsDatabaseName = Get-AzureRmSqlDatabase ` + -ResourceGroupName $WtpResourceGroupName ` + -ServerName $catalogServerName ` + -DatabaseName $databaseName ` + -ErrorAction SilentlyContinue + + + if($TenantAnalyticsCSDatabaseName) +{ + Write-Output "Tenant Analytics Columnstore database '$databaseName' already exists." + exit +} + +Write-output "Initializing the columnstore database '$($config.TenantAnalyticsCSDatabaseName)'..." + +# Create the tenant analytics database +New-AzureRmSqlDatabase ` + -ResourceGroupName $WtpResourceGroupName ` + -ServerName $catalogServerName ` + -DatabaseName $databaseName ` + -RequestedServiceObjectiveName "P1" + +# Pre-create the tenant analytics columnstore table schema + +$commandText = " +CREATE TABLE [dbo].[AllTicketsPurchasesfromAllTenants]( + [VenueId] [int] NULL, + [VenueName] [nvarchar](50) NULL, + [VenueType] [char](30) NULL, + [VenuePostalCode] [char](10) NULL, + [VenueCapacity] [int] NULL, + [TicketPurchaseId] [int] NULL, + [PurchaseDate] [datetime] NULL, + [PurchaseTotal] [money] NULL, + [CustomerId] [int] NULL, + [CustomerPostalCode] [char](10) NULL, + [CountryCode] [char](3) NULL, + [EventId] [int] NULL, + [EventName] [nvarchar](50) NULL, + [EventSubtitle] [nvarchar](50) NULL, + [EventDate] [datetime] NULL, + [job_execution_id] [uniqueidentifier] NULL, + [internal_execution_id] [uniqueidentifier] NULL +) +GO +CREATE CLUSTERED COLUMNSTORE INDEX cci_Tickets +ON [AllTicketsPurchasesfromAllTenants] +GO +" +Write-output "Initializing schema in '$databaseName'..." + +Invoke-Sqlcmd ` + -ServerInstance $fullyQualfiedCatalogServerName ` + -Username $config.CatalogAdminUserName ` + -Password $config.CatalogAdminPassword ` + -Database $databaseName ` + -Query $commandText ` + -ConnectionTimeout 30 ` + -QueryTimeout 30 ` + -EncryptConnection + +Write-Output "Deployment of tenant analytics columnstore database '$databaseName' and table schema completed." \ No newline at end of file diff --git a/Learning Modules/Operational Analytics/Tenant Analytics/TicketPurchasesfromAllTenants.sql b/Learning Modules/Operational Analytics/Tenant Analytics/TicketPurchasesfromAllTenants.sql index 9bd2fc9..f573a76 100644 --- a/Learning Modules/Operational Analytics/Tenant Analytics/TicketPurchasesfromAllTenants.sql +++ b/Learning Modules/Operational Analytics/Tenant Analytics/TicketPurchasesfromAllTenants.sql @@ -1,75 +1,77 @@ --- Connect to and run against the jobaccount database in catalog- server --- Replace below with your user name -DECLARE @WtpUser nvarchar(50); -DECLARE @server1 nvarchar(50); -DECLARE @server2 nvarchar(50); -SET @WtpUser = ''; - --- Add a target group containing server(s) -EXEC [jobs].sp_add_target_group @target_group_name = 'TenantGroup' - --- Add a server target member, includes all databases in tenant server -SET @server1 = 'customers1-' + @WtpUser + '.database.windows.net' - -EXEC [jobs].sp_add_target_group_member -@target_group_name = 'TenantGroup', -@membership_type = 'Include', -@target_type = 'SqlServer', -@refresh_credential_name='myrefreshcred', -@server_name=@server1 - --- Create job to retrieve analytics that are distributed across all the tenants -EXEC jobs.sp_add_job -@job_name='Ticket Purchases from all Tenants', -@description='Collect tenant specific ticket sales data from each tenant database', -@enabled=1, -@schedule_interval_type='Once' - --- Create job step to retrieve analytics that are distributed across all the tenants -SET @server2 = 'catalog-' + @WtpUser + '.database.windows.net' - -EXEC jobs.sp_add_jobstep -@job_name='Ticket Purchases from all Tenants', -@command=N' -WITH Venue_CTE (VenueId, VenueName, VenueType, VenuePostalCode, X) -AS - (SELECT TOP 1 Convert(int, HASHBYTES(''md5'',VenueName)) AS VenueId, VenueName, VenueType, PostalCode AS VenuePostalCode, 1 AS X FROM Venue) -SELECT v.VenueId, v.VenueName, v.VenueType,v.VenuePostalCode, tp.TicketPurchaseId, tp.PurchaseDate, tp.PurchaseTotal, c.CustomerId, c.PostalCode as CustomerPostalCode, c.CountryCode, e.EventId, e.EventName, $(job_execution_id) AS job_execution_id FROM -Venue_CTE as v -INNER JOIN TicketPurchases AS tp ON v.X = 1 -INNER JOIN Tickets AS t ON t.TicketPurchaseId = tp.TicketPurchaseId -INNER JOIN Events AS e ON t.EventId = e.EventId -INNER JOIN Customers AS c ON tp.CustomerId = c.CustomerId', -@credential_name='mydemocred', -@target_group_name='TenantGroup', -@output_type='SqlDatabase', -@output_credential_name='mydemocred', -@output_server_name=@server2, -@output_database_name='tenantanalytics', -@output_table_name='AllTicketsPurchasesfromAllTenants' - --- --- Views --- Job and Job Execution Information and Status --- -SELECT * FROM [jobs].[jobs] -WHERE job_name = 'Ticket Purchases from all Tenants' - -SELECT * FROM [jobs].[jobsteps] -WHERE job_name = 'Ticket Purchases from all Tenants' - -WAITFOR DELAY '00:00:10' ---View parent execution status -SELECT * FROM [jobs].[job_executions] -WHERE job_name = 'Ticket Purchases from all Tenants' and step_id IS NULL - ---View all execution status -SELECT * FROM [jobs].[job_executions] -WHERE job_name = 'Ticket Purchases from all Tenants' - ---Stop a running job, requires active job_execution_id from [jobs].[job_executions] view ---EXEC [jobs].[sp_stop_job] 'E20ECAD1-5A80-4CE5-B246-7A7DA3C24A27' - --- Cleanup ---EXEC [jobs].[sp_delete_job] 'Ticket Purchases from all Tenants' ---EXEC [jobs].[sp_delete_target_group] 'TenantGroup' +-- Connect to and run against the jobaccount database in catalog- server +-- Replace below with your user name +DECLARE @WtpUser nvarchar(50); +DECLARE @server1 nvarchar(50); +DECLARE @server2 nvarchar(50); +SET @WtpUser = ''; + +-- Add a target group containing server(s) +EXEC [jobs].sp_add_target_group @target_group_name = 'TenantGroup' + +-- Add a server target member, includes all databases in tenant server +SET @server1 = 'tenants1-' + @WtpUser + '.database.windows.net' + +EXEC [jobs].sp_add_target_group_member +@target_group_name = 'TenantGroup', +@membership_type = 'Include', +@target_type = 'SqlServer', +@refresh_credential_name='myrefreshcred', +@server_name=@server1 + +-- Create job to retrieve analytics that are distributed across all the tenants +EXEC jobs.sp_add_job +@job_name='Ticket Purchases from all Tenants', +@description='Collect tenant specific ticket sales data from each tenant database', +@enabled=1, +@schedule_interval_type='Once' + +-- Create job step to retrieve analytics that are distributed across all the tenants +SET @server2 = 'catalog-' + @WtpUser + '.database.windows.net' + +EXEC jobs.sp_add_jobstep +@job_name='Ticket Purchases from all Tenants', +@command=N' +WITH Venue_CTE (VenueId, VenueName, VenueType, VenuePostalCode, VenueCapacity, X) +AS + (SELECT TOP 1 Convert(int, HASHBYTES(''md5'',VenueName)) AS VenueId, VenueName, VenueType, PostalCode AS VenuePostalCode, + (SELECT SUM ([SeatRows]*[SeatsPerRow]) FROM [dbo].[Sections]) AS VenueCapacity, + 1 AS X FROM Venue) +SELECT v.VenueId, v.VenueName, v.VenueType,v.VenuePostalCode, v.VenueCapacity, tp.TicketPurchaseId, tp.PurchaseDate, tp.PurchaseTotal, c.CustomerId, c.PostalCode as CustomerPostalCode, c.CountryCode, e.EventId, e.EventName, e.Subtitle as EventSubtitle, e.Date as EventDate, $(job_execution_id) as job_execution_id FROM +Venue_CTE as v +INNER JOIN TicketPurchases AS tp ON v.X = 1 +INNER JOIN Tickets AS t ON t.TicketPurchaseId = tp.TicketPurchaseId +INNER JOIN Events AS e ON t.EventId = e.EventId +INNER JOIN Customers AS c ON tp.CustomerId = c.CustomerId', +@credential_name='mydemocred', +@target_group_name='TenantGroup', +@output_type='SqlDatabase', +@output_credential_name='mydemocred', +@output_server_name=@server2, +@output_database_name='tenantanalytics', +@output_table_name='AllTicketsPurchasesfromAllTenants' + +-- +-- Views +-- Job and Job Execution Information and Status +-- +SELECT * FROM [jobs].[jobs] +WHERE job_name = 'Ticket Purchases from all Tenants' + +SELECT * FROM [jobs].[jobsteps] +WHERE job_name = 'Ticket Purchases from all Tenants' + +WAITFOR DELAY '00:00:10' +--View parent execution status +SELECT * FROM [jobs].[job_executions] +WHERE job_name = 'Ticket Purchases from all Tenants' and step_id IS NULL + +--View all execution status +SELECT * FROM [jobs].[job_executions] +WHERE job_name = 'Ticket Purchases from all Tenants' + +--Stop a running job, requires active job_execution_id from [jobs].[job_executions] view +--EXEC [jobs].[sp_stop_job] '1356A4E1-2A9E-47BF-A1D5-76CFB053DDD3' + +-- Cleanup +--EXEC [jobs].[sp_delete_job] 'Ticket Purchases from all Tenants' +--EXEC [jobs].[sp_delete_target_group] 'TenantGroup' diff --git a/Learning Modules/Operational Analytics/Tenant Analytics/TicketPurchasesfromAllTenantsCS.sql b/Learning Modules/Operational Analytics/Tenant Analytics/TicketPurchasesfromAllTenantsCS.sql new file mode 100644 index 0000000..501dcf0 --- /dev/null +++ b/Learning Modules/Operational Analytics/Tenant Analytics/TicketPurchasesfromAllTenantsCS.sql @@ -0,0 +1,77 @@ +-- Connect to and run against the jobaccount database in catalog- server +-- Replace below with your user name +DECLARE @WtpUser nvarchar(50); +DECLARE @server1 nvarchar(50); +DECLARE @server2 nvarchar(50); +SET @WtpUser = ''; + +-- Add a target group containing server(s) +EXEC [jobs].sp_add_target_group @target_group_name = 'TenantGroupCS' + +-- Add a server target member, includes all databases in tenant server +SET @server1 = 'tenants1-' + @WtpUser + '.database.windows.net' + +EXEC [jobs].sp_add_target_group_member +@target_group_name = 'TenantGroupCS', +@membership_type = 'Include', +@target_type = 'SqlServer', +@refresh_credential_name='myrefreshcred', +@server_name=@server1 + +-- Create job to retrieve analytics that are distributed across all the tenants +EXEC jobs.sp_add_job +@job_name='Ticket Purchases from all Tenants - CS', +@description='Retrieve tenant telemetry data from all tenants', +@enabled=1, +@schedule_interval_type='Once' + +-- Create job step to retrieve analytics that are distributed across all the tenants +SET @server2 = 'catalog-' + @WtpUser + '.database.windows.net' + +EXEC jobs.sp_add_jobstep +@job_name='Ticket Purchases from all Tenants - CS', +@command=N' +WITH Venue_CTE (VenueId, VenueName, VenueType, VenuePostalCode, VenueCapacity, X) +AS + (SELECT TOP 1 Convert(int, HASHBYTES(''md5'',VenueName)) AS VenueId, VenueName, VenueType, PostalCode AS VenuePostalCode, + (SELECT SUM ([SeatRows]*[SeatsPerRow]) FROM [dbo].[Sections]) AS VenueCapacity, + 1 AS X FROM Venue) +SELECT v.VenueId, v.VenueName, v.VenueType,v.VenuePostalCode, v.VenueCapacity, tp.TicketPurchaseId, tp.PurchaseDate, tp.PurchaseTotal, c.CustomerId, c.PostalCode as CustomerPostalCode, c.CountryCode, e.EventId, e.EventName, e.Subtitle as EventSubtitle, e.Date as EventDate, $(job_execution_id) as job_execution_id FROM +Venue_CTE as v +INNER JOIN TicketPurchases AS tp ON v.X = 1 +INNER JOIN Tickets AS t ON t.TicketPurchaseId = tp.TicketPurchaseId +INNER JOIN Events AS e ON t.EventId = e.EventId +INNER JOIN Customers AS c ON tp.CustomerId = c.CustomerId', +@credential_name='mydemocred', +@target_group_name='TenantGroupCS', +@output_type='SqlDatabase', +@output_credential_name='mydemocred', +@output_server_name=@server2, +@output_database_name='tenantanalytics-cs', +@output_table_name='AllTicketsPurchasesfromAllTenants' + +-- +-- Views +-- Job and Job Execution Information and Status +-- +SELECT * FROM [jobs].[jobs] +WHERE job_name = 'Ticket Purchases from all Tenants - CS' + +SELECT * FROM [jobs].[jobsteps] +WHERE job_name = 'Ticket Purchases from all Tenants - CS' + +WAITFOR DELAY '00:00:10' +--View parent execution status +SELECT * FROM [jobs].[job_executions] +WHERE job_name = 'Ticket Purchases from all Tenants - CS' and step_id IS NULL + +--View all execution status +SELECT * FROM [jobs].[job_executions] +WHERE job_name = 'Ticket Purchases from all Tenants - CS' + +--Stop a running job, requires active job_execution_id from [jobs].[job_executions] view +--EXEC [jobs].[sp_stop_job] '2150390F-6991-4652-AAC8-83C8DB73E444' + +-- Cleanup +--EXEC [jobs].[sp_delete_job] 'Ticket Purchases from all Tenants - CS' +--EXEC [jobs].[sp_delete_target_group] 'TenantGroupCS' \ No newline at end of file diff --git a/Learning Modules/Provision and Catalog/Auto Provision/Allocate Tenant Databases WebJob/Common/CatalogAndDatabaseManagement.psm1 b/Learning Modules/Provision and Catalog/Auto Provision/Allocate Tenant Databases WebJob/Common/CatalogAndDatabaseManagement.psm1 index 7657d1e..6e88b77 100644 --- a/Learning Modules/Provision and Catalog/Auto Provision/Allocate Tenant Databases WebJob/Common/CatalogAndDatabaseManagement.psm1 +++ b/Learning Modules/Provision and Catalog/Auto Provision/Allocate Tenant Databases WebJob/Common/CatalogAndDatabaseManagement.psm1 @@ -1216,7 +1216,6 @@ function Remove-CatalogInfoFromTenantDatabase -Password $adminPassword ` -ServerInstance ($TenantDatabase.ServerName + ".database.windows.net") ` -Database $TenantDatabase.DatabaseName ` - -ConnectionTimeout 30 ` -Query $commandText ` } @@ -1231,13 +1230,16 @@ function Remove-ExtendedDatabase [parameter(Mandatory=$true)] [object]$Catalog, + [parameter(Mandatory=$true)] + [string]$ServerName, + [parameter(Mandatory=$true)] [string]$DatabaseName ) $commandText = " DELETE FROM Databases - WHERE DatabaseName = $DatabaseName;" + WHERE ServerName = '$ServerName' AND DatabaseName = '$DatabaseName';" Invoke-SqlAzureWithRetry ` -ServerInstance $Catalog.FullyQualifiedServerName ` @@ -1245,8 +1247,6 @@ function Remove-ExtendedDatabase -Password $config.CatalogAdminPassword ` -Database $Catalog.Database.DatabaseName ` -Query $commandText ` - -ConnectionTimeout 30 ` - -QueryTimeout 30 ` } @@ -1277,9 +1277,7 @@ function Remove-ExtendedElasticPool{ -Database $Catalog.Database.DatabaseName ` -Query $commandText ` -UserName $config.CatalogAdminUserName ` - -Password $config.CatalogAdminPassword ` - -ConnectionTimeout 30 ` - -QueryTimeout 15 + -Password $config.CatalogAdminPassword } @@ -1313,7 +1311,7 @@ function Remove-ExtendedServer <# .SYNOPSIS - Removes extended tenant and associated database meta data entries from catalog + Removes extended tenant entry from catalog #> function Remove-ExtendedTenant { @@ -1341,9 +1339,7 @@ function Remove-ExtendedTenant # Delete the tenant name from the Tenants table $commandText = " DELETE FROM Tenants - WHERE TenantId = $rawkeyHexString; - DELETE FROM Databases - WHERE ServerName = '$ServerName' AND DatabaseName = '$DatabaseName';" + WHERE TenantId = $rawkeyHexString;" Invoke-SqlAzureWithRetry ` -ServerInstance $Catalog.FullyQualifiedServerName ` diff --git a/Learning Modules/Provision and Catalog/Auto Provision/Catalog Sync WebJob/Common/CatalogAndDatabaseManagement.psm1 b/Learning Modules/Provision and Catalog/Auto Provision/Catalog Sync WebJob/Common/CatalogAndDatabaseManagement.psm1 index 7657d1e..6e88b77 100644 --- a/Learning Modules/Provision and Catalog/Auto Provision/Catalog Sync WebJob/Common/CatalogAndDatabaseManagement.psm1 +++ b/Learning Modules/Provision and Catalog/Auto Provision/Catalog Sync WebJob/Common/CatalogAndDatabaseManagement.psm1 @@ -1216,7 +1216,6 @@ function Remove-CatalogInfoFromTenantDatabase -Password $adminPassword ` -ServerInstance ($TenantDatabase.ServerName + ".database.windows.net") ` -Database $TenantDatabase.DatabaseName ` - -ConnectionTimeout 30 ` -Query $commandText ` } @@ -1231,13 +1230,16 @@ function Remove-ExtendedDatabase [parameter(Mandatory=$true)] [object]$Catalog, + [parameter(Mandatory=$true)] + [string]$ServerName, + [parameter(Mandatory=$true)] [string]$DatabaseName ) $commandText = " DELETE FROM Databases - WHERE DatabaseName = $DatabaseName;" + WHERE ServerName = '$ServerName' AND DatabaseName = '$DatabaseName';" Invoke-SqlAzureWithRetry ` -ServerInstance $Catalog.FullyQualifiedServerName ` @@ -1245,8 +1247,6 @@ function Remove-ExtendedDatabase -Password $config.CatalogAdminPassword ` -Database $Catalog.Database.DatabaseName ` -Query $commandText ` - -ConnectionTimeout 30 ` - -QueryTimeout 30 ` } @@ -1277,9 +1277,7 @@ function Remove-ExtendedElasticPool{ -Database $Catalog.Database.DatabaseName ` -Query $commandText ` -UserName $config.CatalogAdminUserName ` - -Password $config.CatalogAdminPassword ` - -ConnectionTimeout 30 ` - -QueryTimeout 15 + -Password $config.CatalogAdminPassword } @@ -1313,7 +1311,7 @@ function Remove-ExtendedServer <# .SYNOPSIS - Removes extended tenant and associated database meta data entries from catalog + Removes extended tenant entry from catalog #> function Remove-ExtendedTenant { @@ -1341,9 +1339,7 @@ function Remove-ExtendedTenant # Delete the tenant name from the Tenants table $commandText = " DELETE FROM Tenants - WHERE TenantId = $rawkeyHexString; - DELETE FROM Databases - WHERE ServerName = '$ServerName' AND DatabaseName = '$DatabaseName';" + WHERE TenantId = $rawkeyHexString;" Invoke-SqlAzureWithRetry ` -ServerInstance $Catalog.FullyQualifiedServerName ` diff --git a/Learning Modules/Provision and Catalog/Auto Provision/Catalog Sync WebJob/Run.ps1 b/Learning Modules/Provision and Catalog/Auto Provision/Catalog Sync WebJob/Run.ps1 index 02bcae9..2ed3cf9 100644 --- a/Learning Modules/Provision and Catalog/Auto Provision/Catalog Sync WebJob/Run.ps1 +++ b/Learning Modules/Provision and Catalog/Auto Provision/Catalog Sync WebJob/Run.ps1 @@ -338,6 +338,19 @@ While (1 -eq 1) } } } + + # validate that all extended database entries have a corresponding extended tenant entry + foreach($catalogDatabase in $catalogDatabases) + { + $compoundDatabaseName = "$($CatalogDatabase.ServerName)/$($CatalogDatabase.DatabaseName)" + + if (-not $tenantDatabasesDict.ContainsKey($compoundDatabaseName)) + { + # if the database entry does not reflect a tenant, remove it (this can happen if the tenent has been removed) + Remove-ExtendedDatabase -Catalog $catalog -DatabaseName $CatalogDatabase.DatabaseName + } + } + } } } diff --git a/Learning Modules/Provision and Catalog/Auto Provision/New-TenantRequest.ps1 b/Learning Modules/Provision and Catalog/Auto Provision/New-TenantRequest.ps1 index 54e5fd1..7351c0a 100644 --- a/Learning Modules/Provision and Catalog/Auto Provision/New-TenantRequest.ps1 +++ b/Learning Modules/Provision and Catalog/Auto Provision/New-TenantRequest.ps1 @@ -41,8 +41,8 @@ Param( $WtpUser = $WtpUser.ToLower() -Import-Module $PSScriptRoot\..\Common\SubscriptionManagement -Force -Import-Module $PSScriptRoot\..\Common\CatalogAndDatabaseManagement -Force +Import-Module $PSScriptRoot\..\..\Common\SubscriptionManagement -Force +Import-Module $PSScriptRoot\..\..\Common\CatalogAndDatabaseManagement -Force $config = Get-Configuration @@ -57,7 +57,7 @@ $catalog = Get-Catalog -ResourceGroupName $WtpResourceGroupName -WtpUser $WtpUse # Validate tenant name $TenantName = $TenantName.Trim() Test-LegalName $TenantName > $null -Test-LegalVenueTypeName -Catalog $catalog -VenueType $VenueType > $null +Test-ValidVenueType -Catalog $catalog -VenueType $VenueType > $null # Compute the tenant key from the tenant name, key to be used to register the tenant in the catalog $tenantKey = Get-TenantKey -TenantName $TenantName diff --git a/Learning Modules/Provision and Catalog/Auto Provision/Provision Buffer Databases WebJob/Common/CatalogAndDatabaseManagement.psm1 b/Learning Modules/Provision and Catalog/Auto Provision/Provision Buffer Databases WebJob/Common/CatalogAndDatabaseManagement.psm1 index 7657d1e..6e88b77 100644 --- a/Learning Modules/Provision and Catalog/Auto Provision/Provision Buffer Databases WebJob/Common/CatalogAndDatabaseManagement.psm1 +++ b/Learning Modules/Provision and Catalog/Auto Provision/Provision Buffer Databases WebJob/Common/CatalogAndDatabaseManagement.psm1 @@ -1216,7 +1216,6 @@ function Remove-CatalogInfoFromTenantDatabase -Password $adminPassword ` -ServerInstance ($TenantDatabase.ServerName + ".database.windows.net") ` -Database $TenantDatabase.DatabaseName ` - -ConnectionTimeout 30 ` -Query $commandText ` } @@ -1231,13 +1230,16 @@ function Remove-ExtendedDatabase [parameter(Mandatory=$true)] [object]$Catalog, + [parameter(Mandatory=$true)] + [string]$ServerName, + [parameter(Mandatory=$true)] [string]$DatabaseName ) $commandText = " DELETE FROM Databases - WHERE DatabaseName = $DatabaseName;" + WHERE ServerName = '$ServerName' AND DatabaseName = '$DatabaseName';" Invoke-SqlAzureWithRetry ` -ServerInstance $Catalog.FullyQualifiedServerName ` @@ -1245,8 +1247,6 @@ function Remove-ExtendedDatabase -Password $config.CatalogAdminPassword ` -Database $Catalog.Database.DatabaseName ` -Query $commandText ` - -ConnectionTimeout 30 ` - -QueryTimeout 30 ` } @@ -1277,9 +1277,7 @@ function Remove-ExtendedElasticPool{ -Database $Catalog.Database.DatabaseName ` -Query $commandText ` -UserName $config.CatalogAdminUserName ` - -Password $config.CatalogAdminPassword ` - -ConnectionTimeout 30 ` - -QueryTimeout 15 + -Password $config.CatalogAdminPassword } @@ -1313,7 +1311,7 @@ function Remove-ExtendedServer <# .SYNOPSIS - Removes extended tenant and associated database meta data entries from catalog + Removes extended tenant entry from catalog #> function Remove-ExtendedTenant { @@ -1341,9 +1339,7 @@ function Remove-ExtendedTenant # Delete the tenant name from the Tenants table $commandText = " DELETE FROM Tenants - WHERE TenantId = $rawkeyHexString; - DELETE FROM Databases - WHERE ServerName = '$ServerName' AND DatabaseName = '$DatabaseName';" + WHERE TenantId = $rawkeyHexString;" Invoke-SqlAzureWithRetry ` -ServerInstance $Catalog.FullyQualifiedServerName ` diff --git a/Learning Modules/Provision and Catalog/Demo-ProvisionAndCatalog.ps1 b/Learning Modules/Provision and Catalog/Demo-ProvisionAndCatalog.ps1 index e103537..8510e6b 100644 --- a/Learning Modules/Provision and Catalog/Demo-ProvisionAndCatalog.ps1 +++ b/Learning Modules/Provision and Catalog/Demo-ProvisionAndCatalog.ps1 @@ -10,6 +10,9 @@ $TenantName = "Red Maple Racing" $VenueType = "motorracing" # Supported venue types: blues, classicalmusic, dance, jazz, judo, motorracing, multipurpose, opera, rockmusic, soccer +# Postal Code of the venue +$PostalCode = "98052" + $DemoScenario = 1 <# Select the demo scenario to run Demo Scenario @@ -48,14 +51,12 @@ if ($DemoScenario -eq 1) -ServerName $serverName ` -PoolName $poolName ` -VenueType $VenueType ` + -PostalCode $PostalCode ` -ErrorAction Stop ` > $null Write-Output "Provisioning complete for tenant '$TenantName'" - # Open the admin page for the new venue - Start-Process "http://admin.wtp.$($wtpUser.Name).trafficmanager.net/$(Get-NormalizedTenantName $TenantName)" - # Open the events page for the new venue Start-Process "http://events.wtp.$($wtpUser.Name).trafficmanager.net/$(Get-NormalizedTenantName $TenantName)" diff --git a/Learning Modules/Provision and Catalog/New-Tenant.ps1 b/Learning Modules/Provision and Catalog/New-Tenant.ps1 deleted file mode 100644 index d3a4466..0000000 --- a/Learning Modules/Provision and Catalog/New-Tenant.ps1 +++ /dev/null @@ -1,84 +0,0 @@ -<# -.SYNOPSIS - Provisions a new Wingtip Tickets Platform (WTP) tenant and registers it in the catalog - -.DESCRIPTION - Creates a new database, imports the WTP tenant bacpac using an ARM template, and initializes - venue information. The tenant database is then in the catalog database. The tenant - name is used as the basis of the database name and to generate the tenant key - used in the catalog. - -.PARAMETER WtpResourceGroupName - The resource group name used during the deployment of the WTP app (case sensitive) - -.PARAMETER WtpUser - The 'User' value that was entered during the deployment of the WTP app - -.PARAMETER TenantName - The name of the tenant being provisioned -#> -function New-TenantX -{ - Param( - [Parameter(Mandatory=$true)] - [string]$WtpResourceGroupName, - - [Parameter(Mandatory=$true)] - [string]$WtpUser, - - [Parameter(Mandatory=$true)] - [string]$TenantName, - - [Parameter(Mandatory=$false)] - [string]$VenueType - ) - - $WtpUser = $WtpUser.ToLower() - - Import-Module $PSScriptRoot\..\Common\SubscriptionManagement -Force - Import-Module $PSScriptRoot\..\Common\CatalogAndDatabaseManagement -Force - - $config = Get-Configuration - - # Ensure logged in to Azure - Initialize-Subscription - - ## MAIN SCRIPT ## ---------------------------------------------------------------------------- - - # Get the catalog - $catalog = Get-Catalog -ResourceGroupName $WtpResourceGroupName -WtpUser $WtpUser - - # Validate tenant name - $TenantName = $TenantName.Trim() - Test-LegalName $TenantName > $null - Test-LegalVenueTypeName $VenueType > $null - - # Compute the tenant key from the tenant name, key to be used to register the tenant in the catalog - $tenantKey = Get-TenantKey -TenantName $TenantName - - # Check if a tenant with this key is aleady registered in the catalog - if (Test-TenantKeyInCatalog -Catalog $catalog -TenantKey $tenantKey) - { - throw "A tenant with name '$TenantName' is already registered in the catalog." - } - - $tenantServerName = $config.TenantServerNameStem + $WtpUser - $tenantPoolName = $config.TenantPoolNameStem + "1" - - # Deploy and initialize a database for this tenant - $tenantDatabase = New-TenantDatabase ` - -ResourceGroupName $WtpResourceGroupName ` - -ServerName $tenantServerName ` - -ElasticPoolName $tenantPoolName ` - -TenantName $TenantName ` - -VenueType $VenueType ` - -WtpUser $WtpUser - - # Register the tenant and database in the catalog - Add-TenantDatabaseToCatalog -Catalog $catalog ` - -TenantName $TenantName ` - -TenantKey $tenantKey ` - -TenantDatabase $tenantDatabase ` - - Write-Output "Provisioning complete for tenant '$TenantName'" -} \ No newline at end of file diff --git a/Learning Modules/Provision and Catalog/New-TenantBatch.ps1 b/Learning Modules/Provision and Catalog/New-TenantBatch.ps1 index fd28f7f..d57d2ab 100644 --- a/Learning Modules/Provision and Catalog/New-TenantBatch.ps1 +++ b/Learning Modules/Provision and Catalog/New-TenantBatch.ps1 @@ -67,6 +67,7 @@ foreach ($newTenant in $NewTenants) { $newTenantName = $newTenant[0].Trim() $newTenantVenueType = $newTenant[1].Trim() + $newTenantPostalCode = $newTenant[2].Trim() try { @@ -83,6 +84,7 @@ foreach ($newTenant in $NewTenants) Name = $newTenantName NormalizedName = $normalizedNewTenantName VenueType = $newTenantVenueType + PostalCode = $newTenantPostalCode } $allNewTenants += $newTenantObj @@ -118,7 +120,7 @@ foreach ($newTenant in $NewTenants) if ($batchDatabaseNames.Count -gt 0) { - Write-Output "Provisioning $($newTenantDatabaseNames.Count) databases..." + Write-Output "Provisioning $($batchDatabaseNames.Count) databases..." try { @@ -164,7 +166,8 @@ foreach($tenant in $allNewTenants) -ServerName $serverName ` -DatabaseName $tenant.NormalizedName ` -TenantName $tenant.Name ` - -VenueType $tenant.VenueType + -VenueType $tenant.VenueType ` + -PostalCode $tenant.PostalCode $tenantKey = Get-TenantKey -TenantName $tenant.Name diff --git a/Learning Modules/Schema Management/Deploy-JobAccount.ps1 b/Learning Modules/Schema Management/Deploy-JobAccount.ps1 index c03494f..cccf1b2 100644 --- a/Learning Modules/Schema Management/Deploy-JobAccount.ps1 +++ b/Learning Modules/Schema Management/Deploy-JobAccount.ps1 @@ -2,7 +2,7 @@ <# .SYNOPSIS - Creates an Elastic job account + Creates an elastic job account and associated database .DESCRIPTION Creates the Job account database and then the job account. Both are created in the resource group @@ -23,7 +23,6 @@ Import-Module "$PSScriptRoot\..\Common\SubscriptionManagement" -Force Import-Module "$PSScriptRoot\..\WtpConfig" -Force $config = Get-Configuration -$TenantName = "Contoso Concert Hall" # Get Azure credentials if not already logged on. Initialize-Subscription @@ -40,6 +39,31 @@ $catalogServerName = $config.CatalogServerNameStem + $WtpUser $fullyQualfiedCatalogServerName = $catalogServerName + ".database.windows.net" $databaseName = $config.JobAccountDatabaseName +# Check if the job account already exists and the latest Azure PowerShell SDK has been installed +try +{ + $jobaccount = Get-AzureRmSqlJobAccount -ResourceGroupName $WtpResourceGroupName ` + -ServerName $CatalogServerName ` + -JobAccountName $($config.JobAccount) +} +catch +{ + if ($_.Exception.Message -like "*'Get-AzureRmSqlJobAccount' is not recognized*") + { + Write-Error "'Get-AzureRmSqlJobAccount' not found. Download and install the Azure PowerShell SDK that includes support for Elastic Jobs: + https://github.com/jaredmoo/azure-powershell/releases" + } +} + +# Check if current Azure subscription is signed up for Preview of Elastic jobs +$registrationStatus = Get-AzureRmProviderFeature -ProviderName Microsoft.Sql -FeatureName sqldb-JobAccounts + +if ($registrationStatus.RegistrationState -eq "NotRegistered") +{ + Write-Error "Your current subscription is not white-listed for the preview of Elastic jobs. Please contact Microsoft to white-list your subscription." + throw +} + # Check the job account database already exists $database = Get-AzureRmSqlDatabase -ResourceGroupName $WtpResourceGroupName ` -ServerName $catalogServerName ` @@ -68,12 +92,6 @@ catch throw } -# Check the job account already exists -$jobaccount = Get-AzureRmSqlJobAccount -ResourceGroupName $WtpResourceGroupName ` - -ServerName $CatalogServerName ` - -JobAccountName $($config.JobAccount) ` - -ErrorAction SilentlyContinue - # Create the job account if it doesn't already exist try { @@ -96,11 +114,12 @@ catch throw } +$credentialName = $config.JobAccountCredentialName $commandText = " CREATE MASTER KEY; GO - CREATE DATABASE SCOPED CREDENTIAL [mydemocred] + CREATE DATABASE SCOPED CREDENTIAL [$credentialName] WITH IDENTITY = N'$($config.CatalogAdminUserName)', SECRET = N'$($config.CatalogAdminPassword)'; GO @@ -141,5 +160,3 @@ $commandText = " Write-Output "Deployment of job account database '$($config.JobAccountDatabaseName)' and job account '$($config.JobAccount)' are complete." -# Open the admin page for the Contoso Concert Hall tenant to view venue types available -Start-Process "http://admin.wtp.$WtpUser.trafficmanager.net/$($normalizedTenantName)" \ No newline at end of file diff --git a/Learning Modules/Schema Management/DeployReferenceData.sql b/Learning Modules/Schema Management/DeployReferenceData.sql index 9fb2824..f9c87f6 100644 --- a/Learning Modules/Schema Management/DeployReferenceData.sql +++ b/Learning Modules/Schema Management/DeployReferenceData.sql @@ -49,18 +49,18 @@ EXEC jobs.sp_add_jobstep @command=N' MERGE INTO [dbo].[VenueTypes] AS [target] USING (VALUES - (''MultiPurposeVenue'',''Multi Purpose Venue'',''Event'', ''Event'',''Events'',''en-us''), - (''ClassicalConcertHall'',''Classical Concert Hall'',''Classical Concert'',''Concert'',''Concerts'',''en-us''), - (''JazzClub'',''Jazz Club'',''Jazz Session'',''Session'',''Sessions'',''en-us''), - (''JudoClub'',''Judo Club'',''Judo Tournament'',''Tournament'',''Tournaments'',''en-us''), - (''SoccerClub'',''Soccer Club'',''Soccer Match'', ''Match'',''Matches'',''en-us''), - (''MotorRacing'',''Motor Racing'',''Car Race'', ''Race'',''Races'',''en-us''), - (''DanceStudio'', ''Dance Studio'', ''Performance'', ''Performance'', ''Performances'',''en-us''), - (''BluesClub'', ''Blues Club'', ''Blues Session'', ''Session'',''Sessions'',''en-us'' ), - (''RockMusicVenue'',''Rock Music Venue'',''Rock Concert'',''Concert'', ''Concerts'',''en-us''), - (''Opera'',''Opera'',''Opera'',''Opera'',''Operas'',''en-us''), - (''MotorCycleRacing'',''Motorcycle Racing'',''Motorcycle Race'', ''Race'', ''Races'', ''en-us''), -- NEW - (''SwimmingClub'',''Swimming Club'',''Swimming Race'',''Race'',''Races'',''en-us'') -- NEW + (''multipurpose'',''Multi-Purpose'',''Event'', ''Event'',''Events'',''en-us''), + (''classicalmusic'',''Classical Music'',''Classical Concert'',''Concert'',''Concerts'',''en-us''), + (''jazz'',''Jazz'',''Jazz Session'',''Session'',''Sessions'',''en-us''), + (''judo'',''Judo'',''Judo Tournament'',''Tournament'',''Tournaments'',''en-us''), + (''soccer'',''Soccer'',''Soccer Match'', ''Match'',''Matches'',''en-us''), + (''motorracing'',''Motor Racing'',''Car Race'', ''Race'',''Races'',''en-us''), + (''dance'', ''Dance'', ''Performance'', ''Performance'', ''Performances'',''en-us''), + (''blues'', ''Blues'', ''Blues Session'', ''Session'',''Sessions'',''en-us'' ), + (''rockmusic'',''Rock Music'',''Rock Concert'',''Concert'', ''Concerts'',''en-us''), + (''opera'',''Opera'',''Opera'',''Opera'',''Operas'',''en-us''), + (''motorcycleracing'',''Motorcycle Racing'',''Motorcycle Race'', ''Race'', ''Races'', ''en-us''), -- NEW + (''swimming'',''Swimming'',''Swimming Race'',''Race'',''Races'',''en-us'') -- NEW ) AS source( VenueType,VenueTypeName,EventTypeName,EventTypeShortName,EventTypeShortNamePlural,[Language] ) diff --git a/Learning Modules/Utilities/Demo-InvokeSqlCmdOnTenantDatabases.ps1 b/Learning Modules/Utilities/Demo-InvokeSqlCmdOnTenantDatabases.ps1 index fdf041e..96b56db 100644 --- a/Learning Modules/Utilities/Demo-InvokeSqlCmdOnTenantDatabases.ps1 +++ b/Learning Modules/Utilities/Demo-InvokeSqlCmdOnTenantDatabases.ps1 @@ -1,27 +1,12 @@ # Helper script for invoking Apply-SQLCommandToTenantDatabases. # Crude way to apply a one-time script against tenant dbs in catalog. Use Elastic Jobs for any serious work...! -# SQL command to be applied. Script should be idempotent as will retry on error. No results are returned, check dbs for success. -$commandText = " - DROP VIEW IF EXISTS VenueEvents - GO - CREATE VIEW VenueEvents AS - SELECT (SELECT TOP 1 VenueName FROM Venue) AS VenueName, EventId, EventName, Date FROM [events] - GO - - DROP VIEW IF EXISTS VenueTicketPurchases - GO - CREATE VIEW VenueTicketPurchases AS - SELECT (SELECT TOP 1 VenueName FROM Venue) AS VenueName, TicketPurchaseId, PurchaseDate, PurchaseTotal, CustomerId FROM [TicketPurchases] - GO - - DROP VIEW IF EXISTS VenueTickets - GO - CREATE VIEW VenueTickets AS - SELECT (SELECT TOP 1 VenueName FROM Venue) AS VenueName, TicketId, RowNumber, SeatNumber, EventId, SectionId, TicketPurchaseId FROM [Tickets] - GO +# SQL script to be applied. Uses Invoke-SqlCmd so can include batches with GO statements. +# Script should be idempotent as will retry on error. No results are returned, check dbs for success. +$commandText = " + -- Add script to be deployed here " - + # query timeout in seconds $queryTimeout = 60 diff --git a/Learning Modules/Utilities/Demo-LoadGenerator.ps1 b/Learning Modules/Utilities/Demo-LoadGenerator.ps1 index 70aacbd..a5a337a 100644 --- a/Learning Modules/Utilities/Demo-LoadGenerator.ps1 +++ b/Learning Modules/Utilities/Demo-LoadGenerator.ps1 @@ -1,9 +1,7 @@ # Invokes load generator script over the tenant databases currently defined in the catalog. - -# NOTE: If additional tenants are added restart the generator to include them. # Duration of the load generation session. Some activity may continue after this time. -$DurationMinutes = 60 +$DurationMinutes = 120 # If SingleTenant is enabled (scenario 4), this specifies the tenant database to be overloaded. # If set to "" a random tenant database is chosen. @@ -42,7 +40,7 @@ if ($DemoScenario -eq 0) if ($DemoScenario -eq 1) { # First, stop and remove any prior running jobs - Write-Output "Stopping any prior jobs. This can take a minute or more... " + Write-Output "`nStopping any prior jobs. This can take a minute or more... " Remove-Job * -Force # Intensity of load, roughly approximates to average eDTU loading on the pool @@ -62,7 +60,7 @@ if ($DemoScenario -eq 1) if ($DemoScenario -eq 2) { # First, stop and remove any prior running jobs - Write-Output "Stopping any prior jobs. This can take a minute or more... " + Write-Output "`nStopping any prior jobs. This can take a minute or more... " Remove-Job * -Force # Intensity of load, roughly approximates to average eDTU loading on the pool @@ -83,7 +81,7 @@ if ($DemoScenario -eq 2) if ($DemoScenario -eq 3) { # First, stop and remove any prior running jobs - Write-Output "Stopping any prior jobs. This can take a minute or more... " + Write-Output "`nStopping any prior jobs. This can take a minute or more... " Remove-Job * -Force # Intensity of load, roughly approximates to average eDTU loading on the pool @@ -103,7 +101,7 @@ if ($DemoScenario -eq 3) if ($DemoScenario -eq 4) { # First, stop and remove any prior running jobs - Write-Output "Stopping any prior jobs. This can take a minute or more... " + Write-Output "`nStopping any prior jobs. This can take a minute or more... " Remove-Job * -Force # Intensity of load, roughly approximates to average eDTU loading on the pool @@ -125,7 +123,7 @@ if ($DemoScenario -eq 4) if ($DemoScenario -eq 5) { # First, stop and remove any prior running jobs - Write-Output "Stopping any prior jobs. This can take a minute or more... " + Write-Output "`nStopping any prior jobs. This can take a minute or more... " Remove-Job * -Force # Intensity of load, roughly approximates to average eDTU loading on the pool diff --git a/Learning Modules/Utilities/FictitiousNames.csv b/Learning Modules/Utilities/FictitiousNames.csv index 34e22c1..ab6eed7 100644 --- a/Learning Modules/Utilities/FictitiousNames.csv +++ b/Learning Modules/Utilities/FictitiousNames.csv @@ -1,4 +1,3 @@ -#,First Name,Last Name,Language,Gender 1,Deb,Sen,Bengali,Male 2,Ajit,Mazumdar,Bengali,Male 3,Shaan,Dasgupta,Bengali,Male diff --git a/Learning Modules/Utilities/Invoke-SqlCmdOnTenantDatabases.ps1 b/Learning Modules/Utilities/Invoke-SqlCmdOnTenantDatabases.ps1 index 209b5ee..635c929 100644 --- a/Learning Modules/Utilities/Invoke-SqlCmdOnTenantDatabases.ps1 +++ b/Learning Modules/Utilities/Invoke-SqlCmdOnTenantDatabases.ps1 @@ -1,10 +1,9 @@ <# .SYNOPSIS - Deploys a SQLcommand against all tenant databases in the catalog sequentially. Assumes command is idempotent as - it will do a simply one-time retry. For serious work use Elastic Jobs. + Deploys a SQL script against all tenant databases in the catalog sequentially and to the golden tenant database. + The script should be idempotent as it will retry on error, dropped connection etc. + Lightweight solution to fanout deployment. Use Elastic Jobs for a more robust solution. #> - - [cmdletbinding()] param( [Parameter(Mandatory=$true)] @@ -24,6 +23,9 @@ Import-Module $PSScriptRoot\..\Common\SubscriptionManagement -Force Import-Module $PSScriptRoot\..\Common\CatalogAndDatabaseManagement -Force $config = Get-Configuration + +## Apply script to deployed tenant databases + $adminUserName = $config.TenantAdminUserName $adminPassword = $config.TenantAdminPassword @@ -32,37 +34,39 @@ $catalog = Get-Catalog ` -ResourceGroupName $WtpResourceGroupName ` -WtpUser $WtpUser - # Get all the databases in the catalog shard map $shards = Get-Shards -ShardMap $catalog.ShardMap foreach ($shard in $Shards) { - try - { - Write-Output "Applying command to database '$($shard.Location.Database)' on server '$($shard.Location.Server)'." - Invoke-Sqlcmd ` - -Username $adminUserName ` - -Password $adminPassword ` - -ServerInstance $shard.Location.Server ` - -Database $shard.Location.Database ` - -ConnectionTimeout 30 ` - -QueryTimeout $QueryTimeout ` - -Query $CommandText ` - -EncryptConnection - } - catch - { - # one time retry if errors - Invoke-Sqlcmd ` - -Username $adminUserName ` - -Password $adminPassword ` - -ServerInstance $shard.Location.Server ` - -Database $shard.Location.Database ` - -ConnectionTimeout 30 ` - -QueryTimeout $QueryTimeout ` - -Query $CommandText ` - -EncryptConnection - } -} \ No newline at end of file + Write-Output "Applying script to database '$($shard.Location.Database)' on server '$($shard.Location.Server)'." + Invoke-SqlcmdWithRetry ` + -Username $adminUserName ` + -Password $adminPassword ` + -ServerInstance $shard.Location.Server ` + -Database $shard.Location.Database ` + -ConnectionTimeout 30 ` + -QueryTimeout $QueryTimeout ` + -Query $CommandText + +} + +## Apply script to the golden tenant database on the catalog server so new tenants databases will have the script applied + +$adminUserName = $config.CatalogAdminUserName +$adminPassword = $config.CatalogAdminPassword + +$catalogServer = $config.CatalogServerNameStem + $WtpUser +$fullyQualifiedCatalogServerName = $catalogServer + ".database.windows.net" +$goldenTenantDatabase = $config.GoldenTenantDatabaseName + + Write-Output "Applying script to database '$goldenTenantDatabase' on server '$catalogServer'." + Invoke-SqlcmdWithRetry ` + -Username $adminUserName ` + -Password $adminPassword ` + -ServerInstance $fullyQualifiedCatalogServerName ` + -Database $goldenTenantDatabase ` + -ConnectionTimeout 30 ` + -QueryTimeout $QueryTimeout ` + -Query $CommandText \ No newline at end of file diff --git a/Learning Modules/Utilities/LoadGenerator.ps1 b/Learning Modules/Utilities/LoadGenerator.ps1 index 7878e2f..73ce9e1 100644 --- a/Learning Modules/Utilities/LoadGenerator.ps1 +++ b/Learning Modules/Utilities/LoadGenerator.ps1 @@ -16,7 +16,7 @@ Param( # Duration of the load generation session in minutes. Due to the way loads are applied, some # activity may continue after this time. - [int]$DurationMinutes = 60, + [int]$DurationMinutes = 120, # If enabled, causes a single tenant to have a specific distinct load applied # Use with SingleTenantIntensity to demonstrate moving a database in or out of a pool @@ -83,267 +83,337 @@ $densityLoadFactor = 0.08 $CatalogServerName = $config.CatalogServerNameStem + $WtpUser -$shards = Get-Shards -ShardMap $catalog.ShardMap -$ServerNames = @() -foreach ($shard in $Shards) -{ - $serverName = $shard.Location.Server.split(".",2)[0] - $ServerNames += $serverName -} +$jobs = @{} -$serverNames = $serverNames| sort | Get-Unique - -# Array that will contain all databases to be targeted -$allDbs = @() +## Start job invocation loop + +$start = Get-Date + +$sleepCount = 0 +$sleep = 10 + +Write-Output "`nStarting job execution. Load generator will look for new databases every $sleep seconds for $durationMinutes ." +write-output "`nUse Ctrl-C to stop invoking jobs. Already started jobs will continue and can be managed:." + +Write-Output "`n Use Get-Job to view status of all jobs" +Write-Output " Use Receive-Job -Keep to view output from an individual job" +Write-Output " Use Stop-Job to stop a job. Use Stop-Job * to stop all jobs (which can take a minute or more)" +Write-Output " Use Remove-Job to remove a job. Use Remove-Job * to remove all jobs. Use -Force to stop and remove.`n" -foreach($serverName in $serverNames) +while (1 -eq 1) { - [array]$serverPools = (Get-AzureRmSqlElasticPool -ResourceGroupName $WtpResourceGroupName -ServerName $serverName).ElasticPoolName - $poolNumber = 1 + $shards = Get-Shards -ShardMap $catalog.ShardMap - foreach($elasticPool in $serverPools) + $ServerNames = @() + foreach ($shard in $Shards) { - # Set up the relative load level for each pool - if($Unbalanced.IsPresent -and $serverPools.Count -gt 1) + $serverName = $shard.Location.Server.split(".",2)[0] + $ServerNames += $serverName + } + + $serverNames = $serverNames| sort | Get-Unique + + # Array that will contain all databases to be targeted + $allDbs = @() + + foreach($serverName in $serverNames) + { + [array]$serverPools = (Get-AzureRmSqlElasticPool -ResourceGroupName $WtpResourceGroupName -ServerName $serverName).ElasticPoolName + $poolNumber = 1 + + foreach($elasticPool in $serverPools) { - # alternating pools on the same server are given high and low loads - if ($poolNumber % 2 -ne 0) + # Set up the relative load level for each pool + if($Unbalanced.IsPresent -and $serverPools.Count -gt 1) { - $loadFactor = $highPoolLoadFactor - Write-Output ("Pool " + $elasticPool + " on " + $serverName + " has high load factor - " + $loadFactor) + # alternating pools on the same server are given high and low loads + if ($poolNumber % 2 -ne 0) + { + $loadFactor = $highPoolLoadFactor + Write-Output ("Pool " + $elasticPool + " on " + $serverName + " has high load factor - " + $loadFactor) + } + else + { + $loadFactor = $lowPoolLoadFactor + Write-Output ("Pool "+ $elasticPool + " on " + $serverName + " has low load factor - " + $loadFactor) + } } else { - $loadFactor = $lowPoolLoadFactor - Write-Output ("Pool "+ $elasticPool + " on " + $serverName + " has low load factor - " + $loadFactor) + # Neutral default for the relative load level for databases in a pool + $loadFactor = 1.0 } - } - else - { - # Neutral default for the relative load level for databases in a pool - $loadFactor = 1.0 - } - $elasticDbs = (Get-AzureRmSqlElasticPoolDatabase -ResourceGroupName $WtpResourceGroupName -ServerName $serverName -ElasticPoolName $elasticPool).DatabaseName + $elasticDbs = (Get-AzureRmSqlElasticPoolDatabase -ResourceGroupName $WtpResourceGroupName -ServerName $serverName -ElasticPoolName $elasticPool).DatabaseName - Foreach($elasticDb in $elasticDbs) - { - # vary the baseline DTU level of each database using a random factor x the input intensity x load factor for the pool + Foreach($elasticDb in $elasticDbs) + { + # vary the baseline DTU level of each database using a random factor x the input intensity x load factor for the pool - $burstFactor = Get-Random -Minimum $burstMinFactor -Maximum $burstMaxFactor # randomizes the intensity of each database - $burstDtu = [math]::Ceiling($Intensity * $BurstFactor * $loadFactor) + $burstFactor = Get-Random -Minimum $burstMinFactor -Maximum $burstMaxFactor # randomizes the intensity of each database + $burstDtu = [math]::Ceiling($Intensity * $BurstFactor * $loadFactor) - # add db with its pool-based load factor to the list - $dbProperties = @{ServerName=($serverName + ".database.windows.net");DatabaseName=$elasticDb;BurstDtu=$burstDtu;LoadFactor=$loadFactor;ElasticPoolName=$elasticPool;PoolDbCount=$elasticDbs.Count} - $db = New-Object PSObject -Property $dbProperties + # add db with its pool-based load factor to the list + $dbProperties = @{ServerName=($serverName + ".database.windows.net");DatabaseName=$elasticDb;BurstDtu=$burstDtu;LoadFactor=$loadFactor;ElasticPoolName=$elasticPool;PoolDbCount=$elasticDbs.Count} + $db = New-Object PSObject -Property $dbProperties - $allDbs += $db - } + $allDbs += $db + } - $poolNumber ++ - } - - Write-Output "" + $poolNumber ++ + } - # Get standalone dbs and add to $allDbs + # Get standalone dbs and add to $allDbs - $StandaloneDbs = (Get-AzureRmSqlDatabase -ResourceGroupName $WtpResourceGroupName -ServerName $serverName | where {$_.CurrentServiceObjectiveName -ne "ElasticPool"} | where {$_.DatabaseName -ne "master"} ).DatabaseName - Foreach ($standaloneDb in $StandaloneDbs) - { - $burstLevel = Get-Random -Minimum $burstMinFactor -Maximum $burstMaxFactor # randomizes the intensity of each database - $burstDtu = [math]::Ceiling($burstLevel * $Intensity) + $StandaloneDbs = (Get-AzureRmSqlDatabase -ResourceGroupName $WtpResourceGroupName -ServerName $serverName | where {$_.CurrentServiceObjectiveName -ne "ElasticPool"} | where {$_.DatabaseName -ne "master"} ).DatabaseName + Foreach ($standaloneDb in $StandaloneDbs) + { + $burstLevel = Get-Random -Minimum $burstMinFactor -Maximum $burstMaxFactor # randomizes the intensity of each database + $burstDtu = [math]::Ceiling($burstLevel * $Intensity) - #store db with a neutral load factor - $dbProperties = @{ServerName=($serverName + ".database.windows.net");DatabaseName=$StandaloneDb;BurstDtu=$burstDtu;LoadFactor=1.0;ElasticPoolName="";PoolDbCount=0.0} - $db = New-Object PSObject -Property $dbProperties + #store db with a neutral load factor + $dbProperties = @{ServerName=($serverName + ".database.windows.net");DatabaseName=$StandaloneDb;BurstDtu=$burstDtu;LoadFactor=1.0;ElasticPoolName="";PoolDbCount=0.0} + $db = New-Object PSObject -Property $dbProperties - $allDbs += $db - } -} + $allDbs += $db + } + } -if ($SingleTenant.IsPresent -and $SingleTenantDatabaseName -ne "") -{ - $SingleTenantDatabaseName = Get-NormalizedTenantname $SingleTenantDatabaseName + if ($SingleTenant.IsPresent -and $SingleTenantDatabaseName -ne "") + { + $SingleTenantDatabaseName = Get-NormalizedTenantname $SingleTenantDatabaseName - #validate that the name is one of the database names about to be processed - $allDBNames = $allDBs | select -ExpandProperty DatabaseName + #validate that the name is one of the database names about to be processed + $allDBNames = $allDBs | select -ExpandProperty DatabaseName - if (-not ($allDBNames -contains $SingleTenantDatabaseName)) - { - throw "The Single Tenant Database Name '$SingleTenantDatabaseName' was not found. Check the spelling and try again." - } -} + if (-not ($allDBNames -contains $SingleTenantDatabaseName)) + { + throw "The Single Tenant Database Name '$SingleTenantDatabaseName' was not found. Check the spelling and try again." + } + } -# spawn jobs to spin up load on each database in $allDbs -# note there are limits to using PS jobs at scale; this should only be used for small scale demonstrations + # spawn jobs to spin up load on each database in $allDbs + # note there are limits to using PS jobs at scale; this should only be used for small scale demonstrations -# Set the end time for all jobs -$endTime = [DateTime]::Now.AddMinutes($DurationMinutes) + # Set the end time for all jobs + $endTime = [DateTime]::Now.AddMinutes($DurationMinutes) -# Script block for job that executes the load generation stored procedure on each database -$scriptBlock = ` - { - param($server,$dbName,$AdminUser,$AdminPassword,$DurationMinutes,$intervalMin,$intervalMax,$burstMinDuration,$burstMaxDuration,$baseDtu,$loadFactor,$densityLoadFactor,$poolDbCount) + # Script block for job that executes the load generation stored procedure on each database + $scriptBlock = ` + { + param($server,$dbName,$AdminUser,$AdminPassword,$DurationMinutes,$intervalMin,$intervalMax,$burstMinDuration,$burstMaxDuration,$baseDtu,$loadFactor,$densityLoadFactor,$poolDbCount) - import-module sqlserver + import-module sqlserver - Write-Output ("Database " + $dbName + "/" + $server + " Load factor: " + $loadFactor + " Density weighting: " + ($densityLoadFactor*$poolDbCount)) + Write-Output ("Database " + $dbName + "/" + $server + " Load factor: " + $loadFactor + " Density weighting: " + ($densityLoadFactor*$poolDbCount)) - $endTime = [DateTime]::Now.AddMinutes($DurationMinutes) + $endTime = [DateTime]::Now.AddMinutes($DurationMinutes) - $firstTime = $true + $firstTime = $true - While ([DateTime]::Now -lt $endTime) - { - # add variable delay before execution, this staggers bursts - # load factor is applied to reduce interval for high or intense loads, and increase interval for low loads - # density load factor extends interval for higher density pools to reduce overloading - if($firstTime) + While ([DateTime]::Now -lt $endTime) { - $snooze = [math]::ceiling((Get-Random -Minimum 0 -Maximum ($intervalMax - $intervalMin)) / $loadFactor) - $snooze = $snooze + ($snooze * $densityLoadFactor * $poolDbCount) - $firstTime = $false - } - else - { - $snooze = [math]::ceiling((Get-Random -Minimum $intervalMin -Maximum $intervalMax) / $loadFactor) - $snooze = $snooze + ($snooze * $densityLoadFactor * $poolDbCount) + # add variable delay before execution, this staggers bursts + # load factor is applied to reduce interval for high or intense loads, and increase interval for low loads + # density load factor extends interval for higher density pools to reduce overloading + if($firstTime) + { + $snooze = [math]::ceiling((Get-Random -Minimum 0 -Maximum ($intervalMax - $intervalMin)) / $loadFactor) + $snooze = $snooze + ($snooze * $densityLoadFactor * $poolDbCount) + $firstTime = $false + } + else + { + $snooze = [math]::ceiling((Get-Random -Minimum $intervalMin -Maximum $intervalMax) / $loadFactor) + $snooze = $snooze + ($snooze * $densityLoadFactor * $poolDbCount) + } + Write-Output ("Snoozing for " + $snooze + " seconds") + Start-Sleep $snooze + + # vary each burst to add realism to the workload + + # vary burst duration + $burstDuration = Get-Random -Minimum $burstMinDuration -Maximum $burstMaxDuration + + # Increase burst duration based on load factor. Has marginal effect on low loadfactor databases. + $burstDuration += ($loadFactor * 2) + + # vary DTU + $dtuVariance = Get-Random -Minimum 0.9 -Maximum 1.1 + $burstDtu = [Math]::ceiling($baseDtu * $dtuVariance) + + # ensure burst DTU doesn't exceed 100 + if($burstDtu -gt 100) + { + $burstDtu = 100 + } + + # configure and submit the SQL script to run the load generator + $sqlScript = "EXEC sp_CpuLoadGenerator @duration_seconds = " + $burstDuration + ", @dtu_to_simulate = " + $burstDtu + try + { + Invoke-Sqlcmd -ServerInstance $server ` + -Database $dbName ` + -Username $AdminUser ` + -Password $AdminPassword ` + -Query $sqlscript ` + -ConnectionTimeout 30 ` + -QueryTimeout 36000 + } + catch + { + write-error $_.Exception.Message + Write-Output ("Error connecting to tenant database " + $dbName + "/" + $server) + } + + [string]$message = $([DateTime]::Now) + Write-Output ( $message + " Starting load: " + $burstDtu + " DTUs for " + $burstDuration + " seconds") + + # exit loop if end time exceeded + if ([DateTime]::Now -gt $endTime) + { + break; + } } - Write-Output ("Snoozing for " + $snooze + " seconds") - Start-Sleep $snooze + } - # vary each burst to add realism to the workload - - # vary burst duration - $burstDuration = Get-Random -Minimum $burstMinDuration -Maximum $burstMaxDuration + # Start a job for each database. Each job runs for the specified session duration and triggers load periodically. + # The base-line load level for each db is set by the entry in $allDbs. Burst duration, interval and DTU are randomized + # slightly within each job to create a more realistic workload - # Increase burst duration based on load factor. Has marginal effect on low loadfactor databases. - $burstDuration += ($loadFactor * 2) + $randomTenantIndex = 0 - # vary DTU - $dtuVariance = Get-Random -Minimum 0.9 -Maximum 1.1 - $burstDtu = [Math]::ceiling($baseDtu * $dtuVariance) + if ($SingleTenant -and $SingleTenantDatabaseName -eq "") + { + $randomTenantIndex = Get-Random -Minimum 1 -Maximum ($allDbs.Count + 1) + } - # ensure burst DTU doesn't exceed 100 - if($burstDtu -gt 100) - { - $burstDtu = 100 - } + $i = 1 - # configure and submit the SQL script to run the load generator - $sqlScript = "EXEC sp_CpuLoadGenerator @duration_seconds = " + $burstDuration + ", @dtu_to_simulate = " + $burstDtu - try - { - Invoke-Sqlcmd -ServerInstance $server ` - -Database $dbName ` - -Username $AdminUser ` - -Password $AdminPassword ` - -Query $sqlscript ` - -ConnectionTimeout 30 ` - -QueryTimeout 36000 - } - catch - { - write-error $_.Exception.Message - Write-Output ("Error connecting to tenant database " + $dbName + "/" + $server) - } + foreach ($db in $allDBs) + { + # skip further processing if job is already started for this database + if ($jobs.ContainsKey($db.DatabaseName)) + { + continue + } - [string]$message = $([DateTime]::Now) - Write-Output ( $message + " Starting load: " + $burstDtu + " DTUs for " + $burstDuration + " seconds") + if($sleeping -eq $true) + { + $sleeping = $false + # emit next job output on a new line + Write-Output "`n" + } - # exit loop if end time exceeded - if ([DateTime]::Now -gt $endTime) + # Customize the load applied for each database + if ($SingleTenant) + { + if ($i -eq $randomTenantIndex) + { + # this is the randomly selected database, so use the single-tenant factors + $burstDtu = $SingleTenantDtu + $loadFactor = $intenseLoadFactor + } + elseif ($randomTenantIndex -eq 0 -and $SingleTenantDatabaseName -eq $db.DatabaseName) { - break; + # this is the named database, so use the single-tenant factors + $burstDtu = $SingleTenantDtu + $loadFactor = $intenseLoadFactor + } + else + { + # use per-db computed factors + $burstDtu = $db.BurstDtu + $loadFactor = $db.LoadFactor } } - } - -# Start a job for each database. Each job runs for the specified session duration and triggers load periodically. -# The base-line load level for each db is set by the entry in $allDbs. Burst duration, interval and DTU are randomized -# slightly within each job to create a more realistic workload - -$randomTenantIndex = 0 + else + { + # use per-db computed factors + $burstDtu = $db.BurstDtu + $loadFactor = $db.LoadFactor + } -if ($SingleTenant -and $SingleTenantDatabaseName -eq "") -{ - $randomTenantIndex = Get-Random -Minimum 1 -Maximum ($allDbs.Count + 1) -} + $poolDbCount = $db.PoolDbCount -$i = 1 + $i ++ -foreach ($db in $allDBs) -{ - # Customize the load applied for each database - if ($SingleTenant) - { - if ($i -eq $randomTenantIndex) + $outputText = " Starting load with load factor $loadFactor with baseline DTU $burstDtu" + if ($db.ElasticPoolName -ne "") { - # this is the randomly selected database, so use the single-tenant factors - $burstDtu = $SingleTenantDtu - $loadFactor = $intenseLoadFactor - } - elseif ($randomTenantIndex -eq 0 -and $SingleTenantDatabaseName -eq $db.DatabaseName) + $outputText += " in pool " + $db.ElasticPoolName + } + else { - # this is the named database, so use the single-tenant factors - $burstDtu = $SingleTenantDtu - $loadFactor = $intenseLoadFactor + $outputText += " standalone" } - else - { - # use per-db computed factors - $burstDtu = $db.BurstDtu - $loadFactor = $db.LoadFactor + + if ($LongerBursts.IsPresent) + { + $outputText += " [BOOSTED]" } + + # start the load generation job for the current database + + $job = Start-Job ` + -ScriptBlock $scriptBlock ` + -Name $db.DatabaseName ` + -ArgumentList $(` + $db.ServerName,$db.DatabaseName,` + $TenantAdminUser,$TenantAdminPassword,` + $DurationMinutes,$intervalMin,$intervalMax,` + $burstMinDuration,$burstMaxDuration,$burstDtu,` + $loadFactor,$densityLoadFactor,$poolDbCount) + + # add job to dictionary of currently running jobs + $jobs += @{$job.Name = $job} + + $outputText = ("Job $($job.Id) $($Job.Name) $outputText") + write-output $outputText } - else + $settings = "`nSettings: Duration: $DurationMinutes mins, Intensity: $intensity, LongerBursts: $LongerBursts, Unbalanced: $Unbalanced, SingleTenant: $SingleTenant" + + if($SingleTenant) { - # use per-db computed factors - $burstDtu = $db.BurstDtu - $loadFactor = $db.LoadFactor + if ($SingleTenantDatabaseName -ne "") + { + $settings += ", Database: $SingleTenantDatabaseName" + } + $settings += ", DTU: $singleTenantDtu" } - $poolDbCount = $db.PoolDbCount + $now = Get-Date + $runtime = ($now - $start).Minutes - $i ++ + if ($runtime -ge $DurationMinutes) + { + Write-Output "`n`nLoad generation session stopping after $runtime minutes" + exit + } - $outputText = " Starting load on " + $db.DatabaseName + "/" + $db.ServerName + " with load factor " + $loadFactor + " Baseline DTU " + $burstDtu - if ($db.ElasticPoolName -ne "") + if ($sleepCount -ge 59) { - $outputText += " in pool " + $db.ElasticPoolName + write-host "." + $sleepCount = 0 } else { - $outputText += " standalone" + write-host -NoNewline "." } - if ($LongerBursts.IsPresent) - { - $outputText += " [BOOSTED]" - } + $sleeping = $true + $sleepCount ++ - $job = Start-Job -ScriptBlock $scriptBlock -ArgumentList $(` - $db.ServerName,$db.DatabaseName,$TenantAdminUser,$TenantAdminPassword,$DurationMinutes,$intervalMin,$intervalMax,$burstMinDuration,$burstMaxDuration,$burstDtu,$loadFactor,$densityLoadFactor,$poolDbCount) - - $outputText = ("Job " + $job.Id + $outputText) - write-output $outputText -} -$settings = "`nSettings: Duration: $DurationMinutes mins, Intensity: $intensity, LongerBursts: $LongerBursts, Unbalanced: $Unbalanced, SingleTenant: $SingleTenant" - -if($SingleTenant) -{ - if ($SingleTenantDatabaseName -ne "") - { - $settings += ", Database: $SingleTenantDatabaseName" - } - $settings += ", DTU: $singleTenantDtu" + Sleep $sleep } +<# Write-output "`nAll database jobs started at $(Get-Date)" Write-Output $settings Write-Output "`nUse Get-Job to view status of all jobs" -Write-Output "Use Receive-Job -Keep to view output from an individual job" -Write-Output "Use Stop-Job to stop a job. Use Stop-Job * to stop all jobs (which can take a minute or more)" -Write-Output "Use Remove-Job to remove a job. Use Remove-Job * to remove all jobs. Use -Force to stop and remove." \ No newline at end of file +Write-Output "Use Receive-Job -Keep to view output from an individual job" +Write-Output "Use Stop-Job to stop a job. Use Stop-Job * to stop all jobs (which can take a minute or more)" +Write-Output "Use Remove-Job to remove a job. Use Remove-Job * to remove all jobs. Use -Force to stop and remove." +#> \ No newline at end of file diff --git a/Learning Modules/Utilities/SeattleZonedPostalCodes.csv b/Learning Modules/Utilities/SeattleZonedPostalCodes.csv index 6392ea1..06efd2a 100644 --- a/Learning Modules/Utilities/SeattleZonedPostalCodes.csv +++ b/Learning Modules/Utilities/SeattleZonedPostalCodes.csv @@ -1,4 +1,3 @@ -Zone,PostalCode 1,98101 1,98154 1,98164 @@ -94,7 +93,28 @@ Zone,PostalCode 7,98047 7,98003 7,98023 -7,98422 -7,98354 -7,98092 +8,98422 +8,98354 +8,98092 7,98372 +8,98422 +8,98421 +8,98402 +8,98403 +8,98405 +8,98407 +8,98404 +8,98409 +8,98408 +8,98443 +8,98445 +8,98446 +8,98373 +8,98374 +8,98375 +8,98207 +8,98201 +8,98205 +8,98271 +8,98270 +8,98258 diff --git a/Learning Modules/Utilities/TicketGenerator2.ps1 b/Learning Modules/Utilities/TicketGenerator2.ps1 index 5999fde..85d8906 100644 --- a/Learning Modules/Utilities/TicketGenerator2.ps1 +++ b/Learning Modules/Utilities/TicketGenerator2.ps1 @@ -12,17 +12,15 @@ Param ( # Resource Group Name entered during deployment [Parameter(Mandatory=$true)] - [String] - $WtpResourceGroupName, + [String]$WtpResourceGroupName, # The user name used entered during deployment [Parameter(Mandatory=$true)] - [String] - $WtpUser + [String]$WtpUser ) Import-Module "$PSScriptRoot\..\Common\SubscriptionManagement" Import-Module "$PSScriptRoot\..\Common\CatalogAndDatabaseManagement" -Force -Import-Module "$PSScriptRoot\..\WtpConfig" -Force +Import-Module "$PSScriptRoot\..\wtpConfig" -Force $ErrorActionPreference = "Stop" @@ -72,7 +70,7 @@ function Get-CurvedSalesForDay else { $curvePercent = ($Curve.60 / 5) } # add some random variation - [decimal] $variance = (-15, -10, -8, -5, -4, 0, 5, 10) | Get-Random + [decimal] $variance = (-10, -8, -5, -4, 0, 5, 10) | Get-Random $curvePercent = $curvePercent + ($curvePercent * $variance/100) if ($curvePercent -lt 0) {$curvePercent = 0} @@ -93,16 +91,7 @@ Initialize-Subscription $startTime = Get-Date $AdminUserName = $config.TenantAdminUsername -$AdminPassword = $config.TenantAdminPassword - -$ServerName = $config.TenantServerNameStem + $WtpUser.ToLower() - -<# uncomment to generate tickets for the golden databases -$WtpResourceGroupName = "wingtip-gold" -$ServerName = "wingtip-customers-gold" -#> - -$FullyQualifiedServerName = $ServerName + ".database.windows.net" +$AdminPassword = $config.TenantAdminPassword # load fictitious customer names, postal codes, $fictitiousNames = Import-Csv -Path ("$PSScriptRoot\FictitiousNames.csv") -Header ("Id","FirstName","LastName","Language","Gender") @@ -119,7 +108,7 @@ foreach ($importCurve in $importCurves) } # create different sets of curves that reflect different venue/event popularities -$popularCurves = $curves.MadRush,$curves.Rush,$curves.SShapedHigh,$curves.FastBurn, $curves.StraightLine, $curves.LastMinuteRush +$popularCurves = $curves.MadRush,$curves.Rush,$curves.SShapedHigh,$curves.FastBurn, $curves.StraightLine, $curves.LastMinuteRush,$curves.MediumBurn $moderateCurves = $Curves.Rush,$Curves.SShapedMedium, $Curves.MediumBurn, $Curves.LastMinute $unpopularCurves = $curves.SShapedLow, $curves.QuickFizzle, $curves.SlowBurn,$curves.LastGasp, $curves.Disappointing @@ -179,6 +168,7 @@ foreach ($venue in $venues) $venueTickets = 0 $venueDatabaseName = $venue.Location.Database + $venueServer = $venue.Location.Server # set the venue popularity, which determines the sales curves used: 1=popular, 2=moderate, 3=unpopular @@ -200,12 +190,12 @@ foreach ($venue in $venues) "unpopular" {$venueCurves = $unpopularCurves} } - Write-Output "Purchasing tickets for $venueDatabaseName ($popularity)" + Write-Output "Purchasing tickets for $venueDatabaseName ($popularity) on server '$($venueServer.Split(".", 2)[0]).'" # add customers to the venue $results = Invoke-SqlAzureWithRetry ` -Username "$AdminUserName" -Password "$AdminPassword" ` - -ServerInstance $venue.Location.Server ` + -ServerInstance $venueServer ` -Database $venueDatabaseName ` -Query $customersSql @@ -228,8 +218,8 @@ foreach ($venue in $venues) $command = "SELECT SUM(SeatRows * SeatsPerRow) AS Capacity FROM Sections" $capacity = Invoke-SqlAzureWithRetry ` -Username "$AdminUserName" -Password "$AdminPassword" ` - -ServerInstance $venue.Location.Server ` - -Database $venue.Location.Database ` + -ServerInstance $venueServer ` + -Database $venueDatabaseName ` -Query $command # get events for this venue @@ -239,8 +229,8 @@ foreach ($venue in $venues) $events = Invoke-SqlAzureWithRetry ` -Username "$AdminUserName" -Password "$AdminPassword" ` - -ServerInstance $venue.Location.Server ` - -Database $venue.Location.Database ` + -ServerInstance $venueServer ` + -Database $venueDatabaseName ` -Query $command $eventCount = 1 @@ -277,8 +267,8 @@ foreach ($venue in $venues) $sections = @() $sections += Invoke-SqlAzureWithRetry ` -Username "$AdminUserName" -Password "$AdminPassword" ` - -ServerInstance $venue.Location.Server ` - -Database $venue.Location.Database ` + -ServerInstance $venueServer ` + -Database $venueDatabaseName ` -Query $command # process sections to create collections of seats from which purchased tickets will be drawn @@ -313,7 +303,7 @@ foreach ($venue in $venues) $ticketStart = $event.Date.AddDays(-60) $today = Get-Date - + # loop over 60 day sales period for($day = 1; $day -le 60 ; $day++) { @@ -338,7 +328,7 @@ foreach ($venue in $venues) if ($ticketsToPurchase -eq 0) { continue - } + } $ticketsPurchased = 0 while ($ticketsPurchased -lt $ticketsToPurchase -and $seating.Count -gt 0 ) @@ -348,12 +338,12 @@ foreach ($venue in $venues) # pick a random customer Id $customerId = Get-Random -Minimum 1 -Maximum $customerCount - # pick number of tickets to purchase (2-10 per person) + # pick number of tickets for this customer to purchase (2-10 per person) $ticketOrder = Get-Random -Minimum 2 -Maximum 10 # ensure ticket order does not cause purchases to exceed tickets to buy for this day $remainingTicketsToBuyThisDay = $ticketsToPurchase - $ticketsPurchased - if ($Ticketorder -gt $remainingTicketsToBuyThisDay) + if ($ticketorder -gt $remainingTicketsToBuyThisDay) { $ticketOrder = $remainingTicketsToBuyThisDay } @@ -380,10 +370,6 @@ foreach ($venue in $venues) $purchasedSeatKey = $preferredSectionSeating.Keys| Sort | Select-Object -First 1 $purchasedSeat = $preferredSectionSeating.$purchasedSeatKey - # set time of day of purchase - distributed randomly over prior 24 hours - $mins = Get-Random -Maximum 1440 -Minimum 0 - $purchaseDate = $purchaseDate.AddMinutes(-$mins) - $PurchaseTotal += $purchasedSeat.Price $ticketsPurchased ++ @@ -410,6 +396,11 @@ foreach ($venue in $venues) $seating.Remove($preferredSectionSeatingKey) } } + + # set time of day of purchase - distributed randomly over prior 24 hours + $mins = Get-Random -Maximum 1440 -Minimum 0 + $purchaseTime = $purchaseDate.AddMinutes(-$mins) + # add ticket purchase to batch if($tpBatch -ge 1000) { @@ -418,33 +409,31 @@ foreach ($venue in $venues) $ticketPurchaseSql += "INSERT INTO [dbo].[TicketPurchases] ([TicketPurchaseId],[CustomerId],[PurchaseDate],[PurchaseTotal]) VALUES`n" $tpBatch = 0 } - - $ticketPurchaseSql += "($ticketPurchaseId,$CustomerId,'$purchaseDate',$PurchaseTotal),`n" + + $ticketPurchaseSql += "($ticketPurchaseId,$CustomerId,'$purchaseTime',$PurchaseTotal),`n" $tpBatch ++ $seatingAssigned = $true - $ticketPurchaseId ++ - } + $ticketPurchaseId ++ + + } # tickets one customer $totalTicketPurchases ++ $totalTickets += $ticketOrder $eventTickets += $ticketOrder $venueTickets += $ticketOrder - - # per customer purchases - } - - # daily purchases - } + + } # all customer orders (ticket purchases) for one day + + } # purchases for all 60 days Write-Output " $eventTickets tickets purchased" $eventCount ++ - - # per event purchases - } + + } # per event purchases - Write-Output " $venueTickets tickets purchased for $($venue.Location.Database)" + Write-Output " $venueTickets tickets purchased for $venueDatabaseName" # Finalize batched SQL commands for this venue and execute @@ -456,8 +445,8 @@ foreach ($venue in $venues) $ticketPurchasesExec = Invoke-SqlAzureWithRetry ` -Username "$AdminUserName" ` -Password "$AdminPassword" ` - -ServerInstance $venue.Location.Server ` - -Database $venue.Location.Database ` + -ServerInstance $venueServer ` + -Database $venueDatabaseName ` -Query $ticketPurchaseSql ` -QueryTimeout 120 @@ -468,8 +457,8 @@ foreach ($venue in $venues) $ticketsExec = Invoke-SqlAzureWithRetry ` -Username "$AdminUserName" ` -Password "$AdminPassword" ` - -ServerInstance $venue.Location.Server ` - -Database $venue.Location.Database ` + -ServerInstance $venueServer ` + -Database $venueDatabaseName ` -Query $ticketSql ` -QueryTimeout 120 diff --git a/Learning Modules/WtpConfig.psm1 b/Learning Modules/WtpConfig.psm1 index 448eb84..19eab56 100644 --- a/Learning Modules/WtpConfig.psm1 +++ b/Learning Modules/WtpConfig.psm1 @@ -8,7 +8,7 @@ function Get-Configuration { $configuration = @{` - TemplatesLocationUrl = "https://wtpdeploystorageaccount.blob.core.windows.net/templates" + TemplatesLocationUrl = "https://wingtipsaas.blob.core.windows.net/templates" TenantDatabaseTemplate = "tenantdatabasetemplate.json" TenantDatabaseCopyTemplate = "tenantdatabasecopytemplate.json" TenantDatabaseBatchTemplate = "tenantdatabasebatchtemplate.json" @@ -18,7 +18,7 @@ function Get-Configuration LogAnalyticsWorkspaceNameStem = "wtploganalytics-" LogAnalyticsDeploymentLocation = "westcentralus" DatabaseAndBacpacTemplate = "databaseandbacpactemplate.json" - TenantBacpacUrl = "https://wtpdeploystorageaccount.blob.core.windows.net/wingtip-bacpacsvold/wingtiptenantdb.bacpac" + TenantBacpacUrl = "https://wingtipsaas.blob.core.windows.net/bacpacs/wingtiptenantdb.bacpac" GoldenTenantDatabaseName = "baseTenantDB" CatalogDatabaseName = "tenantcatalog" CatalogServerNameStem = "catalog-" @@ -33,36 +33,37 @@ function Get-Configuration CatalogManagementAppSku = "standard" CatalogManagementAppSkuCode = "S1" CatalogManagementAppWorkerSize = 0 - CatalogSyncWebJobNameStem = "catalogsync-" - AutoProvisionWebJobNameStem = "autoprovision-" ServicePrincipalPassword = "P@ssword1" JobAccount = "jobaccount" JobAccountDatabaseName = "jobaccount" + JobAccountCredentialName = "mydemocred" TenantAnalyticsDatabaseName = "tenantanalytics" AdhocAnalyticsDatabaseName = "adhocanalytics" AdhocAnalyticsDatabaseServiceObjective = "S0" AdhocAnalyticsBacpacUrl = "https://wtpdeploystorageaccount.blob.core.windows.net/wingtip-bacpacsvold/adhoctenantanalytics.bacpac" + SearchDatabaseName = "tenantsearch" + SearchDatabaseServiceObjective = "S0" StorageKeyType = "SharedAccessKey" StorageAccessKey = (ConvertTo-SecureString -String "?" -AsPlainText -Force) DefaultVenueType = "multipurpose" TenantNameBatch = @( - ("Poplar Dance Academy","dance"), - ("Blue Oak Jazz Club","blues"), - ("Juniper Jammers Jazz","jazz"), - ("Sycamore Symphony","classicalmusic"), - ("Hornbeam HipHop","dance"), - ("Mahogany Soccer","soccer"), - ("Lime Tree Track","motorracing"), - ("Balsam Blues Club","blues"), - ("Tamarind Studio","dance"), - ("Star Anise Judo", "judo"), - ("Cottonwood Concert Hall","classicalmusic"), - ("Mangrove Soccer Club","soccer"), - ("Foxtail Rock","rockmusic"), - ("Osage Opera","opera"), - ("Papaya Players","soccer"), - ("Magnolia Motor Racing","motorracing"), - ("Sorrel Soccer","soccer") + ("Poplar Dance Academy","dance","98402"), + ("Blue Oak Jazz Club","blues","98201"), + ("Juniper Jammers Jazz","jazz","98032"), + ("Sycamore Symphony","classicalmusic","98004"), + ("Hornbeam HipHop","dance","98036"), + ("Mahogany Soccer","soccer","98032"), + ("Lime Tree Track","motorracing","98115"), + ("Balsam Blues Club","blues","98104"), + ("Tamarind Studio","dance","98072"), + ("Star Anise Judo", "judo","98103"), + ("Cottonwood Concert Hall","classicalmusic","98402"), + ("Mangrove Soccer Club","soccer","98036"), + ("Foxtail Rock","rockmusic","98107"), + ("Osage Opera","opera","98101"), + ("Papaya Players","soccer","98116"), + ("Magnolia Motor Racing","motorracing","98040"), + ("Sorrel Soccer","soccer","98188") ) } return $configuration diff --git a/README.md b/README.md index db93d56..1f3619f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,17 @@ ## WingtipSaaS -Wingtip Tickets "SaaS in a Box" sample application, tutorials, and documentation +Wingtip Tickets Platform sample SaaS application and management scripts. -This project provides a sample SaaS application that embodies many common SaaS patterns that can be used with Azure SQL Database. The sample is based on an event-management and ticket-selling scenario for small venues. Each venue is a 'tenant' of the SaaS application. The sample uses a single-tenant or tenant-per-database model, with a database created for each venue. These databases are hosted in elastic database pools to provide easy performance management, and to cost-effectively accommodate the unpredictable usage patterns of these small venues and their customers. An additional catalog database holds the mapping between tenants and their databases. This mapping is managed using the Shard Map Management features of the Elastic Scale Client Library. +This project provides a sample SaaS application that embodies many common SaaS patterns that can be used with Azure SQL Database. The sample is based on an event-management and ticket-selling scenario for small venues. Each venue is a 'tenant' of the SaaS application. The sample uses a database-per-tenant model, with a database created for each venue. These databases are hosted in elastic database pools to provide easy performance management, and to cost-effectively accommodate the unpredictable usage patterns of these small venues and their customers. An additional catalog database holds the mapping between tenants and their databases. This mapping is managed using the Shard Map Management features of the Elastic Scale Client Library. The basic application, which includes three pre-defined databases for three venues, can be installed in your Azure subscription under a single ARM resource group. To uninstall the application, delete the resource group from the Azure Portal. -NOTE: if you install the application you will be charged for the Azure resources created. Actual costs incurred are based on your subscription offer type but are nominal if the application is not scaled up unreasonably and is deleted promptly after you have finished exploring each tutorial. +NOTE: if you install the application you will be charged for the Azure resources created. Actual costs incurred are based on your subscription offer type but are nominal if the application is not scaled up unreasonably and is deleted promptly after you have finished exploring the tutorials. -The following four key scenarios are explored in the sample: -- A venue admin registering a venue, which provisions a new tenant database and registers it in the catalog -- A venue admin configuring their venue, specifying the type of events it hosts, seating, upcoming events and ticket prices -- An end customer of a venue browsing events and purchasing tickets -- A devops at the SaaS vendor managing the fleet of tenant databases +More information about the sample app and the associated tutorials can be found here: [https://aka.ms/sqldbsaastutorial](https://aka.ms/sqldbsaastutorial) -In addition there is support for demonstrating the app, including a load generator script that simulates the kinds of unpredictable workloads that occur in this kind of SaaS scenario and whch can be used to explore the performance characteristics of databases deployed into elastic database pools, and try out various performance management scenarios. - -A range of tutorials will be developed and added to the sample over time that explore additional SaaS patterns. + + + ## License Microsoft Wingtip SaaS sample application and tutorials are licensed under the MIT license. See the [LICENSE](https://github.com/Microsoft/WingtipSaaS/blob/master/license) file for more details. diff --git a/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/WingtipTenantDB.sqlproj b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/WingtipTenantDB.sqlproj index f8eecf3..60c03fc 100644 --- a/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/WingtipTenantDB.sqlproj +++ b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/WingtipTenantDB.sqlproj @@ -83,6 +83,7 @@ + diff --git a/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/StoredProcedures/sp_DeleteEvent.sql b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/StoredProcedures/sp_DeleteEvent.sql new file mode 100644 index 0000000..7384d60 --- /dev/null +++ b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/StoredProcedures/sp_DeleteEvent.sql @@ -0,0 +1,22 @@ +CREATE PROCEDURE [dbo].[sp_DeleteEvent] + @EventId int +AS +BEGIN + SET NOCOUNT ON; + + DECLARE @Tickets int = (SELECT Count(*) FROM dbo.Tickets WHERE EventId = @EventId) + + IF @Tickets > 0 + BEGIN + RAISERROR ('Error. Cannot delete events for which tickets have been purchased.', 11, 1) + RETURN 1 + END + + DELETE FROM dbo.[EventSections] + WHERE EventId = @EventId + + DELETE FROM dbo.[Events] + WHERE EventId = @EventId + + RETURN 0 +END \ No newline at end of file diff --git a/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/TicketFacts.sql b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/TicketFacts.sql new file mode 100644 index 0000000..ea77b51 --- /dev/null +++ b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/TicketFacts.sql @@ -0,0 +1,16 @@ + CREATE VIEW [dbo].[TicketFacts] AS + SELECT v.VenueId, v.VenueName, v.VenueType,v.VenuePostalCode, v.VenueCapacity, + tp.TicketPurchaseId, tp.PurchaseDate, tp.PurchaseTotal, + t.RowNumber, t.SeatNumber, + c.CustomerId, c.PostalCode AS CustomerPostalCode, c.CountryCode, + e.EventId, e.EventName, e.Subtitle AS EventSubtitle, e.Date AS EventDate + FROM ( + SELECT (SELECT TOP 1 VenueId FROM [dbo].[Venues]) AS VenueId, + VenueName, VenueType, PostalCode AS VenuePostalCode, + (SELECT SUM ([SeatRows]*[SeatsPerRow]) FROM [dbo].[Sections]) AS VenueCapacity, + 1 AS X FROM Venue + ) as v + INNER JOIN [dbo].[TicketPurchases] AS tp ON v.X = 1 + INNER JOIN [dbo].[Tickets] AS t ON t.TicketPurchaseId = tp.TicketPurchaseId + INNER JOIN [dbo].[Events] AS e ON t.EventId = e.EventId + INNER JOIN [dbo].[Customers] AS c ON tp.CustomerId = c.CustomerId \ No newline at end of file diff --git a/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenueEvents.sql b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenueEvents.sql new file mode 100644 index 0000000..2d33898 --- /dev/null +++ b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenueEvents.sql @@ -0,0 +1,2 @@ + CREATE VIEW [dbo].[VenueEvents] AS + SELECT (SELECT TOP 1 VenueId FROM Venues) AS VenueId, EventId, EventName, Subtitle, Date FROM [events] \ No newline at end of file diff --git a/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenueTicketPurchases.sql b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenueTicketPurchases.sql new file mode 100644 index 0000000..6c3663b --- /dev/null +++ b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenueTicketPurchases.sql @@ -0,0 +1,2 @@ + CREATE VIEW [dbo].[VenueTicketPurchases] AS + SELECT (SELECT TOP 1 VenueId FROM Venues) AS VenueId, TicketPurchaseId, PurchaseDate, PurchaseTotal, CustomerId FROM [TicketPurchases] \ No newline at end of file diff --git a/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenueTickets.sql b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenueTickets.sql new file mode 100644 index 0000000..12351d1 --- /dev/null +++ b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenueTickets.sql @@ -0,0 +1,2 @@ + CREATE VIEW [dbo].[VenueTickets] AS + SELECT (SELECT TOP 1 VenueId FROM Venues) AS VenueId, TicketId, RowNumber, SeatNumber, EventId, SectionId, TicketPurchaseId FROM [Tickets] \ No newline at end of file diff --git a/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenuesGlobal.sql b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenuesGlobal.sql new file mode 100644 index 0000000..3e466aa --- /dev/null +++ b/WTP Application Project Files/WTP Application Project Files/WingtipSaaSDatabase/WingtipTenantDB/dbo/Views/VenuesGlobal.sql @@ -0,0 +1,2 @@ + CREATE VIEW [dbo].[Venues] AS + SELECT Convert(int, HASHBYTES('md5',VenueName)) AS VenueId, VenueName, VenueType, AdminEmail, PostalCode, CountryCode, @@ServerName as Server, DB_NAME() AS [DatabaseName] FROM [Venue] \ No newline at end of file