From e1fb0a971f594394d7fa4980333c0eab1d5d48f4 Mon Sep 17 00:00:00 2001 From: dayland <48474707+dayland@users.noreply.github.com> Date: Wed, 29 May 2024 16:00:15 +0100 Subject: [PATCH 01/11] chore: Update tiktoken dependency to version 0.7.0 to support gpt-4o --- app/backend/core/modelhelper.py | 6 ++++-- app/backend/requirements.txt | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/backend/core/modelhelper.py b/app/backend/core/modelhelper.py index 970d5ed8a..758d3c7be 100644 --- a/app/backend/core/modelhelper.py +++ b/app/backend/core/modelhelper.py @@ -8,12 +8,14 @@ "gpt-35-turbo-16k": 16385, "gpt-3.5-turbo-16k": 16385, "gpt-4": 8192, - "gpt-4-32k": 32768 + "gpt-4-32k": 32768, + "gpt-4o": 128000 } AOAI_2_OAI = { "gpt-35-turbo": "gpt-3.5-turbo", - "gpt-35-turbo-16k": "gpt-3.5-turbo-16k" + "gpt-35-turbo-16k": "gpt-3.5-turbo-16k", + "gpt-4o": "gpt-4o" } diff --git a/app/backend/requirements.txt b/app/backend/requirements.txt index 4cf360db7..a73e97aba 100644 --- a/app/backend/requirements.txt +++ b/app/backend/requirements.txt @@ -8,7 +8,7 @@ openai==1.17.0 azure-search-documents==11.4.0b11 azure-storage-blob==12.16.0 azure-cosmos == 4.3.1 -tiktoken == 0.5.2 +tiktoken == 0.7.0 fastapi == 0.109.1 fastapi-utils == 0.2.1 uvicorn == 0.23.2 From 3038069da65c39e94072bf6be8932c7c26793980 Mon Sep 17 00:00:00 2001 From: Ashvith <49021230+asbanger@users.noreply.github.com> Date: Wed, 29 May 2024 23:31:01 +0800 Subject: [PATCH 02/11] Create manual_app_registration.md --- docs/deployment/manual_app_registration.md | 71 ++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 docs/deployment/manual_app_registration.md diff --git a/docs/deployment/manual_app_registration.md b/docs/deployment/manual_app_registration.md new file mode 100644 index 000000000..b440169e8 --- /dev/null +++ b/docs/deployment/manual_app_registration.md @@ -0,0 +1,71 @@ +# App Registration Creation Guide + +If you are unable to obtain the permission at the tenant level described in [Azure account requirements](https://github.com/microsoft/PubSec-Info-Assistant/tree/v1.0?tab=readme-ov-file#azure-account-requirements), then this guide will provide a manual workaround for you. + +## Tenant Administrator: Manual Steps to create App Registrations + +Here are the details of each step: + +### Create a Random Sequence File +Manually create a random string for your environment. This ensures unique service names to avoid DNS conflicts. + +1. In VSCode, open the `/infra` folder. +2. Create a new subfolder named `.state`. +3. Create a new subfolder under the `.state` folder. This folder MUST have the same value as the **WORKSPACE** parameter in the `local.env` file. +4. In that new folder, create a text file named `random.txt`. +5. Edit the TXT file and enter a 1 to 5 character sequence. The value must be a combination of letters a-z, A-Z, or numbers 0-9. + +### Have Administrator Create Two AD App Registrations and Enterprise Applications +An Administrator in the tenant would need to create two Azure AD App Registrations and Service Principals for you manually. + +#### First AD App Registration: Securing the Information Assistant Web Application +The first AD App Registration will be used to secure the Information Assistant web application and will need to ensure the following settings: + +**Azure AD App Registration** +| Setting | Value | +|---|---| +| name | `infoasst_web_access_<>` | +| sign-in-audience | AzureADMyOrg | +| identifier-uris | `api://infoasst-<>` | +| web-redirect-uris | `https://infoasst-web-<>.azurewebsites.net/.auth/login/aad/callback` | +| enable-access-token-issuance | true | +| enable-id-token-issuance | true | + +**Azure AD Enterprise Application (optional)** +If you desire to have the Information Assistant website secured by explicit membership, then the following settings will need to be updated: + +| Setting | Value | +|---|---| +| name | `infoasst_web_access_<>` | +| appRoleAssignmentRequired | true | + +#### Second AD App Registration: Querying Azure Management Plane APIs +The second AD App Registration will be used to query the Azure management plane APIs for Azure service details. It needs the following settings: + +**Azure AD App Registration** +| Setting | Value | +|---|---| +| name | `infoasst_mgmt_access_<>` | +| sign-in-audience | AzureADMyOrg | + +### Information to Obtain from Tenant Administrator +You will need to obtain the following information from your tenant Administrator to continue: +- Web Access App Registration Client ID (guid) +- Web Access Service Principal ID (guid) +- Management Access App Registration Client ID (guid) +- Management Access App Registration Client Secret +- Management Access Service Principal ID (guid) + +### Adjust Code in Infrastructure Deployment +In the file `scripts/inf-create.sh`, between lines 97 - 138, you would need to replace the code with the following sample. Update parameters with values provided by your tenant Administrator. + +```bash +signedInUserId=$(az ad signed-in-user show --query id --output tsv) +kvAccessObjectId=$signedInUserId +aadWebAppId= +aadWebSPId= +aadMgmtAppId= +aadMgmtAppSecret= +aadMgmtSPId= + +# Rest of your script continues... From 81783f11eba26b71240c533126d47e5ea71bf442 Mon Sep 17 00:00:00 2001 From: Ashvith <49021230+asbanger@users.noreply.github.com> Date: Wed, 29 May 2024 23:32:45 +0800 Subject: [PATCH 03/11] Update inf-create.sh --- scripts/inf-create.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/inf-create.sh b/scripts/inf-create.sh index 8dbdbe121..4e358a4a8 100755 --- a/scripts/inf-create.sh +++ b/scripts/inf-create.sh @@ -60,6 +60,14 @@ then az account set -s "$ARM_SUBSCRIPTION_ID" fi +#If you are unable to obtain the permission at the tenant level described in Azure account requirements, you can set the following to true provided you have created Azure AD App Registrations. + +#export TF_VAR_isInAutomation=true +#export TF_VAR_aadWebClientId="" +#export TF_VAR_aadMgmtClientId="" +#export TF_VAR_aadMgmtServicePrincipalId="" +#export TF_VAR_aadMgmtClientSecret="" + # prepare vars for the users you wish to assign to the security group object_ids=() From f84e11da06252e21a1d421d2e2e8d5b0f2be1729 Mon Sep 17 00:00:00 2001 From: Ashvith <49021230+asbanger@users.noreply.github.com> Date: Thu, 30 May 2024 15:02:53 +0100 Subject: [PATCH 04/11] update documentation --- docs/deployment/manual_app_registration.md | 71 ++++++++++++++-------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/docs/deployment/manual_app_registration.md b/docs/deployment/manual_app_registration.md index b440169e8..7f9af6142 100644 --- a/docs/deployment/manual_app_registration.md +++ b/docs/deployment/manual_app_registration.md @@ -1,27 +1,25 @@ # App Registration Creation Guide -If you are unable to obtain the permission at the tenant level described in [Azure account requirements](https://github.com/microsoft/PubSec-Info-Assistant/tree/v1.0?tab=readme-ov-file#azure-account-requirements), then this guide will provide a manual workaround for you. +If you are unable to obtain the permission at the tenant level described in [Azure account requirements](https://github.com/microsoft/PubSec-Info-Assistant/tree/v1.0?tab=readme-ov-file#azure-account-requirements), you can follow the guidance below to create a manual app registration. ## Tenant Administrator: Manual Steps to create App Registrations Here are the details of each step: -### Create a Random Sequence File -Manually create a random string for your environment. This ensures unique service names to avoid DNS conflicts. +### 1. Create a Random Sequence File -1. In VSCode, open the `/infra` folder. -2. Create a new subfolder named `.state`. -3. Create a new subfolder under the `.state` folder. This folder MUST have the same value as the **WORKSPACE** parameter in the `local.env` file. -4. In that new folder, create a text file named `random.txt`. -5. Edit the TXT file and enter a 1 to 5 character sequence. The value must be a combination of letters a-z, A-Z, or numbers 0-9. +Manually create a random string for your environment, which should be a 1 to 5 character sequence. The value must be a combination of letters a-z, A-Z, or numbers 0-9. + +### 2. Have Tenant Administrator Create Two AD App Registrations -### Have Administrator Create Two AD App Registrations and Enterprise Applications An Administrator in the tenant would need to create two Azure AD App Registrations and Service Principals for you manually. #### First AD App Registration: Securing the Information Assistant Web Application + The first AD App Registration will be used to secure the Information Assistant web application and will need to ensure the following settings: -**Azure AD App Registration** +##### Azure AD App Registration** + | Setting | Value | |---|---| | name | `infoasst_web_access_<>` | @@ -31,7 +29,8 @@ The first AD App Registration will be used to secure the Information Assistant w | enable-access-token-issuance | true | | enable-id-token-issuance | true | -**Azure AD Enterprise Application (optional)** +#### Azure AD Enterprise Application (optional) + If you desire to have the Information Assistant website secured by explicit membership, then the following settings will need to be updated: | Setting | Value | @@ -40,32 +39,54 @@ If you desire to have the Information Assistant website secured by explicit memb | appRoleAssignmentRequired | true | #### Second AD App Registration: Querying Azure Management Plane APIs + The second AD App Registration will be used to query the Azure management plane APIs for Azure service details. It needs the following settings: -**Azure AD App Registration** +##### Azure AD App Registration + | Setting | Value | |---|---| | name | `infoasst_mgmt_access_<>` | | sign-in-audience | AzureADMyOrg | -### Information to Obtain from Tenant Administrator +### 3. Information to Obtain from Tenant Administrator + You will need to obtain the following information from your tenant Administrator to continue: + - Web Access App Registration Client ID (guid) - Web Access Service Principal ID (guid) - Management Access App Registration Client ID (guid) -- Management Access App Registration Client Secret - Management Access Service Principal ID (guid) +- Management Access App Registration Client Secret (string) + +These values will be used to update the code in the Infrastructure Deployment section. + +### 4. Adjust Code in Infrastructure Deployment + +In the file `scripts/inf-create.sh`, between lines 63 - 69, you would need to uncomment the code and update parameters with values provided by your tenant Administrator. + +export TF_VAR_isInAutomation=true +export TF_VAR_aadWebClientId="" +export TF_VAR_aadMgmtClientId="" +export TF_VAR_aadMgmtServicePrincipalId="" +export TF_VAR_aadMgmtClientSecret="" + +- Management Access App Registration Client Secret + +### 4.Adjust Code in Infrastructure Deployment + +In the file `scripts/inf-create.sh`, between lines 63 - 69, you would need to uncomment the code and update parameters with values provided by your tenant Administrator. + +export TF_VAR_isInAutomation=true +export TF_VAR_aadWebClientId="" +export TF_VAR_aadMgmtClientId="" +export TF_VAR_aadMgmtServicePrincipalId="" +export TF_VAR_aadMgmtClientSecret="" + +### 5. Resume the deployment as per the deployment procedure -### Adjust Code in Infrastructure Deployment -In the file `scripts/inf-create.sh`, between lines 97 - 138, you would need to replace the code with the following sample. Update parameters with values provided by your tenant Administrator. +After completing the step4, you can resume back the deployment steps mentioned the documentation -```bash -signedInUserId=$(az ad signed-in-user show --query id --output tsv) -kvAccessObjectId=$signedInUserId -aadWebAppId= -aadWebSPId= -aadMgmtAppId= -aadMgmtAppSecret= -aadMgmtSPId= +### 6: Update the AD App Registration -# Rest of your script continues... +Once Terraform completes the deployment of the infrastructure, update the `identifier-uris` and `web-redirect-uris` with the newly generated random_string created during the Terraform deployment. From d7b41c555240be25031bb8d1af30ac5f559d5699 Mon Sep 17 00:00:00 2001 From: Ashvith <49021230+asbanger@users.noreply.github.com> Date: Thu, 30 May 2024 16:06:37 +0100 Subject: [PATCH 05/11] updating the documentation --- docs/deployment/manual_app_registration.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/deployment/manual_app_registration.md b/docs/deployment/manual_app_registration.md index 7f9af6142..13a3adfcb 100644 --- a/docs/deployment/manual_app_registration.md +++ b/docs/deployment/manual_app_registration.md @@ -65,23 +65,13 @@ These values will be used to update the code in the Infrastructure Deployment se In the file `scripts/inf-create.sh`, between lines 63 - 69, you would need to uncomment the code and update parameters with values provided by your tenant Administrator. +``` export TF_VAR_isInAutomation=true export TF_VAR_aadWebClientId="" export TF_VAR_aadMgmtClientId="" export TF_VAR_aadMgmtServicePrincipalId="" export TF_VAR_aadMgmtClientSecret="" - -- Management Access App Registration Client Secret - -### 4.Adjust Code in Infrastructure Deployment - -In the file `scripts/inf-create.sh`, between lines 63 - 69, you would need to uncomment the code and update parameters with values provided by your tenant Administrator. - -export TF_VAR_isInAutomation=true -export TF_VAR_aadWebClientId="" -export TF_VAR_aadMgmtClientId="" -export TF_VAR_aadMgmtServicePrincipalId="" -export TF_VAR_aadMgmtClientSecret="" +``` ### 5. Resume the deployment as per the deployment procedure From 2a5a0ed5b706fae0df992caa1e2cf6b4c7b4d364 Mon Sep 17 00:00:00 2001 From: Ashvith <49021230+asbanger@users.noreply.github.com> Date: Thu, 30 May 2024 16:17:02 +0100 Subject: [PATCH 06/11] update link back to document --- docs/deployment/deployment.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/deployment/deployment.md b/docs/deployment/deployment.md index 973a6483c..1503fa12b 100644 --- a/docs/deployment/deployment.md +++ b/docs/deployment/deployment.md @@ -166,6 +166,10 @@ run-data-migration Run the data migration moving data from one resourc manual-inf-destroy A command triggered by a user to destroy a resource group, associated resources, and related Entra items ``` +## Configure AD app registration ( manual steps ) + +If you have insufficient permissions at the tenant level (Application Administrator Entry Role), follow the guide to complete the deployment [manual app registration](/docs/deployment/manual_app_registration.md). + ## Configure authentication and authorization If you have chosen to enable authentication and authorization for your deployment by setting the environment variable `REQUIRE_WEBSITE_SECURITY_MEMBERSHIP` to `true`, you will need to configure it at this point. Please see [Known Issues](/docs/knownissues.md#error-your-adminstrator-has-configured-the-application-infoasst_web_access_xxxxx-to-block-users) section for guidance on how to configure. From e5279bf1eb7886130f4da44f7df4578a9d0a876e Mon Sep 17 00:00:00 2001 From: Ashvith <49021230+asbanger@users.noreply.github.com> Date: Thu, 30 May 2024 16:28:12 +0100 Subject: [PATCH 07/11] update the documenation --- docs/deployment/manual_app_registration.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/deployment/manual_app_registration.md b/docs/deployment/manual_app_registration.md index 13a3adfcb..0355a38c8 100644 --- a/docs/deployment/manual_app_registration.md +++ b/docs/deployment/manual_app_registration.md @@ -2,7 +2,7 @@ If you are unable to obtain the permission at the tenant level described in [Azure account requirements](https://github.com/microsoft/PubSec-Info-Assistant/tree/v1.0?tab=readme-ov-file#azure-account-requirements), you can follow the guidance below to create a manual app registration. -## Tenant Administrator: Manual Steps to create App Registrations +## Tenant Administrator: Manual steps to create app registrations Here are the details of each step: @@ -10,7 +10,7 @@ Here are the details of each step: Manually create a random string for your environment, which should be a 1 to 5 character sequence. The value must be a combination of letters a-z, A-Z, or numbers 0-9. -### 2. Have Tenant Administrator Create Two AD App Registrations +### 2. Have the Tenant Administrator Should Create Two AD App Registrations An Administrator in the tenant would need to create two Azure AD App Registrations and Service Principals for you manually. @@ -49,7 +49,7 @@ The second AD App Registration will be used to query the Azure management plane | name | `infoasst_mgmt_access_<>` | | sign-in-audience | AzureADMyOrg | -### 3. Information to Obtain from Tenant Administrator +### 3. Information to obtain from Tenant Administrator You will need to obtain the following information from your tenant Administrator to continue: @@ -59,9 +59,9 @@ You will need to obtain the following information from your tenant Administrator - Management Access Service Principal ID (guid) - Management Access App Registration Client Secret (string) -These values will be used to update the code in the Infrastructure Deployment section. +These values will be used to update the code in the infrastructure deployment section. -### 4. Adjust Code in Infrastructure Deployment +### 4. Adjust code in infrastructure deployment In the file `scripts/inf-create.sh`, between lines 63 - 69, you would need to uncomment the code and update parameters with values provided by your tenant Administrator. From 90d4f69f164b8865fef1916903f24a3468f3917d Mon Sep 17 00:00:00 2001 From: Ashvith <49021230+asbanger@users.noreply.github.com> Date: Mon, 3 Jun 2024 23:07:41 +0800 Subject: [PATCH 08/11] Update vite.config.ts This update will fix the vite build error --- app/frontend/vite.config.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/frontend/vite.config.ts b/app/frontend/vite.config.ts index b6703eba1..208229ec7 100644 --- a/app/frontend/vite.config.ts +++ b/app/frontend/vite.config.ts @@ -8,7 +8,10 @@ export default defineConfig({ build: { outDir: "../backend/static", emptyOutDir: true, - sourcemap: true + sourcemap: true, + rollupOptions: { + external: ['__vite-browser-external'] + } }, server: { proxy: { From 4569fbb58ff322d7b50c9dc3677d0151bc7da9d3 Mon Sep 17 00:00:00 2001 From: KronemeyerJoshua Date: Mon, 3 Jun 2024 11:20:44 -0500 Subject: [PATCH 09/11] Update storage privacy settings --- infra/core/storage/storage-account.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/core/storage/storage-account.tf b/infra/core/storage/storage-account.tf index 19b5bd81e..390ab6247 100644 --- a/infra/core/storage/storage-account.tf +++ b/infra/core/storage/storage-account.tf @@ -12,6 +12,7 @@ resource "azurerm_storage_account" "storage" { access_tier = var.accessTier min_tls_version = var.minimumTlsVersion enable_https_traffic_only = true + allow_nested_items_to_be_public = false network_rules { default_action = "Allow" From a72343897fc38ce919c8fa299caa2c8947e28cf8 Mon Sep 17 00:00:00 2001 From: Josh <4635851+KronemeyerJoshua@users.noreply.github.com> Date: Mon, 3 Jun 2024 15:42:13 -0500 Subject: [PATCH 10/11] whitespace change to trigger build --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8c75f27ad..eac64aaba 100644 --- a/Makefile +++ b/Makefile @@ -87,4 +87,4 @@ run-data-migration: ## Run the data migration moving data from one resource grou python ./scripts/extract-content.py manual-inf-destroy: ## A command triggered by a user to destroy a resource group, associated resources, and related Entra items - @./scripts/inf-manual-destroy.sh \ No newline at end of file + @./scripts/inf-manual-destroy.sh From 7e6cd90f71d2ec70a2af606eca33143eb08abf51 Mon Sep 17 00:00:00 2001 From: Ashvith <49021230+asbanger@users.noreply.github.com> Date: Thu, 6 Jun 2024 22:44:57 +0800 Subject: [PATCH 11/11] Update deployment.md --- docs/deployment/deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deployment/deployment.md b/docs/deployment/deployment.md index 1503fa12b..f03e821ca 100644 --- a/docs/deployment/deployment.md +++ b/docs/deployment/deployment.md @@ -168,7 +168,7 @@ manual-inf-destroy A command triggered by a user to destroy a resource ## Configure AD app registration ( manual steps ) -If you have insufficient permissions at the tenant level (Application Administrator Entry Role), follow the guide to complete the deployment [manual app registration](/docs/deployment/manual_app_registration.md). +If you have insufficient permissions at the tenant level (Application Administrator Entra Role), follow the guide to complete the deployment [manual app registration](/docs/deployment/manual_app_registration.md). ## Configure authentication and authorization