diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 989f7387..4a1ce599 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -18,5 +18,8 @@ jobs: run: | python -m pip install --upgrade pip pip install -r ClientAdvisor/App/requirements.txt - - name: Run flake8 - run: flake8 --config=ClientAdvisor/App/.flake8 ClientAdvisor/App + pip install -r ResearchAssistant/App/requirements.txt + - name: Run flake8 and pylint + run: | + flake8 --config=ClientAdvisor/App/.flake8 ClientAdvisor/App + flake8 --config=ResearchAssistant/App/.flake8 ResearchAssistant/App \ No newline at end of file diff --git a/ClientAdvisor/App/.flake8 b/ClientAdvisor/App/.flake8 index 74ee71d5..93f63e5d 100644 --- a/ClientAdvisor/App/.flake8 +++ b/ClientAdvisor/App/.flake8 @@ -1,4 +1,5 @@ [flake8] max-line-length = 88 -extend-ignore = E501, E203 -exclude = .venv, frontend, \ No newline at end of file +extend-ignore = E501 +exclude = .venv, frontend +ignore = E203, W503, G004, G200 \ No newline at end of file diff --git a/ClientAdvisor/App/requirements-dev.txt b/ClientAdvisor/App/requirements-dev.txt index 25950285..5d246093 100644 --- a/ClientAdvisor/App/requirements-dev.txt +++ b/ClientAdvisor/App/requirements-dev.txt @@ -15,6 +15,6 @@ httpx==0.28.0 flake8==7.1.1 black==24.8.0 autoflake==2.3.1 -isort==5.13.2pytest-asyncio==0.24.0 -pytest-cov==5.0.0 -isort==5.13.2 \ No newline at end of file +isort==5.13.2 +pytest-asyncio==0.24.0 +pytest-cov==5.0.0 \ No newline at end of file diff --git a/ClientAdvisor/App/requirements.txt b/ClientAdvisor/App/requirements.txt index e3bf3db1..20851cbe 100644 --- a/ClientAdvisor/App/requirements.txt +++ b/ClientAdvisor/App/requirements.txt @@ -1,3 +1,4 @@ +# Core requirements azure-identity==1.15.0 # Flask[async]==2.3.2 openai==1.55.3 @@ -12,9 +13,14 @@ aiohttp==3.10.2 quart-session==3.0.0 pymssql==2.3.0 httpx==0.28.0 + +# Linting and formatting tools flake8==7.1.1 black==24.8.0 autoflake==2.3.1 isort==5.13.2 + +# Testing tools +pytest>=8.2,<9 # Compatible version for pytest-asyncio pytest-asyncio==0.24.0 -pytest-cov==5.0.0 +pytest-cov==5.0.0 \ No newline at end of file diff --git a/ClientAdvisor/AzureFunction/function_app.py b/ClientAdvisor/AzureFunction/function_app.py index f9bfd8dc..16b34c23 100644 --- a/ClientAdvisor/AzureFunction/function_app.py +++ b/ClientAdvisor/AzureFunction/function_app.py @@ -79,28 +79,33 @@ def get_SQL_Response( ) deployment = os.environ.get("AZURE_OPEN_AI_DEPLOYMENT_MODEL") - sql_prompt = f'''A valid T-SQL query to find {query} for tables and columns provided below: - 1. Table: Clients - Columns: ClientId,Client,Email,Occupation,MaritalStatus,Dependents - 2. Table: InvestmentGoals - Columns: ClientId,InvestmentGoal - 3. Table: Assets - Columns: ClientId,AssetDate,Investment,ROI,Revenue,AssetType - 4. Table: ClientSummaries - Columns: ClientId,ClientSummary - 5. Table: InvestmentGoalsDetails - Columns: ClientId,InvestmentGoal,TargetAmount,Contribution - 6. Table: Retirement - Columns: ClientId,StatusDate,RetirementGoalProgress,EducationGoalProgress - 7.Table: ClientMeetings - Columns: ClientId,ConversationId,Title,StartTime,EndTime,Advisor,ClientEmail - Use Investement column from Assets table as value always. - Assets table has snapshots of values by date. Do not add numbers across different dates for total values. - Do not use client name in filter. - Do not include assets values unless asked for. - Always use ClientId = {clientid} in the query filter. - Always return client name in the query. - Only return the generated sql query. do not return anything else''' + sql_prompt = os.environ.get("AZURE_SQL_SYSTEM_PROMPT") + if sql_prompt: + sql_prompt = sql_prompt.replace("{query}", query) + sql_prompt = sql_prompt.replace("{clientid}", clientid) + else: + sql_prompt = f'''A valid T-SQL query to find {query} for tables and columns provided below: + 1. Table: Clients + Columns: ClientId,Client,Email,Occupation,MaritalStatus,Dependents + 2. Table: InvestmentGoals + Columns: ClientId,InvestmentGoal + 3. Table: Assets + Columns: ClientId,AssetDate,Investment,ROI,Revenue,AssetType + 4. Table: ClientSummaries + Columns: ClientId,ClientSummary + 5. Table: InvestmentGoalsDetails + Columns: ClientId,InvestmentGoal,TargetAmount,Contribution + 6. Table: Retirement + Columns: ClientId,StatusDate,RetirementGoalProgress,EducationGoalProgress + 7.Table: ClientMeetings + Columns: ClientId,ConversationId,Title,StartTime,EndTime,Advisor,ClientEmail + Use Investement column from Assets table as value always. + Assets table has snapshots of values by date. Do not add numbers across different dates for total values. + Do not use client name in filter. + Do not include assets values unless asked for. + Always use ClientId = {clientid} in the query filter. + Always return client name in the query. + Only return the generated sql query. do not return anything else''' try: completion = client.chat.completions.create( @@ -156,9 +161,11 @@ def get_answers_from_calltranscripts( ) query = question - system_message = '''You are an assistant who provides wealth advisors with helpful information to prepare for client meetings. - You have access to the client’s meeting call transcripts. - You can use this information to answer questions about the clients''' + system_message = os.environ.get("AZURE_CALL_TRANSCRIPT_SYSTEM_PROMPT") + if not system_message: + system_message = '''You are an assistant who provides wealth advisors with helpful information to prepare for client meetings. + You have access to the client’s meeting call transcripts. + You can use this information to answer questions about the clients''' completion = client.chat.completions.create( model = deployment, @@ -259,13 +266,15 @@ async def stream_openai_text(req: Request) -> StreamingResponse: settings.max_tokens = 800 settings.temperature = 0 - system_message = '''you are a helpful assistant to a wealth advisor. - Do not answer any questions not related to wealth advisors queries. - If the client name and client id do not match, only return - Please only ask questions about the selected client or select another client to inquire about their details. do not return any other information. - Only use the client name returned from database in the response. - If you cannot answer the question, always return - I cannot answer this question from the data available. Please rephrase or add more details. - ** Remove any client identifiers or ids or numbers or ClientId in the final response. - ''' + system_message = os.environ.get("AZURE_OPENAI_STREAM_TEXT_SYSTEM_PROMPT") + if not system_message: + system_message = '''you are a helpful assistant to a wealth advisor. + Do not answer any questions not related to wealth advisors queries. + If the client name and client id do not match, only return - Please only ask questions about the selected client or select another client to inquire about their details. do not return any other information. + Only use the client name returned from database in the response. + If you cannot answer the question, always return - I cannot answer this question from the data available. Please rephrase or add more details. + ** Remove any client identifiers or ids or numbers or ClientId in the final response. + ''' user_query = query.replace('?',' ') diff --git a/ClientAdvisor/Deployment/bicep/deploy_azure_function.bicep b/ClientAdvisor/Deployment/bicep/deploy_azure_function.bicep new file mode 100644 index 00000000..ba800153 --- /dev/null +++ b/ClientAdvisor/Deployment/bicep/deploy_azure_function.bicep @@ -0,0 +1,167 @@ +@description('Specifies the location for resources.') +param solutionName string +param solutionLocation string +@secure() +param azureOpenAIApiKey string +param azureOpenAIApiVersion string +param azureOpenAIEndpoint string +@secure() +param azureSearchAdminKey string +param azureSearchServiceEndpoint string +param azureSearchIndex string +param sqlServerName string +param sqlDbName string +param sqlDbUser string +@secure() +param sqlDbPwd string +param functionAppVersion string +@description('Azure Function App SQL System Prompt') +param sqlSystemPrompt string +@description('Azure Function App CallTranscript System Prompt') +param callTranscriptSystemPrompt string +@description('Azure Function App Stream Text System Prompt') +param streamTextSystemPrompt string + +var functionAppName = '${solutionName}fn' +var azureOpenAIDeploymentModel = 'gpt-4' +var azureOpenAIEmbeddingDeployment = 'text-embedding-ada-002' +var valueOne = '1' + +resource storageAccount 'Microsoft.Storage/storageAccounts@2021-04-01' = { + name: '${solutionName}fnstorage' + location: solutionLocation + sku: { + name: 'Standard_LRS' + } + kind: 'StorageV2' + properties: { + allowSharedKeyAccess: false + } +} + +resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { + name: 'workspace-${solutionName}' + location: solutionLocation +} + +resource ApplicationInsights 'Microsoft.Insights/components@2020-02-02' = { + name: functionAppName + location: solutionLocation + kind: 'web' + properties: { + Application_Type: 'web' + publicNetworkAccessForIngestion: 'Enabled' + publicNetworkAccessForQuery: 'Enabled' + WorkspaceResourceId: logAnalyticsWorkspace.id + } +} + +resource containerAppEnv 'Microsoft.App/managedEnvironments@2022-06-01-preview' = { + name: '${solutionName}env' + location: solutionLocation + sku: { + name: 'Consumption' + } + properties: { + appLogsConfiguration: { + destination: 'log-analytics' + logAnalyticsConfiguration: { + customerId: logAnalyticsWorkspace.properties.customerId + sharedKey: logAnalyticsWorkspace.listKeys().primarySharedKey + } + } + } +} + +resource functionApp 'Microsoft.Web/sites@2024-04-01' = { + name: functionAppName + location: solutionLocation + kind: 'functionapp' + identity: { + type: 'SystemAssigned' + } + properties: { + managedEnvironmentId: containerAppEnv.id + siteConfig: { + linuxFxVersion: 'DOCKER|bycwacontainerreg.azurecr.io/byc-wa-fn:${functionAppVersion}' + appSettings: [ + { + name: 'APPINSIGHTS_INSTRUMENTATIONKEY' + value: reference(ApplicationInsights.id, '2015-05-01').InstrumentationKey + } + { + name: 'AZURE_OPEN_AI_API_KEY' + value: azureOpenAIApiKey + } + { + name: 'AZURE_OPEN_AI_DEPLOYMENT_MODEL' + value: azureOpenAIDeploymentModel + } + { + name: 'AZURE_OPEN_AI_ENDPOINT' + value: azureOpenAIEndpoint + } + { + name: 'AZURE_OPENAI_EMBEDDING_DEPLOYMENT' + value: azureOpenAIEmbeddingDeployment + } + { + name: 'OPENAI_API_VERSION' + value: azureOpenAIApiVersion + } + { + name: 'AZURE_AI_SEARCH_API_KEY' + value: azureSearchAdminKey + } + { + name: 'AZURE_AI_SEARCH_ENDPOINT' + value: azureSearchServiceEndpoint + } + { + name: 'AZURE_SEARCH_INDEX' + value: azureSearchIndex + } + { + name: 'PYTHON_ENABLE_INIT_INDEXING' + value: valueOne + } + { + name: 'PYTHON_ISOLATE_WORKER_DEPENDENCIES' + value: valueOne + } + { + name: 'SQLDB_CONNECTION_STRING' + value: 'TBD' + } + { + name: 'SQLDB_SERVER' + value: sqlServerName + } + { + name: 'SQLDB_DATABASE' + value: sqlDbName + } + { + name: 'SQLDB_USERNAME' + value: sqlDbUser + } + { + name: 'SQLDB_PASSWORD' + value: sqlDbPwd + } + { + name: 'AZURE_SQL_SYSTEM_PROMPT' + value: sqlSystemPrompt + } + { + name: 'AZURE_CALL_TRANSCRIPT_SYSTEM_PROMPT' + value: callTranscriptSystemPrompt + } + { + name: 'AZURE_OPENAI_STREAM_TEXT_SYSTEM_PROMPT' + value: streamTextSystemPrompt + } + ] + } + } +} diff --git a/ClientAdvisor/Deployment/bicep/deploy_azure_function_script.bicep b/ClientAdvisor/Deployment/bicep/deploy_azure_function_script.bicep deleted file mode 100644 index 2ad7ff55..00000000 --- a/ClientAdvisor/Deployment/bicep/deploy_azure_function_script.bicep +++ /dev/null @@ -1,42 +0,0 @@ -@description('Specifies the location for resources.') -param solutionName string -param solutionLocation string -param resourceGroupName string -param identity string -param baseUrl string -@secure() -param azureOpenAIApiKey string -param azureOpenAIApiVersion string -param azureOpenAIEndpoint string -@secure() -param azureSearchAdminKey string -param azureSearchServiceEndpoint string -param azureSearchIndex string -param sqlServerName string -param sqlDbName string -param sqlDbUser string -@secure() -param sqlDbPwd string -param functionAppVersion string - -resource deploy_azure_function 'Microsoft.Resources/deploymentScripts@2020-10-01' = { - kind:'AzureCLI' - name: 'deploy_azure_function' - location: solutionLocation // Replace with your desired location - identity:{ - type:'UserAssigned' - userAssignedIdentities: { - '${identity}' : {} - } - } - properties: { - azCliVersion: '2.50.0' - primaryScriptUri: '${baseUrl}Deployment/scripts/create_azure_functions.sh' // deploy-azure-synapse-pipelines.sh - arguments: '${solutionName} ${solutionLocation} ${resourceGroupName} ${baseUrl} ${azureOpenAIApiKey} ${azureOpenAIApiVersion} ${azureOpenAIEndpoint} ${azureSearchAdminKey} ${azureSearchServiceEndpoint} ${azureSearchIndex} ${sqlServerName} ${sqlDbName} ${sqlDbUser} ${sqlDbPwd} ${functionAppVersion}' // Specify any arguments for the script - timeout: 'PT1H' // Specify the desired timeout duration - retentionInterval: 'PT1H' // Specify the desired retention interval - cleanupPreference:'OnSuccess' - } -} - - diff --git a/ClientAdvisor/Deployment/bicep/main.bicep b/ClientAdvisor/Deployment/bicep/main.bicep index 0bbf984f..3e5a110e 100644 --- a/ClientAdvisor/Deployment/bicep/main.bicep +++ b/ClientAdvisor/Deployment/bicep/main.bicep @@ -13,13 +13,46 @@ param cosmosLocation string // param fabricWorkspaceId string var resourceGroupLocation = resourceGroup().location -var resourceGroupName = resourceGroup().name // var subscriptionId = subscription().subscriptionId var solutionLocation = resourceGroupLocation var baseUrl = 'https://raw.githubusercontent.com/microsoft/Build-your-own-copilot-Solution-Accelerator/main/ClientAdvisor/' var appversion = 'latest' +var functionAppSqlPrompt = '''A valid T-SQL query to find {query} for tables and columns provided below: + 1. Table: Clients + Columns: ClientId,Client,Email,Occupation,MaritalStatus,Dependents + 2. Table: InvestmentGoals + Columns: ClientId,InvestmentGoal + 3. Table: Assets + Columns: ClientId,AssetDate,Investment,ROI,Revenue,AssetType + 4. Table: ClientSummaries + Columns: ClientId,ClientSummary + 5. Table: InvestmentGoalsDetails + Columns: ClientId,InvestmentGoal,TargetAmount,Contribution + 6. Table: Retirement + Columns: ClientId,StatusDate,RetirementGoalProgress,EducationGoalProgress + 7.Table: ClientMeetings + Columns: ClientId,ConversationId,Title,StartTime,EndTime,Advisor,ClientEmail + Use Investement column from Assets table as value always. + Assets table has snapshots of values by date. Do not add numbers across different dates for total values. + Do not use client name in filter. + Do not include assets values unless asked for. + Always use ClientId = {clientid} in the query filter. + Always return client name in the query. + Only return the generated sql query. do not return anything else''' + +var functionAppCallTranscriptSystemPrompt = '''You are an assistant who provides wealth advisors with helpful information to prepare for client meetings. + You have access to the client’s meeting call transcripts. + You can use this information to answer questions about the clients''' + +var functionAppStreamTextSystemPrompt = '''You are a helpful assistant to a wealth advisor. + Do not answer any questions not related to wealth advisors queries. + If the client name and client id do not match, only return - Please only ask questions about the selected client or select another client to inquire about their details. do not return any other information. + Only use the client name returned from database in the response. + If you cannot answer the question, always return - I cannot answer this question from the data available. Please rephrase or add more details. + ** Remove any client identifiers or ids or numbers or ClientId in the final response.''' + // ========== Managed Identity ========== // module managedIdentityModule 'deploy_managed_identity.bicep' = { name: 'deploy_managed_identity' @@ -101,12 +134,11 @@ module uploadFiles 'deploy_upload_files_script.bicep' = { dependsOn:[storageAccountModule] } -module azureFunctions 'deploy_azure_function_script.bicep' = { - name : 'deploy_azure_function_script' +module azureFunctions 'deploy_azure_function.bicep' = { + name : 'deploy_azure_function' params:{ solutionName: solutionPrefix solutionLocation: solutionLocation - resourceGroupName:resourceGroupName azureOpenAIApiKey:azOpenAI.outputs.openAIOutput.openAPIKey azureOpenAIApiVersion:'2024-02-15-preview' azureOpenAIEndpoint:azOpenAI.outputs.openAIOutput.openAPIEndpoint @@ -117,9 +149,10 @@ module azureFunctions 'deploy_azure_function_script.bicep' = { sqlDbName:sqlDBModule.outputs.sqlDbOutput.sqlDbName sqlDbUser:sqlDBModule.outputs.sqlDbOutput.sqlDbUser sqlDbPwd:sqlDBModule.outputs.sqlDbOutput.sqlDbPwd - identity:managedIdentityModule.outputs.managedIdentityOutput.id - baseUrl:baseUrl functionAppVersion: appversion + sqlSystemPrompt: functionAppSqlPrompt + callTranscriptSystemPrompt: functionAppCallTranscriptSystemPrompt + streamTextSystemPrompt: functionAppStreamTextSystemPrompt } dependsOn:[storageAccountModule] } diff --git a/ClientAdvisor/Deployment/bicep/main.json b/ClientAdvisor/Deployment/bicep/main.json index 275c12cb..007e39f1 100644 --- a/ClientAdvisor/Deployment/bicep/main.json +++ b/ClientAdvisor/Deployment/bicep/main.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "15258689682412466487" + "version": "0.32.4.45862", + "templateHash": "8614046715488453239" } }, "parameters": { @@ -26,10 +26,12 @@ }, "variables": { "resourceGroupLocation": "[resourceGroup().location]", - "resourceGroupName": "[resourceGroup().name]", "solutionLocation": "[variables('resourceGroupLocation')]", "baseUrl": "https://raw.githubusercontent.com/microsoft/Build-your-own-copilot-Solution-Accelerator/main/ClientAdvisor/", - "appversion": "latest" + "appversion": "latest", + "functionAppSqlPrompt": "A valid T-SQL query to find {query} for tables and columns provided below:\r\n 1. Table: Clients\r\n Columns: ClientId,Client,Email,Occupation,MaritalStatus,Dependents\r\n 2. Table: InvestmentGoals\r\n Columns: ClientId,InvestmentGoal\r\n 3. Table: Assets\r\n Columns: ClientId,AssetDate,Investment,ROI,Revenue,AssetType\r\n 4. Table: ClientSummaries\r\n Columns: ClientId,ClientSummary\r\n 5. Table: InvestmentGoalsDetails\r\n Columns: ClientId,InvestmentGoal,TargetAmount,Contribution\r\n 6. Table: Retirement\r\n Columns: ClientId,StatusDate,RetirementGoalProgress,EducationGoalProgress\r\n 7.Table: ClientMeetings\r\n Columns: ClientId,ConversationId,Title,StartTime,EndTime,Advisor,ClientEmail\r\n Use Investement column from Assets table as value always.\r\n Assets table has snapshots of values by date. Do not add numbers across different dates for total values.\r\n Do not use client name in filter.\r\n Do not include assets values unless asked for.\r\n Always use ClientId = {clientid} in the query filter.\r\n Always return client name in the query.\r\n Only return the generated sql query. do not return anything else", + "functionAppCallTranscriptSystemPrompt": "You are an assistant who provides wealth advisors with helpful information to prepare for client meetings.\r\n You have access to the client’s meeting call transcripts.\r\n You can use this information to answer questions about the clients", + "functionAppStreamTextSystemPrompt": "You are a helpful assistant to a wealth advisor.\r\n Do not answer any questions not related to wealth advisors queries.\r\n If the client name and client id do not match, only return - Please only ask questions about the selected client or select another client to inquire about their details. do not return any other information.\r\n Only use the client name returned from database in the response.\r\n If you cannot answer the question, always return - I cannot answer this question from the data available. Please rephrase or add more details.\r\n ** Remove any client identifiers or ids or numbers or ClientId in the final response." }, "resources": [ { @@ -56,8 +58,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "12508267066278938117" + "version": "0.32.4.45862", + "templateHash": "9540019694218374629" } }, "parameters": { @@ -145,8 +147,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "11828480827139544665" + "version": "0.32.4.45862", + "templateHash": "12718237112242025023" } }, "parameters": { @@ -308,8 +310,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "3549774837624453156" + "version": "0.32.4.45862", + "templateHash": "13214455762521164459" } }, "parameters": { @@ -404,7 +406,8 @@ "publicAccess": "None" }, "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts/blobServices', parameters('saName'), 'default')]" + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', parameters('saName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', parameters('saName'))]" ] }, { @@ -466,8 +469,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "12627712322898660298" + "version": "0.32.4.45862", + "templateHash": "12781397079288954316" } }, "parameters": { @@ -624,8 +627,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "17239529093958196867" + "version": "0.32.4.45862", + "templateHash": "6507317467445174187" } }, "parameters": { @@ -706,8 +709,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "5720304969592308179" + "version": "0.32.4.45862", + "templateHash": "13153152178869896502" } }, "parameters": { @@ -794,8 +797,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "2043450591502080061" + "version": "0.32.4.45862", + "templateHash": "10512077094934475379" } }, "parameters": { @@ -925,8 +928,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "6087269027573253102" + "version": "0.32.4.45862", + "templateHash": "11104800647186344148" } }, "parameters": { @@ -982,7 +985,7 @@ { "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", - "name": "deploy_azure_function_script", + "name": "deploy_azure_function", "properties": { "expressionEvaluationOptions": { "scope": "inner" @@ -995,9 +998,6 @@ "solutionLocation": { "value": "[variables('solutionLocation')]" }, - "resourceGroupName": { - "value": "[variables('resourceGroupName')]" - }, "azureOpenAIApiKey": { "value": "[reference(resourceId('Microsoft.Resources/deployments', 'deploy_azure_open_ai'), '2022-09-01').outputs.openAIOutput.value.openAPIKey]" }, @@ -1028,14 +1028,17 @@ "sqlDbPwd": { "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_sql_db'), '2022-09-01').outputs.sqlDbOutput.value.sqlDbPwd]" }, - "identity": { - "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_managed_identity'), '2022-09-01').outputs.managedIdentityOutput.value.id]" - }, - "baseUrl": { - "value": "[variables('baseUrl')]" - }, "functionAppVersion": { "value": "[variables('appversion')]" + }, + "sqlSystemPrompt": { + "value": "[variables('functionAppSqlPrompt')]" + }, + "callTranscriptSystemPrompt": { + "value": "[variables('functionAppCallTranscriptSystemPrompt')]" + }, + "streamTextSystemPrompt": { + "value": "[variables('functionAppStreamTextSystemPrompt')]" } }, "template": { @@ -1044,8 +1047,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "10231530585958508769" + "version": "0.32.4.45862", + "templateHash": "11955391860946221428" } }, "parameters": { @@ -1058,15 +1061,6 @@ "solutionLocation": { "type": "string" }, - "resourceGroupName": { - "type": "string" - }, - "identity": { - "type": "string" - }, - "baseUrl": { - "type": "string" - }, "azureOpenAIApiKey": { "type": "securestring" }, @@ -1099,29 +1093,186 @@ }, "functionAppVersion": { "type": "string" + }, + "sqlSystemPrompt": { + "type": "string", + "metadata": { + "description": "Azure Function App SQL System Prompt" + } + }, + "callTranscriptSystemPrompt": { + "type": "string", + "metadata": { + "description": "Azure Function App CallTranscript System Prompt" + } + }, + "streamTextSystemPrompt": { + "type": "string", + "metadata": { + "description": "Azure Function App Stream Text System Prompt" + } } }, + "variables": { + "functionAppName": "[format('{0}fn', parameters('solutionName'))]", + "azureOpenAIDeploymentModel": "gpt-4", + "azureOpenAIEmbeddingDeployment": "text-embedding-ada-002", + "valueOne": "1" + }, "resources": [ { - "type": "Microsoft.Resources/deploymentScripts", - "apiVersion": "2020-10-01", - "name": "deploy_azure_function", - "kind": "AzureCLI", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2021-04-01", + "name": "[format('{0}fnstorage', parameters('solutionName'))]", "location": "[parameters('solutionLocation')]", - "identity": { - "type": "UserAssigned", - "userAssignedIdentities": { - "[format('{0}', parameters('identity'))]": {} - } + "sku": { + "name": "Standard_LRS" }, + "kind": "StorageV2", "properties": { - "azCliVersion": "2.50.0", - "primaryScriptUri": "[format('{0}Deployment/scripts/create_azure_functions.sh', parameters('baseUrl'))]", - "arguments": "[format('{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10} {11} {12} {13} {14}', parameters('solutionName'), parameters('solutionLocation'), parameters('resourceGroupName'), parameters('baseUrl'), parameters('azureOpenAIApiKey'), parameters('azureOpenAIApiVersion'), parameters('azureOpenAIEndpoint'), parameters('azureSearchAdminKey'), parameters('azureSearchServiceEndpoint'), parameters('azureSearchIndex'), parameters('sqlServerName'), parameters('sqlDbName'), parameters('sqlDbUser'), parameters('sqlDbPwd'), parameters('functionAppVersion'))]", - "timeout": "PT1H", - "retentionInterval": "PT1H", - "cleanupPreference": "OnSuccess" + "allowSharedKeyAccess": false } + }, + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2022-10-01", + "name": "[format('workspace-{0}', parameters('solutionName'))]", + "location": "[parameters('solutionLocation')]" + }, + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('functionAppName')]", + "location": "[parameters('solutionLocation')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "publicNetworkAccessForIngestion": "Enabled", + "publicNetworkAccessForQuery": "Enabled", + "WorkspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', format('workspace-{0}', parameters('solutionName')))]" + }, + "dependsOn": [ + "[resourceId('Microsoft.OperationalInsights/workspaces', format('workspace-{0}', parameters('solutionName')))]" + ] + }, + { + "type": "Microsoft.App/managedEnvironments", + "apiVersion": "2022-06-01-preview", + "name": "[format('{0}env', parameters('solutionName'))]", + "location": "[parameters('solutionLocation')]", + "sku": { + "name": "Consumption" + }, + "properties": { + "appLogsConfiguration": { + "destination": "log-analytics", + "logAnalyticsConfiguration": { + "customerId": "[reference(resourceId('Microsoft.OperationalInsights/workspaces', format('workspace-{0}', parameters('solutionName'))), '2022-10-01').customerId]", + "sharedKey": "[listKeys(resourceId('Microsoft.OperationalInsights/workspaces', format('workspace-{0}', parameters('solutionName'))), '2022-10-01').primarySharedKey]" + } + } + }, + "dependsOn": [ + "[resourceId('Microsoft.OperationalInsights/workspaces', format('workspace-{0}', parameters('solutionName')))]" + ] + }, + { + "type": "Microsoft.Web/sites", + "apiVersion": "2024-04-01", + "name": "[variables('functionAppName')]", + "location": "[parameters('solutionLocation')]", + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "managedEnvironmentId": "[resourceId('Microsoft.App/managedEnvironments', format('{0}env', parameters('solutionName')))]", + "siteConfig": { + "linuxFxVersion": "[format('DOCKER|bycwacontainerreg.azurecr.io/byc-wa-fn:{0}', parameters('functionAppVersion'))]", + "appSettings": [ + { + "name": "APPINSIGHTS_INSTRUMENTATIONKEY", + "value": "[reference(resourceId('Microsoft.Insights/components', variables('functionAppName')), '2015-05-01').InstrumentationKey]" + }, + { + "name": "AZURE_OPEN_AI_API_KEY", + "value": "[parameters('azureOpenAIApiKey')]" + }, + { + "name": "AZURE_OPEN_AI_DEPLOYMENT_MODEL", + "value": "[variables('azureOpenAIDeploymentModel')]" + }, + { + "name": "AZURE_OPEN_AI_ENDPOINT", + "value": "[parameters('azureOpenAIEndpoint')]" + }, + { + "name": "AZURE_OPENAI_EMBEDDING_DEPLOYMENT", + "value": "[variables('azureOpenAIEmbeddingDeployment')]" + }, + { + "name": "OPENAI_API_VERSION", + "value": "[parameters('azureOpenAIApiVersion')]" + }, + { + "name": "AZURE_AI_SEARCH_API_KEY", + "value": "[parameters('azureSearchAdminKey')]" + }, + { + "name": "AZURE_AI_SEARCH_ENDPOINT", + "value": "[parameters('azureSearchServiceEndpoint')]" + }, + { + "name": "AZURE_SEARCH_INDEX", + "value": "[parameters('azureSearchIndex')]" + }, + { + "name": "PYTHON_ENABLE_INIT_INDEXING", + "value": "[variables('valueOne')]" + }, + { + "name": "PYTHON_ISOLATE_WORKER_DEPENDENCIES", + "value": "[variables('valueOne')]" + }, + { + "name": "SQLDB_CONNECTION_STRING", + "value": "TBD" + }, + { + "name": "SQLDB_SERVER", + "value": "[parameters('sqlServerName')]" + }, + { + "name": "SQLDB_DATABASE", + "value": "[parameters('sqlDbName')]" + }, + { + "name": "SQLDB_USERNAME", + "value": "[parameters('sqlDbUser')]" + }, + { + "name": "SQLDB_PASSWORD", + "value": "[parameters('sqlDbPwd')]" + }, + { + "name": "AZURE_SQL_SYSTEM_PROMPT", + "value": "[parameters('sqlSystemPrompt')]" + }, + { + "name": "AZURE_CALL_TRANSCRIPT_SYSTEM_PROMPT", + "value": "[parameters('callTranscriptSystemPrompt')]" + }, + { + "name": "AZURE_OPENAI_STREAM_TEXT_SYSTEM_PROMPT", + "value": "[parameters('streamTextSystemPrompt')]" + } + ] + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Insights/components', variables('functionAppName'))]", + "[resourceId('Microsoft.App/managedEnvironments', format('{0}env', parameters('solutionName')))]" + ] } ] } @@ -1129,7 +1280,6 @@ "dependsOn": [ "[resourceId('Microsoft.Resources/deployments', 'deploy_azure_open_ai')]", "[resourceId('Microsoft.Resources/deployments', 'deploy_ai_search_service')]", - "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_managed_identity')]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_sql_db')]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_storage_account')]" ] @@ -1157,8 +1307,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "14069568468347897481" + "version": "0.32.4.45862", + "templateHash": "11501780755841251697" } }, "parameters": { @@ -1186,7 +1336,7 @@ } }, "dependsOn": [ - "[resourceId('Microsoft.Resources/deployments', 'deploy_azure_function_script')]", + "[resourceId('Microsoft.Resources/deployments', 'deploy_azure_function')]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_managed_identity')]" ] }, @@ -1271,8 +1421,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "5271454688668874284" + "version": "0.32.4.45862", + "templateHash": "16538647807599840496" } }, "parameters": { @@ -1763,8 +1913,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "17388889661434191538" + "version": "0.32.4.45862", + "templateHash": "9968723784632879247" } }, "parameters": { @@ -1961,8 +2111,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "16594210839276135691" + "version": "0.32.4.45862", + "templateHash": "4502112701228496974" } }, "parameters": { @@ -2606,8 +2756,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.34.60546", - "templateHash": "8033637033572984239" + "version": "0.32.4.45862", + "templateHash": "2813064152180428298" }, "description": "Creates a SQL role assignment under an Azure Cosmos DB account." }, diff --git a/ClientAdvisor/Deployment/data/clientdata.zip b/ClientAdvisor/Deployment/data/clientdata.zip index f3c18a97..be62bedb 100644 Binary files a/ClientAdvisor/Deployment/data/clientdata.zip and b/ClientAdvisor/Deployment/data/clientdata.zip differ diff --git a/ClientAdvisor/Deployment/scripts/create_azure_functions.sh b/ClientAdvisor/Deployment/scripts/create_azure_functions.sh deleted file mode 100644 index d7d1a3b9..00000000 --- a/ClientAdvisor/Deployment/scripts/create_azure_functions.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# Variables -solutionName="$1" -solutionLocation="$2" -resourceGroupName="$3" -baseUrl="$4" -azureOpenAIApiKey="$5" -azureOpenAIApiVersion="$6" -azureOpenAIEndpoint="$7" -azureSearchAdminKey="$8" -azureSearchServiceEndpoint="$9" -azureSearchIndex="${10}" -sqlServerName="${11}" -sqlDbName="${12}" -sqlDbUser="${13}" -sqlDbPwd="${14}" -functionAppVersion="${15}" - -azureOpenAIDeploymentModel="gpt-4" -azureOpenAIEmbeddingDeployment="text-embedding-ada-002" - -env_name=${solutionName}"env" -storageAccount=${solutionName}"fnstorage" -functionappname=${solutionName}"fn" -valueone="1" - -# sqlDBConn="DRIVER={ODBC Driver 18 for SQL Server};SERVER="${sqlServerName}".database.windows.net;DATABASE="${sqlDbName}";UID="${sqlDbUser}";PWD="${sqlDbPwd} - -#sqlDBConn="DRIVER={ODBC Driver 18 for SQL Server};SERVER=${sqlServerName}.database.windows.net;DATABASE=${sqlDbName};UID=${sqlDbUser};PWD=${sqlDbPwd}" -sqlDBConn="TBD" - -az containerapp env create --name $env_name --enable-workload-profiles --resource-group $resourceGroupName --location $solutionLocation - -az storage account create --name $storageAccount --location eastus --resource-group $resourceGroupName --sku Standard_LRS --allow-shared-key-access false - -az functionapp create --resource-group $resourceGroupName --name $functionappname \ - --environment $env_name --storage-account $storageAccount \ - --functions-version 4 --runtime python \ - --image bycwacontainerreg.azurecr.io/byc-wa-fn:$functionAppVersion - -# Sleep for 120 seconds -echo "Waiting for 120 seconds to ensure the Function App is properly created..." -sleep 60 - -az functionapp config appsettings set --name $functionappname -g $resourceGroupName \ - --settings AZURE_OPEN_AI_API_KEY=$azureOpenAIApiKey AZURE_OPEN_AI_DEPLOYMENT_MODEL=$azureOpenAIDeploymentModel \ - AZURE_OPEN_AI_ENDPOINT=$azureOpenAIEndpoint AZURE_OPENAI_EMBEDDING_DEPLOYMENT=$azureOpenAIEmbeddingDeployment \ - OPENAI_API_VERSION=$azureOpenAIApiVersion \ - AZURE_AI_SEARCH_API_KEY=$azureSearchAdminKey AZURE_AI_SEARCH_ENDPOINT=$azureSearchServiceEndpoint \ - AZURE_SEARCH_INDEX=$azureSearchIndex \ - PYTHON_ENABLE_INIT_INDEXING=$valueone PYTHON_ISOLATE_WORKER_DEPENDENCIES=$valueone \ - SQLDB_CONNECTION_STRING=$sqlDBConn \ - SQLDB_SERVER=$sqlServerName SQLDB_DATABASE=$sqlDbName SQLDB_USERNAME=$sqlDbUser SQLDB_PASSWORD=$sqlDbPwd \ No newline at end of file diff --git a/ClientAdvisor/Deployment/scripts/fabric_scripts/data/clientdata/Assets.csv b/ClientAdvisor/Deployment/scripts/fabric_scripts/data/clientdata/Assets.csv index 87bb5c37..4cdd3276 100644 --- a/ClientAdvisor/Deployment/scripts/fabric_scripts/data/clientdata/Assets.csv +++ b/ClientAdvisor/Deployment/scripts/fabric_scripts/data/clientdata/Assets.csv @@ -168,26 +168,26 @@ ClientId,AssetDate,Investment,ROI,Revenue,AssetType 10006,8/1/2023,721222,-0.02,707518.78,Equities 10006,8/1/2023,624820,0.0,623445.4,Bonds 10004,10/1/2023,219809,0.11,244449.59,Cash -10004,10/1/2023,219809,1.12,466126.97,Equities -10004,10/1/2023,219809,0.11,244449.59,Bonds -10004,11/1/2023,219809,0.94,426297.57,Equities -10004,6/1/2023,219809,0.92,422406.96,Equities -10004,12/1/2023,219809,0.86,409767.94,Equities -10004,5/1/2023,219809,0.85,407372.02,Equities +10004,10/1/2023,190000,0.07,203300,Equities +10004,10/1/2023,75000,0.07,80250,Bonds +10004,11/1/2023,180000,0.06,190800,Equities +10004,6/1/2023,235000,0.16,272600,Equities +10004,12/1/2023,180000,0.04,187200,Equities +10004,5/1/2023,220000,0.2,264000,Equities 10004,8/1/2023,219809,0.06,232821.69,Cash -10004,8/1/2023,219809,0.59,349869.99,Equities -10004,8/1/2023,219809,0.06,232821.69,Bonds +10004,8/1/2023,198000,0.09,215820,Equities +10004,8/1/2023,85000,0.09,92650,Bonds 10004,9/1/2023,219809,0.06,232513.96,Cash -10004,9/1/2023,219809,0.58,346858.6,Equities -10004,9/1/2023,219809,0.06,232513.96,Bonds +10004,9/1/2023,190000,0.08,205200,Equities +10004,9/1/2023,80000,0.08,86400,Bonds 10004,7/1/2023,219809,0.05,231810.57,Cash -10004,7/1/2023,219809,0.55,339780.75,Equities -10004,7/1/2023,219809,0.05,231810.57,Bonds +10004,7/1/2023,242000,0.1,266200,Equities +10004,7/1/2023,99000,0.1,108900,Bonds 10004,11/1/2023,219809,0.09,240449.07,Cash -10004,11/1/2023,219809,0.09,240449.07,Bonds +10004,11/1/2023,70000,0.06,74200,Bonds 10004,6/1/2023,219809,0.09,240075.39,Cash -10004,6/1/2023,219809,0.09,240075.39,Bonds +10004,6/1/2023,93000,0.16,107880,Bonds 10004,12/1/2023,219809,0.09,238800.5,Cash -10004,12/1/2023,219809,0.09,238800.5,Bonds +10004,12/1/2023,70000,0.04,728000,Bonds 10004,5/1/2023,219809,0.09,238558.71,Cash -10004,5/1/2023,219809,0.09,238558.71,Bonds \ No newline at end of file +10004,5/1/2023,90000,0.02,108000,Bonds diff --git a/ResearchAssistant/App/.flake8 b/ResearchAssistant/App/.flake8 index c462975a..93f63e5d 100644 --- a/ResearchAssistant/App/.flake8 +++ b/ResearchAssistant/App/.flake8 @@ -1,4 +1,5 @@ [flake8] max-line-length = 88 -extend-ignore = E501, E203 -exclude = .venv, frontend, \ No newline at end of file +extend-ignore = E501 +exclude = .venv, frontend +ignore = E203, W503, G004, G200 \ No newline at end of file diff --git a/ResearchAssistant/App/requirements.txt b/ResearchAssistant/App/requirements.txt index 2ef649d2..94c812e3 100644 --- a/ResearchAssistant/App/requirements.txt +++ b/ResearchAssistant/App/requirements.txt @@ -10,4 +10,10 @@ pytest-cov==6.0.0 flake8==7.1.1 black==24.10.0 autoflake==2.3.1 -isort==5.13.2 \ No newline at end of file +isort==5.13.2 +pylint==2.17.4 + +# Testing tools +pytest>=8.2,<9 # Compatible version for pytest-asyncio +pytest-asyncio==0.24.0 +pytest-cov==5.0.0 \ No newline at end of file