diff --git a/Allfiles/Labs/14/.env b/Allfiles/Labs/14/.env new file mode 100644 index 0000000..97c2a86 --- /dev/null +++ b/Allfiles/Labs/14/.env @@ -0,0 +1,14 @@ +# PostgreSQL connection +PGHOST= +PGUSER=pgAdmin +PGPASSWORD= +PGDATABASE=ContosoHelpDesk + +# Azure OpenAI +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_ENDPOINT= # e.g., https://oai-learn--.openai.azure.com +OPENAI_API_VERSION=2024-02-15-preview + +# Deployment names (match the Bicep resources) +OPENAI_EMBED_DEPLOYMENT=embedding +OPENAI_CHAT_DEPLOYMENT=chat diff --git a/Allfiles/Labs/14/CompanyPolicies.py b/Allfiles/Labs/14/CompanyPolicies.py new file mode 100644 index 0000000..b15caea --- /dev/null +++ b/Allfiles/Labs/14/CompanyPolicies.py @@ -0,0 +1,55 @@ +import os +from contextlib import contextmanager +from dotenv import load_dotenv +import psycopg2 +from langchain_openai import AzureChatOpenAI + +load_dotenv() + +# Create a short-lived PostgreSQL connection +@contextmanager +def get_conn(): + conn = psycopg2.connect( + host=os.getenv("PGHOST"), + user=os.getenv("PGUSER"), + password=os.getenv("PGPASSWORD"), + dbname=os.getenv("PGDATABASE"), + connect_timeout=10, + ) + try: + yield conn + finally: + conn.close() + +# Retrieve top-k rows by cosine similarity (embedding must be present) + + + +# Format retrieved chunks for the model prompt + + + +# Call Azure OpenAI to answer using the provided context + + + +# Main: prompt, retrieve, answer, loop on demand +if __name__ == "__main__": + while True: + q = input("Enter your question (or press Enter to use a sample): ").strip() \ + or "How many vacation days do employees get?" + + chunks = retrieve_chunks(q, top_k=5) + if not chunks: + print("\nNo relevant content found.") + else: + answer = generate_answer(q, chunks) + print("\n--- Answer ---\n", answer) + + again = input("\nAsk another question? [y/N]: ").strip().lower() + if again in ("y", "yes"): + os.system("cls" if os.name == "nt" else "clear") + continue + else: + print("Goodbye!") + break diff --git a/Allfiles/Labs/14/requirements.txt b/Allfiles/Labs/14/requirements.txt new file mode 100644 index 0000000..88c51d1 --- /dev/null +++ b/Allfiles/Labs/14/requirements.txt @@ -0,0 +1,3 @@ +psycopg2-binary +python-dotenv +langchain-openai diff --git a/Allfiles/Labs/Shared/company-policies.csv b/Allfiles/Labs/Shared/company-policies.csv new file mode 100644 index 0000000..0cfb92d --- /dev/null +++ b/Allfiles/Labs/Shared/company-policies.csv @@ -0,0 +1,109 @@ +title,department,policy_text,topic +Performance Review Policy,HR,Employees receive formal performance reviews annually.,hr_lifecycle +Dress Code Policy,HR,Employees are expected to dress in business casual attire.,general +Warranty Policy,Support,Products are covered under warranty for one year.,legal_records +Account Management Policy,Operations,Accounts must be reviewed quarterly.,access_accounts +Anti-Harassment Policy,HR,Harassment of any kind is not tolerated.,ethics_compliance +Recognition Program Policy,HR,Employees may nominate peers for recognition awards.,general +Software Installation Policy,IT,Only approved software may be installed on company devices.,general +Workplace Accommodation Policy,HR,Employees may request reasonable accommodations.,general +Data Privacy Policy,Legal,Employees must protect customer data and follow privacy regulations.,data_privacy +Work From Abroad Policy,HR,Employees may work abroad for up to 30 days with approval.,leave_time +File Naming Policy,Operations,Files must follow naming conventions for easy retrieval.,general +Churn Prevention Policy,Marketing,Customer churn must be analyzed and addressed.,customer_support +Event Sponsorship Policy,Marketing,Event sponsorships must align with company goals.,marketing_comms +Company Vehicle Policy,Operations,Use of company vehicles must be logged and approved.,facilities_safety +Onboarding Policy,HR,New hires must complete onboarding within their first week.,hr_lifecycle +Legal Hold Policy,Legal,Legal holds must be applied to relevant documents.,legal_records +Budget Planning Policy,Finance,Departments must submit budgets by Q4.,general +Holiday Schedule Policy,HR,Company holidays are published annually and observed accordingly.,leave_time +Support Ticket Policy,IT,Support tickets must be resolved within SLA.,customer_support +Supplier Evaluation Policy,Operations,Suppliers must be evaluated annually.,procurement_suppliers +Parking Policy,Facilities,Parking spaces are assigned and must be used accordingly.,facilities_safety +Equal Opportunity Policy,Operations,The company is committed to equal opportunity employment.,general +Customer Interaction Policy,Operations,Employees must maintain professionalism in all customer interactions.,general +Customer Feedback Policy,Operations,Customer feedback must be logged and reviewed.,customer_support +Marketing Approval Policy,Marketing,Marketing materials must be approved before release.,marketing_comms +Remote Work Policy,HR,Employees may work remotely up to three days per week with manager approval.,leave_time +Code of Conduct,HR,All employees must adhere to professional behavior standards.,ethics_compliance +Meeting Room Booking Policy,Operations,Meeting rooms must be booked in advance using the portal.,operations_process +Internal Communication Policy,Operations,Use company channels for internal communication.,marketing_comms +Mobile Device Policy,IT,Employees may request company-issued mobile devices for work.,devices_assets +Training Reimbursement Policy,Finance,Employees may be reimbursed for approved external training.,travel_expense +Client Gift Policy,Operations,Client gifts must comply with company guidelines.,ethics_compliance +Campaign Review Policy,Operations,Marketing campaigns must be reviewed quarterly.,marketing_comms +Employee Referral Policy,Operations,Employees may refer candidates for open positions.,hr_lifecycle +Offer Letter Policy,Operations,Offer letters must follow approved templates.,hr_lifecycle +Work Anniversary Recognition Policy,HR,Employees are recognized on their work anniversaries.,general +Conflict of Interest Policy,Legal,Employees must disclose any potential conflicts of interest.,ethics_compliance +Business Continuity Policy,Operations,Departments must maintain business continuity plans.,operations_process +Office Supplies Policy,Facilities,Office supplies must be ordered through the approved vendor.,facilities_safety +System Downtime Policy,IT,Planned downtimes are communicated in advance.,operations_process +Delivery Confirmation Policy,Operations,Deliveries must be confirmed upon receipt.,customer_support +Hardware Upgrade Policy,Operations,Hardware upgrades must be requested through IT.,devices_assets +Exit Interview Policy,HR,Departing employees are encouraged to participate in an exit interview.,hr_lifecycle +Security Incident Policy,IT,Security incidents must be reported immediately.,security +Promotion Policy,HR,Promotions are based on performance and business needs.,hr_lifecycle +Sales Incentive Policy,Sales,Sales incentives must be documented and approved.,general +Probation Period Policy,HR,New hires are subject to a 90-day probation period.,hr_lifecycle +Mentorship Program Policy,HR,Employees may participate in the mentorship program.,hr_lifecycle +Public Speaking Policy,Marketing,Employees must get approval before public speaking engagements.,marketing_comms +Sustainability Policy,Operations,Departments should consider sustainability in operations.,general +Remote Access Policy,IT,Remote access must use secure VPN connections.,access_accounts +Incident Response Policy,IT,Incident response procedures must be followed during disruptions.,general +Inventory Management Policy,Finance,Inventory must be tracked and reconciled monthly.,operations_process +Asset Disposal Policy,Operations,IT assets must be disposed of securely.,devices_assets +Meeting Minutes Policy,Operations,Meeting minutes must be documented and shared.,operations_process +Office Access Policy,Facilities,Access badges must be worn and not shared.,facilities_safety +Product Return Policy,Support,Returns must follow the documented return process.,customer_support +Email Signature Policy,Operations,All employees must use the approved email signature format.,marketing_comms +Cloud Storage Policy,Operations,Use approved cloud storage for company documents.,security +Shipping Policy,Finance,Shipping must use approved carriers.,legal_records +Social Media Policy,Marketing,Employees must not disclose confidential information on social media.,marketing_comms +Remote Meeting Policy,Operations,Remote meetings should use approved video conferencing tools.,general +Interview Panel Policy,HR,Interview panels must include diverse representation.,hr_lifecycle +Expense Approval Policy,Finance,All expenses over $500 require prior approval from Finance.,travel_expense +Customer Onboarding Policy,HR,New customers must complete onboarding steps.,hr_lifecycle +Document Retention Policy,Operations,Documents must be retained according to legal requirements.,legal_records +Cybersecurity Awareness Policy,Operations,Employees must complete annual cybersecurity training.,security +Corporate Card Policy,Finance,Corporate card usage must comply with expense guidelines.,travel_expense +Flexible Hours Policy,Operations,Employees may adjust work hours with manager approval.,leave_time +Sick Leave Policy,HR,Employees accrue sick leave monthly for personal or family illness.,leave_time +Social Event Policy,Marketing,Social events must be approved and budgeted.,general +Content Publishing Policy,Operations,Content must be reviewed before publishing.,marketing_comms +Team Building Policy,Operations,Team building events must be approved and budgeted.,facilities_safety +Home Office Equipment Policy,HR,Employees may request ergonomic equipment for remote work.,devices_assets +Laptop Usage Policy,IT,Company laptops must be used for business purposes only.,leave_time +Procurement Policy,Finance,All purchases must follow the procurement process.,procurement_suppliers +Bereavement Leave Policy,HR,Employees may take up to 5 days of bereavement leave.,leave_time +Disaster Recovery Policy,Operations,IT must maintain disaster recovery protocols.,general +Petty Cash Policy,Finance,Petty cash usage must be documented and reconciled monthly.,travel_expense +Brand Guidelines Policy,Marketing,All materials must follow brand guidelines.,marketing_comms +Project Kickoff Policy,Engineering,Projects must begin with a formal kickoff meeting.,engineering_it +Contract Review Policy,Operations,Contracts must be reviewed by Legal before signing.,legal_records +Parental Leave Policy,HR,Eligible employees may take up to 12 weeks of paid parental leave.,leave_time +Background Check Policy,Operations,All new hires must pass background checks.,general +Tuition Assistance Policy,HR,Employees may apply for tuition assistance for approved programs.,general +Internet Usage Policy,IT,Company internet should be used for work-related activities only.,general +Whistleblower Policy,Legal,Employees may report unethical behavior anonymously.,ethics_compliance +Exit Clearance Policy,Operations,Departing employees must complete exit clearance.,general +Vendor Management Policy,Finance,Vendors must be approved and reviewed annually.,procurement_suppliers +Quality Assurance Policy,Engineering,Products must pass quality checks before release.,general +Conference Attendance Policy,Legal,Employees may attend conferences with prior approval.,legal_records +Media Contact Policy,Operations,Only authorized employees may speak to the media.,general +Diversity and Inclusion Policy,Operations,The company promotes a diverse and inclusive workplace.,general +Workplace Safety Policy,Facilities,Employees must follow safety procedures and report hazards.,facilities_safety +Workplace Cleanliness Policy,Facilities,Employees are responsible for maintaining clean workspaces.,facilities_safety +Helpdesk Ticket Policy,IT,All IT issues must be submitted via helpdesk ticket.,customer_support +Meeting Etiquette Policy,Operations,Meetings should start on time and follow an agenda.,general +Lead Management Policy,Sales,Sales leads must be tracked in the CRM.,general +IT Asset Management Policy,IT,All IT assets must be tracked and returned upon exit.,devices_assets +Password Policy,IT,Passwords must be changed every 90 days and meet complexity requirements.,security +Travel Reimbursement Policy,Finance,Employees must submit receipts for travel expenses within 30 days.,travel_expense +Press Release Policy,Legal,Press releases must be approved by Marketing and Legal.,marketing_comms +Vacation Policy,HR,Employees receive 15 vacation days per year. Unused days may roll over.,leave_time +Escalation Policy,Operations,Customer issues must follow the escalation process.,customer_support +Job Posting Policy,HR,All internal job postings are listed on the company portal.,general +Company Property Policy,Operations,Employees must return company property upon exit.,devices_assets +Change Management Policy,Engineering,Changes to systems must follow the change management process.,operations_process +Renewal Policy,Finance,Renewals must be initiated 60 days before expiration.,legal_records diff --git a/Allfiles/Labs/Shared/company_policies.csv b/Allfiles/Labs/Shared/company_policies.csv deleted file mode 100644 index 8852a24..0000000 --- a/Allfiles/Labs/Shared/company_policies.csv +++ /dev/null @@ -1,109 +0,0 @@ -title,department,policy_text -Performance Review Policy,Sales,Employees receive formal performance reviews annually. -Dress Code Policy,Facilities,Employees are expected to dress in business casual attire. -Warranty Policy,Sales,Products are covered under warranty for one year. -Account Management Policy,Sales,Accounts must be reviewed quarterly. -Anti-Harassment Policy,Marketing,Harassment of any kind is not tolerated. -Recognition Program Policy,Facilities,Employees may nominate peers for recognition awards. -Software Installation Policy,Sales,Only approved software may be installed on company devices. -Workplace Accommodation Policy,Finance,Employees may request reasonable accommodations. -Data Privacy Policy,Operations,Employees must protect customer data and follow privacy regulations. -Work From Abroad Policy,Facilities,Employees may work abroad for up to 30 days with approval. -File Naming Policy,Marketing,Files must follow naming conventions for easy retrieval. -Churn Prevention Policy,Sales,Customer churn must be analyzed and addressed. -Event Sponsorship Policy,Facilities,Event sponsorships must align with company goals. -Company Vehicle Policy,Legal,Use of company vehicles must be logged and approved. -Onboarding Policy,Sales,New hires must complete onboarding within their first week. -Legal Hold Policy,IT,Legal holds must be applied to relevant documents. -Budget Planning Policy,HR,Departments must submit budgets by Q4. -Holiday Schedule Policy,Facilities,Company holidays are published annually and observed accordingly. -Support Ticket Policy,Operations,Support tickets must be resolved within SLA. -Supplier Evaluation Policy,Facilities,Suppliers must be evaluated annually. -Parking Policy,HR,Parking spaces are assigned and must be used accordingly. -Equal Opportunity Policy,HR,The company is committed to equal opportunity employment. -Customer Interaction Policy,Operations,Employees must maintain professionalism in all customer interactions. -Customer Feedback Policy,Operations,Customer feedback must be logged and reviewed. -Marketing Approval Policy,Operations,Marketing materials must be approved before release. -Remote Work Policy,Facilities,Employees may work remotely up to three days per week with manager approval. -Code of Conduct,Legal,All employees must adhere to professional behavior standards. -Meeting Room Booking Policy,Sales,Meeting rooms must be booked in advance using the portal. -Internal Communication Policy,IT,Use company channels for internal communication. -Mobile Device Policy,IT,Employees may request company-issued mobile devices for work. -Training Reimbursement Policy,Sales,Employees may be reimbursed for approved external training. -Client Gift Policy,HR,Client gifts must comply with company guidelines. -Campaign Review Policy,HR,Marketing campaigns must be reviewed quarterly. -Employee Referral Policy,Finance,Employees may refer candidates for open positions. -Offer Letter Policy,Marketing,Offer letters must follow approved templates. -Work Anniversary Recognition Policy,Marketing,Employees are recognized on their work anniversaries. -Conflict of Interest Policy,Marketing,Employees must disclose any potential conflicts of interest. -Business Continuity Policy,HR,Departments must maintain business continuity plans. -Office Supplies Policy,Operations,Office supplies must be ordered through the approved vendor. -System Downtime Policy,Operations,Planned downtimes are communicated in advance. -Delivery Confirmation Policy,Marketing,Deliveries must be confirmed upon receipt. -Hardware Upgrade Policy,IT,Hardware upgrades must be requested through IT. -Exit Interview Policy,Facilities,Departing employees are encouraged to participate in an exit interview. -Security Incident Policy,IT,Security incidents must be reported immediately. -Promotion Policy,Finance,Promotions are based on performance and business needs. -Sales Incentive Policy,Marketing,Sales incentives must be documented and approved. -Probation Period Policy,Facilities,New hires are subject to a 90-day probation period. -Mentorship Program Policy,HR,Employees may participate in the mentorship program. -Public Speaking Policy,Legal,Employees must get approval before public speaking engagements. -Sustainability Policy,Finance,Departments should consider sustainability in operations. -Remote Access Policy,Legal,Remote access must use secure VPN connections. -Incident Response Policy,Legal,Incident response procedures must be followed during disruptions. -Inventory Management Policy,Sales,Inventory must be tracked and reconciled monthly. -Asset Disposal Policy,Sales,IT assets must be disposed of securely. -Meeting Minutes Policy,Sales,Meeting minutes must be documented and shared. -Office Access Policy,HR,Access badges must be worn and not shared. -Product Return Policy,Marketing,Returns must follow the documented return process. -Email Signature Policy,HR,All employees must use the approved email signature format. -Cloud Storage Policy,Operations,Use approved cloud storage for company documents. -Shipping Policy,HR,Shipping must use approved carriers. -Social Media Policy,Finance,Employees must not disclose confidential information on social media. -Remote Meeting Policy,Operations,Remote meetings should use approved video conferencing tools. -Interview Panel Policy,HR,Interview panels must include diverse representation. -Expense Approval Policy,Legal,All expenses over $500 require prior approval from Finance. -Customer Onboarding Policy,HR,New customers must complete onboarding steps. -Document Retention Policy,Legal,Documents must be retained according to legal requirements. -Cybersecurity Awareness Policy,Marketing,Employees must complete annual cybersecurity training. -Corporate Card Policy,Operations,Corporate card usage must comply with expense guidelines. -Flexible Hours Policy,HR,Employees may adjust work hours with manager approval. -Sick Leave Policy,Sales,Employees accrue sick leave monthly for personal or family illness. -Social Event Policy,Finance,Social events must be approved and budgeted. -Content Publishing Policy,Marketing,Content must be reviewed before publishing. -Team Building Policy,Finance,Team building events must be approved and budgeted. -Home Office Equipment Policy,Legal,Employees may request ergonomic equipment for remote work. -Laptop Usage Policy,IT,Company laptops must be used for business purposes only. -Procurement Policy,Marketing,All purchases must follow the procurement process. -Bereavement Leave Policy,Legal,Employees may take up to 5 days of bereavement leave. -Disaster Recovery Policy,Facilities,IT must maintain disaster recovery protocols. -Petty Cash Policy,Legal,Petty cash usage must be documented and reconciled monthly. -Brand Guidelines Policy,Facilities,All materials must follow brand guidelines. -Project Kickoff Policy,IT,Projects must begin with a formal kickoff meeting. -Contract Review Policy,Finance,Contracts must be reviewed by Legal before signing. -Parental Leave Policy,Marketing,Eligible employees may take up to 12 weeks of paid parental leave. -Background Check Policy,Sales,All new hires must pass background checks. -Tuition Assistance Policy,HR,Employees may apply for tuition assistance for approved programs. -Internet Usage Policy,Finance,Company internet should be used for work-related activities only. -Whistleblower Policy,HR,Employees may report unethical behavior anonymously. -Exit Clearance Policy,Sales,Departing employees must complete exit clearance. -Vendor Management Policy,Marketing,Vendors must be approved and reviewed annually. -Quality Assurance Policy,HR,Products must pass quality checks before release. -Conference Attendance Policy,Operations,Employees may attend conferences with prior approval. -Media Contact Policy,Finance,Only authorized employees may speak to the media. -Diversity and Inclusion Policy,Operations,The company promotes a diverse and inclusive workplace. -Workplace Safety Policy,HR,Employees must follow safety procedures and report hazards. -Workplace Cleanliness Policy,Sales,Employees are responsible for maintaining clean workspaces. -Helpdesk Ticket Policy,Operations,All IT issues must be submitted via helpdesk ticket. -Meeting Etiquette Policy,Facilities,Meetings should start on time and follow an agenda. -Lead Management Policy,Finance,Sales leads must be tracked in the CRM. -IT Asset Management Policy,Facilities,All IT assets must be tracked and returned upon exit. -Password Policy,HR,Passwords must be changed every 90 days and meet complexity requirements. -Travel Reimbursement Policy,Facilities,Employees must submit receipts for travel expenses within 30 days. -Press Release Policy,Legal,Press releases must be approved by Marketing and Legal. -Vacation Policy,IT,Employees receive 15 vacation days per year. Unused days may roll over. -Escalation Policy,Facilities,Customer issues must follow the escalation process. -Job Posting Policy,Legal,All internal job postings are listed on the company portal. -Company Property Policy,Facilities,Employees must return company property upon exit. -Change Management Policy,HR,Changes to systems must follow the change management process. -Renewal Policy,Legal,Renewals must be initiated 60 days before expiration. diff --git a/Allfiles/Labs/Shared/deploy-all-plus-foundry.bicep b/Allfiles/Labs/Shared/deploy-all-plus-foundry.bicep new file mode 100644 index 0000000..5e7505c --- /dev/null +++ b/Allfiles/Labs/Shared/deploy-all-plus-foundry.bicep @@ -0,0 +1,258 @@ +@description('Location for all resources.') +param location string = resourceGroup().location + +@description('Unique name for the Azure Database for PostgreSQL.') +param serverName string = 'psql-learn-${resourceGroup().location}-${uniqueString(resourceGroup().id)}' + +@description('The version of PostgreSQL to use.') +param postgresVersion string = '16' + +@description('Login name of the database administrator.') +@minLength(1) +param adminLogin string = 'pgAdmin' + +@description('Password for the database administrator.') +@minLength(8) +@secure() +param adminLoginPassword string + +@description('Name of the database.') +@minLength(1) +param databaseName string = 'rentals' + +@description('Unique name for the Azure OpenAI service.') +param azureOpenAIServiceName string = 'oai-learn-${resourceGroup().location}-${uniqueString(resourceGroup().id)}' + +@description('Unique name for the Azure AI Language service account.') +param languageServiceName string = 'lang-learn-${resourceGroup().location}-${uniqueString(resourceGroup().id)}' + +@description('Unique name for the Azure AI Translator service account.') +param translatorServiceName string = 'trn-learn-${resourceGroup().location}-${uniqueString(resourceGroup().id)}' + +@description('Unique name for the Storage Account.') +param storageAccountName string = 'st${uniqueString(resourceGroup().id)}' + +@description('Unique name for the AI Foundry Hub.') +param aiHubName string = 'aihub-learn-${resourceGroup().location}-${uniqueString(resourceGroup().id)}' + +@description('Unique name for the AI Foundry Project.') +param aiProjectName string = 'aiproj-learn-${resourceGroup().location}-${uniqueString(resourceGroup().id)}' + +@description('Restore the service instead of creating a new instance. This is useful if you previously soft-delted the service and want to restore it. If you are restoring a service, set this to true. Otherwise, leave this as false.') +param restore bool = false + +@description('Creates a PostgreSQL Flexible Server.') +resource postgreSQLFlexibleServer 'Microsoft.DBforPostgreSQL/flexibleServers@2023-03-01-preview' = { + name: serverName + location: location + sku: { + name: 'Standard_D2ds_v4' + tier: 'GeneralPurpose' + } + properties: { + administratorLogin: adminLogin + administratorLoginPassword: adminLoginPassword + authConfig: { + activeDirectoryAuth: 'Disabled' + passwordAuth: 'Enabled' + tenantId: subscription().tenantId + } + backup: { + backupRetentionDays: 7 + geoRedundantBackup: 'Disabled' + } + createMode: 'Default' + highAvailability: { + mode: 'Disabled' + } + storage: { + autoGrow: 'Disabled' + storageSizeGB: 32 + tier: 'P10' + } + version: postgresVersion + } +} + +@description('Firewall rule that checks the "Allow public access from any Azure service within Azure to this server" box.') +resource allowAllAzureServicesAndResourcesWithinAzureIps 'Microsoft.DBforPostgreSQL/flexibleServers/firewallRules@2023-03-01-preview' = { + name: 'AllowAllAzureServicesAndResourcesWithinAzureIps' + parent: postgreSQLFlexibleServer + properties: { + startIpAddress: '0.0.0.0' + endIpAddress: '0.0.0.0' + } +} + +@description('Firewall rule to allow all IP addresses to connect to the server. Should only be used for lab purposes.') +resource allowAll 'Microsoft.DBforPostgreSQL/flexibleServers/firewallRules@2023-03-01-preview' = { + name: 'AllowAll' + parent: postgreSQLFlexibleServer + properties: { + startIpAddress: '0.0.0.0' + endIpAddress: '255.255.255.255' + } +} + +@description('Creates the "rentals" database in the PostgreSQL Flexible Server.') +resource rentalsDatabase 'Microsoft.DBforPostgreSQL/flexibleServers/databases@2023-03-01-preview' = { + name: databaseName + parent: postgreSQLFlexibleServer + properties: { + charset: 'UTF8' + collation: 'en_US.UTF8' + } +} + +@description('Configures the "azure.extensions" parameter to allowlist extensions.') +resource allowlistExtensions 'Microsoft.DBforPostgreSQL/flexibleServers/configurations@2023-03-01-preview' = { + name: 'azure.extensions' + parent: postgreSQLFlexibleServer + dependsOn: [allowAllAzureServicesAndResourcesWithinAzureIps, allowAll, rentalsDatabase] // Ensure the database is created and configured before setting the parameter, as it requires a "restart." + properties: { + source: 'user-override' + value: 'azure_ai,vector' + } +} + +@description('Creates an Azure OpenAI service.') +resource azureOpenAIService 'Microsoft.CognitiveServices/accounts@2023-05-01' = { + name: azureOpenAIServiceName + location: location + kind: 'OpenAI' + sku: { + name: 'S0' + tier: 'Standard' + } + properties: { + customSubDomainName: azureOpenAIServiceName + publicNetworkAccess: 'Enabled' + restore: restore + } +} + +@description('Creates an embedding deployment for the Azure OpenAI service.') +resource azureOpenAIEmbeddingDeployment 'Microsoft.CognitiveServices/accounts/deployments@2023-05-01' = { + name: 'embedding' + parent: azureOpenAIService + sku: { + name: 'Standard' + capacity: 30 + } + properties: { + model: { + name: 'text-embedding-ada-002' + version: '2' + format: 'OpenAI' + } + } +} + +@description('Creates an Azure AI Language service account.') +resource languageService 'Microsoft.CognitiveServices/accounts@2023-05-01' = { + name: languageServiceName + location: location + kind: 'TextAnalytics' + sku: { + name: 'S' + } + properties: { + customSubDomainName: languageServiceName + publicNetworkAccess: 'Enabled' + restore: restore + } +} + +@description('Creates an Azure AI Translator service account.') +resource translatorService 'Microsoft.CognitiveServices/accounts@2023-05-01' = { + name: translatorServiceName + location: location + kind: 'TextTranslation' + sku: { + name: 'S1' + } + properties: { + customSubDomainName: translatorServiceName + publicNetworkAccess: 'Enabled' + restore: restore + } +} + +@description('Creates a Storage Account for AI Foundry Hub.') +resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { + name: storageAccountName + location: location + sku: { + name: 'Standard_LRS' + } + kind: 'StorageV2' + properties: { + accessTier: 'Hot' + allowBlobPublicAccess: false + minimumTlsVersion: 'TLS1_2' + } +} + +@description('Creates an Azure AI Foundry Hub.') +resource aiHub 'Microsoft.MachineLearningServices/workspaces@2024-04-01' = { + name: aiHubName + location: location + kind: 'Hub' + identity: { + type: 'SystemAssigned' + } + sku: { + name: 'Basic' + tier: 'Basic' + } + properties: { + friendlyName: aiHubName + storageAccount: storageAccount.id + publicNetworkAccess: 'Enabled' + } +} + +@description('Creates an Azure AI Foundry Project.') +resource aiProject 'Microsoft.MachineLearningServices/workspaces@2024-04-01' = { + name: aiProjectName + location: location + kind: 'Project' + identity: { + type: 'SystemAssigned' + } + sku: { + name: 'Basic' + tier: 'Basic' + } + properties: { + friendlyName: aiProjectName + hubResourceId: aiHub.id + publicNetworkAccess: 'Enabled' + } + dependsOn: [ + aiHub + ] +} + +output serverFqdn string = postgreSQLFlexibleServer.properties.fullyQualifiedDomainName +output serverName string = postgreSQLFlexibleServer.name +output databaseName string = rentalsDatabase.name + +output azureOpenAIServiceName string = azureOpenAIService.name +output azureOpenAIEndpoint string = azureOpenAIService.properties.endpoint +output azureOpenAIEmbeddingDeploymentName string = azureOpenAIEmbeddingDeployment.name + +output languageServiceName string = languageService.name +output languageServiceEndpoint string = languageService.properties.endpoint + +output translatorServiceName string = translatorService.name +output translatorServiceEndpoint string = translatorService.properties.endpoint + +output storageAccountName string = storageAccount.name +output storageAccountId string = storageAccount.id + +output aiHubName string = aiHub.name +output aiHubId string = aiHub.id + +output aiProjectName string = aiProject.name +output aiProjectId string = aiProject.id diff --git a/Allfiles/Labs/Shared/deploy-all.bicep b/Allfiles/Labs/Shared/deploy-all.bicep new file mode 100644 index 0000000..223794a --- /dev/null +++ b/Allfiles/Labs/Shared/deploy-all.bicep @@ -0,0 +1,183 @@ +@description('Location for all resources.') +param location string = resourceGroup().location + +@description('Unique name for the Azure Database for PostgreSQL.') +param serverName string = 'psql-learn-${resourceGroup().location}-${uniqueString(resourceGroup().id)}' + +@description('The version of PostgreSQL to use.') +param postgresVersion string = '16' + +@description('Login name of the database administrator.') +@minLength(1) +param adminLogin string = 'pgAdmin' + +@description('Password for the database administrator.') +@minLength(8) +@secure() +param adminLoginPassword string + +@description('Name of the database.') +@minLength(1) +param databaseName string = 'rentals' + +@description('Unique name for the Azure OpenAI service.') +param azureOpenAIServiceName string = 'oai-learn-${resourceGroup().location}-${uniqueString(resourceGroup().id)}' + +@description('Unique name for the Azure AI Language service account.') +param languageServiceName string = 'lang-learn-${resourceGroup().location}-${uniqueString(resourceGroup().id)}' +@description('Unique name for the Azure AI Translator service account.') +param translatorServiceName string = 'trn-learn-${resourceGroup().location}-${uniqueString(resourceGroup().id)}' + + +@description('Restore the service instead of creating a new instance. This is useful if you previously soft-delted the service and want to restore it. If you are restoring a service, set this to true. Otherwise, leave this as false.') +param restore bool = false + +@description('Creates a PostgreSQL Flexible Server.') +resource postgreSQLFlexibleServer 'Microsoft.DBforPostgreSQL/flexibleServers@2023-03-01-preview' = { + name: serverName + location: location + sku: { + name: 'Standard_D2ds_v4' + tier: 'GeneralPurpose' + } + properties: { + administratorLogin: adminLogin + administratorLoginPassword: adminLoginPassword + authConfig: { + activeDirectoryAuth: 'Disabled' + passwordAuth: 'Enabled' + tenantId: subscription().tenantId + } + backup: { + backupRetentionDays: 7 + geoRedundantBackup: 'Disabled' + } + createMode: 'Default' + highAvailability: { + mode: 'Disabled' + } + storage: { + autoGrow: 'Disabled' + storageSizeGB: 32 + tier: 'P10' + } + version: postgresVersion + } +} + +@description('Firewall rule that checks the "Allow public access from any Azure service within Azure to this server" box.') +resource allowAllAzureServicesAndResourcesWithinAzureIps 'Microsoft.DBforPostgreSQL/flexibleServers/firewallRules@2023-03-01-preview' = { + name: 'AllowAllAzureServicesAndResourcesWithinAzureIps' + parent: postgreSQLFlexibleServer + properties: { + startIpAddress: '0.0.0.0' + endIpAddress: '0.0.0.0' + } +} + +@description('Firewall rule to allow all IP addresses to connect to the server. Should only be used for lab purposes.') +resource allowAll 'Microsoft.DBforPostgreSQL/flexibleServers/firewallRules@2023-03-01-preview' = { + name: 'AllowAll' + parent: postgreSQLFlexibleServer + properties: { + startIpAddress: '0.0.0.0' + endIpAddress: '255.255.255.255' + } +} + +@description('Creates the "rentals" database in the PostgreSQL Flexible Server.') +resource rentalsDatabase 'Microsoft.DBforPostgreSQL/flexibleServers/databases@2023-03-01-preview' = { + name: databaseName + parent: postgreSQLFlexibleServer + properties: { + charset: 'UTF8' + collation: 'en_US.UTF8' + } +} + +@description('Configures the "azure.extensions" parameter to allowlist extensions.') +resource allowlistExtensions 'Microsoft.DBforPostgreSQL/flexibleServers/configurations@2023-03-01-preview' = { + name: 'azure.extensions' + parent: postgreSQLFlexibleServer + dependsOn: [allowAllAzureServicesAndResourcesWithinAzureIps, allowAll, rentalsDatabase] // Ensure the database is created and configured before setting the parameter, as it requires a "restart." + properties: { + source: 'user-override' + value: 'azure_ai,vector' + } +} + +@description('Creates an Azure OpenAI service.') +resource azureOpenAIService 'Microsoft.CognitiveServices/accounts@2023-05-01' = { + name: azureOpenAIServiceName + location: location + kind: 'OpenAI' + sku: { + name: 'S0' + tier: 'Standard' + } + properties: { + customSubDomainName: azureOpenAIServiceName + publicNetworkAccess: 'Enabled' + restore: restore + } +} + +@description('Creates an embedding deployment for the Azure OpenAI service.') +resource azureOpenAIEmbeddingDeployment 'Microsoft.CognitiveServices/accounts/deployments@2023-05-01' = { + name: 'embedding' + parent: azureOpenAIService + sku: { + name: 'Standard' + capacity: 30 + } + properties: { + model: { + name: 'text-embedding-ada-002' + version: '2' + format: 'OpenAI' + } + } +} + +@description('Creates an Azure AI Language service account.') +resource languageService 'Microsoft.CognitiveServices/accounts@2023-05-01' = { + name: languageServiceName + location: location + kind: 'TextAnalytics' + sku: { + name: 'S' + } + properties: { + customSubDomainName: languageServiceName + publicNetworkAccess: 'Enabled' + restore: restore + } +} + +output serverFqdn string = postgreSQLFlexibleServer.properties.fullyQualifiedDomainName +output serverName string = postgreSQLFlexibleServer.name +output databaseName string = rentalsDatabase.name + +output azureOpenAIServiceName string = azureOpenAIService.name +output azureOpenAIEndpoint string = azureOpenAIService.properties.endpoint +output azureOpenAIEmbeddingDeploymentName string = azureOpenAIEmbeddingDeployment.name + +output languageServiceName string = languageService.name +output languageServiceEndpoint string = languageService.properties.endpoint + +@description('Creates an Azure AI Translator service account.') +resource translatorService 'Microsoft.CognitiveServices/accounts@2023-05-01' = { + name: translatorServiceName + location: location + kind: 'TextTranslation' + sku: { + name: 'S1' + } + properties: { + customSubDomainName: translatorServiceName + publicNetworkAccess: 'Enabled' + restore: restore + } +} +output translatorServiceName string = translatorService.name +output translatorServiceEndpoint string = translatorService.properties.endpoint diff --git a/Allfiles/Labs/Shared/deploy.aoai-deployments.bicep b/Allfiles/Labs/Shared/deploy.aoai-deployments.bicep new file mode 100644 index 0000000..fc0e713 --- /dev/null +++ b/Allfiles/Labs/Shared/deploy.aoai-deployments.bicep @@ -0,0 +1,62 @@ +@description('Name of the existing Azure OpenAI account.') +param azureOpenAIServiceName string + +@description('Embedding model to deploy.') +param embeddingModelName string = 'text-embedding-ada-002' + +@description('Embedding model version.') +param embeddingModelVersion string = '2' + +@description('Capacity for embedding deployment (keep small for labs).') +param embeddingCapacity int = 1 + +@description('Chat model to deploy.') +param chatModelName string = 'gpt-4o-mini' + +@description('Chat model version.') +param chatModelVersion string = '2024-07-18' + +@description('Capacity for chat deployment (keep small for labs).') +param chatCapacity int = 1 + +// Reference the existing AOAI account (the parent) +resource aoai 'Microsoft.CognitiveServices/accounts@2023-05-01' existing = { + name: azureOpenAIServiceName +} + +// Child deployment: Embeddings (uses the parent property) +resource embedding 'Microsoft.CognitiveServices/accounts/deployments@2023-05-01' = { + name: 'embedding' + parent: aoai + sku: { + name: 'Standard' + capacity: embeddingCapacity + } + properties: { + model: { + name: embeddingModelName + version: embeddingModelVersion + format: 'OpenAI' + } + } +} + +// Child deployment: Chat (uses the parent property) +resource chat 'Microsoft.CognitiveServices/accounts/deployments@2023-05-01' = { + name: 'chat' + parent: aoai + sku: { + name: 'Standard' + capacity: chatCapacity + } + properties: { + model: { + name: chatModelName + version: chatModelVersion + format: 'OpenAI' + } + } +} + +output azureOpenAIEmbeddingDeploymentName string = embedding.name +output azureOpenAIChatDeploymentName string = chat.name diff --git a/Allfiles/Labs/Shared/deploy.bicep b/Allfiles/Labs/Shared/deploy.bicep index 17fbf8f..6dabb19 100644 --- a/Allfiles/Labs/Shared/deploy.bicep +++ b/Allfiles/Labs/Shared/deploy.bicep @@ -16,6 +16,10 @@ param adminLogin string = 'pgAdmin' @secure() param adminLoginPassword string +@description('Name of the database.') +@minLength(1) +param databaseName string = 'rentals' + @description('Unique name for the Azure OpenAI service.') param azureOpenAIServiceName string = 'oai-learn-${resourceGroup().location}-${uniqueString(resourceGroup().id)}' @@ -80,7 +84,7 @@ resource allowAll 'Microsoft.DBforPostgreSQL/flexibleServers/firewallRules@2023- @description('Creates the "rentals" database in the PostgreSQL Flexible Server.') resource rentalsDatabase 'Microsoft.DBforPostgreSQL/flexibleServers/databases@2023-03-01-preview' = { - name: 'rentals' + name: databaseName parent: postgreSQLFlexibleServer properties: { charset: 'UTF8' diff --git a/Allfiles/Labs/Shared/deploy.core.bicep b/Allfiles/Labs/Shared/deploy.core.bicep new file mode 100644 index 0000000..eb39429 --- /dev/null +++ b/Allfiles/Labs/Shared/deploy.core.bicep @@ -0,0 +1,158 @@ +@description('Location for all resources.') +param location string = resourceGroup().location + +@description('Unique name for the Azure Database for PostgreSQL.') +param serverName string = 'psql-learn-${location}-${uniqueString(resourceGroup().id)}' + +@description('PostgreSQL major version.') +param postgresVersion string = '16' + +@description('Login name of the database administrator.') +@minLength(1) +param adminLogin string = 'pgAdmin' + +@description('Password for the database administrator.') +@minLength(8) +@secure() +param adminLoginPassword string + +@description('Name of the database.') +@minLength(1) +param databaseName string = 'rentals' + +@description('Unique name for the Azure OpenAI service.') +param azureOpenAIServiceName string = 'oai-learn-${location}-${uniqueString(resourceGroup().id)}' + +@description('Unique name for the Azure AI Language service account.') +param languageServiceName string = 'lang-learn-${location}-${uniqueString(resourceGroup().id)}' + +@description('Restore soft-deleted resources instead of creating new ones.') +param restore bool = false + +// ------------------------- +// PostgreSQL Flexible Server +// ------------------------- +resource postgreSQLFlexibleServer 'Microsoft.DBforPostgreSQL/flexibleServers@2023-03-01-preview' = { + name: serverName + location: location + sku: { + name: 'Standard_D2ds_v4' + tier: 'GeneralPurpose' + } + properties: { + administratorLogin: adminLogin + administratorLoginPassword: adminLoginPassword + authConfig: { + activeDirectoryAuth: 'Disabled' + passwordAuth: 'Enabled' + tenantId: subscription().tenantId + } + backup: { + backupRetentionDays: 7 + geoRedundantBackup: 'Disabled' + } + createMode: 'Default' + highAvailability: { + mode: 'Disabled' + } + storage: { + autoGrow: 'Disabled' + storageSizeGB: 32 + tier: 'P10' + } + version: postgresVersion + } +} + +@description('Allow public access from any Azure service within Azure to this server.') +resource allowAllAzureServicesAndResourcesWithinAzureIps 'Microsoft.DBforPostgreSQL/flexibleServers/firewallRules@2023-03-01-preview' = { + name: 'AllowAllAzureServicesAndResourcesWithinAzureIps' + parent: postgreSQLFlexibleServer + properties: { + startIpAddress: '0.0.0.0' + endIpAddress: '0.0.0.0' + } +} + +@description('Allow all IP addresses (lab use only).') +resource allowAll 'Microsoft.DBforPostgreSQL/flexibleServers/firewallRules@2023-03-01-preview' = { + name: 'AllowAll' + parent: postgreSQLFlexibleServer + properties: { + startIpAddress: '0.0.0.0' + endIpAddress: '255.255.255.255' + } +} + +@description('Create the database.') +resource db 'Microsoft.DBforPostgreSQL/flexibleServers/databases@2023-03-01-preview' = { + name: databaseName + parent: postgreSQLFlexibleServer + properties: { + charset: 'UTF8' + collation: 'en_US.UTF8' + } +} + +// Allow-list extensions at the server level +resource allowlistExtensions 'Microsoft.DBforPostgreSQL/flexibleServers/configurations@2023-03-01-preview' = { + name: 'azure.extensions' + parent: postgreSQLFlexibleServer + dependsOn: [ + allowAllAzureServicesAndResourcesWithinAzureIps + allowAll + db + ] + properties: { + source: 'user-override' + value: 'azure_ai,vector' + } +} + +// ------------------------- +// Cognitive Services: OpenAI +// ------------------------- +resource azureOpenAIService 'Microsoft.CognitiveServices/accounts@2023-05-01' = { + name: azureOpenAIServiceName + location: location + kind: 'OpenAI' + sku: { + name: 'S0' + tier: 'Standard' + } + properties: { + customSubDomainName: azureOpenAIServiceName + publicNetworkAccess: 'Enabled' + restore: restore + } +} + +// ------------------------- +// Cognitive Services: Language +// ------------------------- +resource languageService 'Microsoft.CognitiveServices/accounts@2023-05-01' = { + name: languageServiceName + location: location + kind: 'TextAnalytics' + sku: { + name: 'S' + } + properties: { + customSubDomainName: languageServiceName + publicNetworkAccess: 'Enabled' + restore: restore + } +} + +// ------------------------- +// Outputs +// ------------------------- +output serverFqdn string = postgreSQLFlexibleServer.properties.fullyQualifiedDomainName +output serverName string = postgreSQLFlexibleServer.name +output databaseName string = db.name + +output azureOpenAIServiceName string = azureOpenAIService.name +output azureOpenAIEndpoint string = azureOpenAIService.properties.endpoint + +output languageServiceName string = languageService.name +output languageServiceEndpoint string = languageService.properties.endpoint diff --git a/Instructions/Labs/12-explore-azure-ai-extension.md b/Instructions/Labs/12-explore-azure-ai-extension.md index a45a221..c83f7df 100644 --- a/Instructions/Labs/12-explore-azure-ai-extension.md +++ b/Instructions/Labs/12-explore-azure-ai-extension.md @@ -447,6 +447,65 @@ The `azure_ml` schema lets functions connect to Azure ML services directly from By providing an endpoint and key, you can connect to an Azure ML deployed endpoint like you connected to your Azure OpenAI and Azure AI Services endpoints. Interacting with Azure ML requires having a trained and deployed model, so it is out of scope for this exercise, and you are not setting up that connection to try it out here. +### Explore semantic operators + +The `azure_ai` extension includes a small set of semantic operators that let you work with generative AI models directly from SQL. These operators help you generate content, evaluate statements, extract information, and rank documents. Each operator uses the model settings you configured earlier in `azure_ai.settings`. + +Start by reviewing the available operators: + +- `azure_ai.generate` – generates text and can return structured JSON when a schema is supplied. +- `azure_ai.is_true` – evaluates whether a statement is likely to be true. +- `azure_ai.extract` – pulls specific fields or values from unstructured text. +- `azure_ai.rank` – returns documents ordered by relevance to a query. + +1. Run the following query to generate text using `azure_ai.generate`. This example summarizes a listing description: + + ```sql + SELECT azure_ai.generate( + prompt => 'Summarize this listing: ' || description + ) + FROM listings + LIMIT 1; + ``` + +1. Next, use `azure_ai.is_true` to evaluate whether a review expresses a particular claim: + + ```sql + SELECT + id, + comments, + azure_ai.is_true( + 'This review is positive: ' || comments + ) AS is_positive + FROM reviews + LIMIT 3; + ``` + +1. Use `azure_ai.extract` to pull structured details out of free-form text. In this example, extract information about location and amenities: + + ```sql + SELECT azure_ai.extract( + description, + ARRAY['location', 'amenities'] + ) + FROM listings + LIMIT 1; + ``` + +1. Finally, try the `azure_ai.rank` operator. Pass a query and an array of listing descriptions to see how the operator ranks them for relevance: + + ```sql + SELECT * + FROM azure_ai.rank( + 'quiet neighborhood apartment', + ARRAY( + SELECT description FROM listings LIMIT 5 + ) + ); + ``` + +Each operator returns model-generated output that you can use directly in queries, views, and application logic. These operators give you a way to integrate generative AI behaviors into your database without leaving SQL. + ## Clean up Once you have completed this exercise, delete the Azure resources you created. You are charged for the configured capacity, not how much the database is used. Follow these instructions to delete your resource group and all resources you created for this lab. diff --git a/Instructions/Labs/14-exercise-build-rag-application-postgresql-python.md b/Instructions/Labs/14-exercise-build-rag-application-postgresql-python.md new file mode 100644 index 0000000..5f5c59e --- /dev/null +++ b/Instructions/Labs/14-exercise-build-rag-application-postgresql-python.md @@ -0,0 +1,539 @@ +In this scenario, you’re building a small internal assistant for the company’s policy questions at Contoso. You set up a table in Azure Database for PostgreSQL, load the CSV of policies, and store an embedding for each policy so the database can match questions by meaning, not just keywords. You add a vector index to keep lookups fast. Then you write a short Python script that asks for a question, fetches the most relevant policies, and prints an answer based only on those policies, including the policy title. + +By the end of this exercise, you will: + +- Enable database extensions that power embeddings and vector search. +- Generate in-database embeddings for your data. +- Add a vector index to keep search fast. +- Write a small RAG Python program that retrieves top chunks and produces a grounded answer. + +## Before you start + +You need an [Azure subscription](https://azure.microsoft.com/free) with administrative rights, and you must be approved for Azure OpenAI access in that subscription. If you need Azure OpenAI access, apply at the [Azure OpenAI limited access](https://learn.microsoft.com/legal/cognitive-services/openai/limited-access) page. + +### Deploy resources into your Azure subscription + +*If you already have a nonproduction Azure Database for PostgreSQL server and a nonproduction Azure OpenAI resource setup, you can skip this section.* + +This step guides you through using Azure CLI commands from the Azure Cloud Shell to create a resource group and run a Bicep script to deploy the Azure services necessary for completing this exercise into your Azure subscription. + +1. Open a web browser and navigate to the [Azure portal](https://portal.azure.com/). + +1. Select the **Cloud Shell** icon in the Azure portal toolbar to open a new [Cloud Shell](https://learn.microsoft.com/azure/cloud-shell/overview) pane at the bottom of your browser window. + + ![Screenshot of the Azure toolbar with the Cloud Shell icon highlighted by a red box.](media/14-portal-toolbar-cloud-shell.png) + + If prompted, select the required options to open a *Bash* shell. If you previously used a *PowerShell* console, switch it to a *Bash* shell. + +1. At the Cloud Shell prompt, enter the following to clone the GitHub repo containing exercise resources: + + ```bash + git clone --branch "postgresql-ai-update" --single-branch --depth 1 https://github.com/MicrosoftLearning/mslearn-postgresql.git + ``` + +1. Next, you run three commands to define variables to reduce redundant typing when using Azure CLI commands to create Azure resources. The variables represent the name to assign to your resource group (`RG_NAME`), the Azure region (`REGION`) into which resources are deployed, and a randomly generated password for the PostgreSQL administrator sign in (`ADMIN_PASSWORD`). + + In the first command, the region assigned to the corresponding variable is `westus3`, but you can also replace it with a location of your preference. However, if replacing the default, you must select another [Azure region that supports abstractive summarization](https://learn.microsoft.com/azure/ai-services/language-service/summarization/region-support) to ensure you can complete all of the tasks in the modules in this learning path. + + ```bash + REGION=westus3 + ``` + + The following command assigns the name to be used for the resource group that houses all the resources used in this exercise. The resource group name assigned to the corresponding variable is `rg-learn-postgresql-ai-$REGION`, where `$REGION` is the location you previously specified. However, you can change it to any other resource group name that suits your preference. + + ```bash + RG_NAME=rg-learn-postgresql-ai-$REGION + ``` + + The final command randomly generates a password for the PostgreSQL admin sign in. **Make sure you copy it** to a safe place to use later to connect to your PostgreSQL. + + ```bash + a=() + for i in {a..z} {A..Z} {0..9}; + do + a[$RANDOM]=$i + done + ADMIN_PASSWORD=$(IFS=; echo "${a[*]::18}") + echo "Your randomly generated PostgreSQL admin user's password is:" + echo $ADMIN_PASSWORD + ``` + +1. *Only run this command if you want to change your current subscription*. If you have access to more than one Azure subscription, and your default subscription isn't the one in which you want to create the resource group and other resources for this exercise, run this command to set the appropriate subscription, replacing the `` token with either the name or ID of the subscription you want to use: + + ```azurecli + az account set --subscription + ``` + +1. Run the following Azure CLI command to create your resource group: + + ```azurecli + az group create --name $RG_NAME --location $REGION + ``` + +1. Finally, use the Azure CLI to execute Bicep deployment scripts to provision Azure resources in your resource group: + + ```azurecli + #1 Core infra: PostgreSQL + DB + firewall + server param, AOAI account, Language account + az deployment group create \ + --resource-group "$RG_NAME" \ + --template-file "mslearn-postgresql/Allfiles/Labs/Shared/deploy.core.bicep" \ + --parameters restore=false adminLogin=pgAdmin adminLoginPassword="$ADMIN_PASSWORD" databaseName=ContosoHelpDesk + + AOAI=$(az cognitiveservices account list -g "$RG_NAME" --query "[?kind=='OpenAI'].name | [0]" -o tsv) + + #2 Wait for the parent AOAI account to finish provisioning + echo "Waiting for AOAI account to be ready..." + while true; do + STATE=$(az cognitiveservices account show -g "$RG_NAME" -n "$AOAI" --query "properties.provisioningState" -o tsv) + echo "provisioningState=$STATE" + [ "$STATE" = "Succeeded" ] && break + sleep 10 + done + + #3 OpenAI deployments: embedding + chat + az deployment group create \ + --resource-group "$RG_NAME" \ + --template-file "mslearn-postgresql/Allfiles/Labs/Shared/deploy.aoai-deployments.bicep" \ + --parameters azureOpenAIServiceName="$AOAI" + ``` + + The Bicep deployment scripts provisions the Azure services required to complete this exercise into your resource group. The resources deployed include an Azure Database for PostgreSQL server, Azure OpenAI, an Azure AI Language service. The Bicep script also performs some configuration steps, such as adding the `azure_ai` and `vector` extensions to the PostgreSQL server's _allowlist_ (via the `azure.extensions` server parameter), creating a database named `ContosoHelpDesk` on the server, and adding a deployment named `embedding` using the `text-embedding-ada-002` model to your Azure OpenAI service. Finally it adds a deployment named `chat` using the `gpt-4o-mini` model to your Azure OpenAI service. The Bicep file shares all modules in this learning path, so you might only use some of the deployed resources in some exercises. + + The deployment typically takes several minutes to complete. You can monitor it from the Cloud Shell or navigate to the **Deployments** page for the resource group you previously created and observe the deployment progress there. + +1. Take note of the resource names and their corresponding ID, and the PostgreSQL server's fully qualified domain name (FQDN), username, and password, as you need them later. + +### Troubleshooting deployment errors + +You could encounter a few errors when running the Bicep deployment script. *If no errors are encountered, skip this section.* + +- If you previously ran the Bicep deployment script for this learning path and later deleted the resources, you could receive an error message like the following if you're attempting to rerun the script within 48 hours of deleting the resources: + + ```bash + {"code": "InvalidTemplateDeployment", "message": "The template deployment 'deploy' is not valid according to the validation procedure. The tracking id is '4e87a33d-a0ac-4aec-88d8-177b04c1d752'. See inner errors for details."} + + Inner Errors: + {"code": "FlagMustBeSetForRestore", "message": "An existing resource with ID '/subscriptions/{subscriptionId}/resourceGroups/rg-learn-postgresql-ai-eastus/providers/Microsoft.CognitiveServices/accounts/{accountName}' has been soft-deleted. To restore the resource, you must specify 'restore' to be 'true' in the property. If you don't want to restore existing resource, please purge it first."} + ``` + + If you receive this message, modify the `azure deployment group create` command previously to set the `restore` parameter equal to `true` and rerun it. + +- If the selected region is restricted from provisioning specific resources, you must set the `REGION` variable to a different location and rerun the commands to create the resource group and run the Bicep deployment script. + + ```bash + {"status":"Failed","error":{"code":"DeploymentFailed","target":"/subscriptions/{subscriptionId}/resourceGroups/{resourceGrouName}/providers/Microsoft.Resources/deployments/{deploymentName}","message":"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.","details":[{"code":"ResourceDeploymentFailure","target":"/subscriptions/{subscriptionId}/resourceGroups/{resourceGrouName}/providers/Microsoft.DBforPostgreSQL/flexibleServers/{serverName}","message":"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.","details":[{"code":"RegionIsOfferRestricted","message":"Subscriptions are restricted from provisioning in this region. Please choose a different region. For exceptions to this rule please open a support request with Issue type of 'Service and subscription limits'. See https://review.learn.microsoft.com/en-us/azure/postgresql/flexible-server/how-to-request-quota-increase for more details."}]}]}} + ``` + +- If the script is unable to create an AI resource due to the requirement to accept the responsible AI agreement, you get the following error. If you get that error, use the Azure portal user interface to create an Azure AI Services resource, and then rerun the deployment script. + + ```bash + {"code": "InvalidTemplateDeployment", "message": "The template deployment 'deploy' is not valid according to the validation procedure. The tracking id is 'f8412edb-6386-4192-a22f-43557a51ea5f'. See inner errors for details."} + + Inner Errors: + {"code": "ResourceKindRequireAcceptTerms", "message": "This subscription cannot create TextAnalytics until you agree to Responsible AI terms for this resource. You can agree to Responsible AI terms by creating a resource through the Azure Portal then trying again. For more detail go to https://go.microsoft.com/fwlink/?linkid=2164190"} + ``` + +## Connect to your database using psql in the Azure Cloud Shell + +You connect to the `ContosoHelpDesk` database on your Azure Database for PostgreSQL server using the [psql command-line utility](https://www.postgresql.org/docs/current/app-psql.html) from the [Azure Cloud Shell](https://learn.microsoft.com/azure/cloud-shell/overview). + +1. In the [Azure portal](https://portal.azure.com/), navigate to your newly created Azure Database for PostgreSQL server. + +1. In the resource menu, under **Settings**, select **Databases** select **Connect** for the `ContosoHelpDesk` database. Selecting **Connect** doesn't actually connect you to the database; it simply provides instructions for connecting to the database using various methods. Review the instructions to **Connect from browser or locally** and use those instructions to connect using the Azure Cloud Shell. + + ![Screenshot of the Azure Database for PostgreSQL Databases page. Databases and Connect for the ContosoHelpDesk database are highlighted by red boxes.](media/14-postgresql-database-connect.png) + +1. At the "Password for user pgAdmin" prompt in the Cloud Shell, enter the randomly generated password for the **pgAdmin** sign in. + + Once you sign in, the `psql` prompt for the `ContosoHelpDesk` database is displayed. + +1. Throughout the remainder of this exercise, you continue working in the Cloud Shell, so it helps to expand the pane within your browser window by selecting the **Maximize** button at the top right of the pane. + + ![Screenshot of the Azure Cloud Shell pane with the Maximize button highlighted by a red box.](media/14-azure-cloud-shell-pane-maximize.png) + +## Setup: Configure extensions + +To store and query vectors, and to generate embeddings, you need to allowlist and enable two extensions for Azure Database for PostgreSQL: `vector` and `azure_ai`. + +1. To allowlist both extensions, add `vector` and `azure_ai` to the server parameter `azure.extensions`, as per the instructions provided in [How to use PostgreSQL extensions](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-extensions#how-to-use-postgresql-extensions). + +1. Run the following SQL command to enable the `vector` and `azure_ai` extensions. For detailed instructions, read [How to enable and use `pgvector` on Azure Database for PostgreSQL](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/how-to-use-pgvector#enable-extension). + + On *ContosoHelpDesk* prompt, run the following SQL commands: + + ```sql + -- Enable required extensions + CREATE EXTENSION vector; + CREATE EXTENSION azure_ai; + ``` + +1. To enable the `azure_ai` extension, run the following SQL command. You need the endpoint and API key for the Azure OpenAI resource. For detailed instructions, read [Enable the `azure_ai` extension](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/generative-ai-azure-overview#enable-the-azure_ai-extension). + + On the *ContosoHelpDesk* prompt, run the following commands: + + ```sql + -- Configure Azure OpenAI (requires azure_ai_settings_manager role) + SELECT azure_ai.set_setting('azure_openai.endpoint', 'https://.openai.azure.com'); -- e.g., https://YOUR-RESOURCE.openai.azure.com + SELECT azure_ai.set_setting('azure_openai.subscription_key', ''); + ``` + +## Populate the database with sample data + +Before you use the `azure_ai` extension, add a table to the `ContosoHelpDesk` database and populate them with sample data so you have information to work with as you create your application. + +1. On the **ContosoHelpDesk** prompt, run the following commands to create the `company_policies` table for storing company policy data: + + ```sql + -- Create table for policies and embeddings (matches CSV columns) + DROP TABLE IF EXISTS company_policies CASCADE; + + CREATE TABLE company_policies ( + policy_id BIGSERIAL PRIMARY KEY, + title TEXT NOT NULL, + department TEXT NOT NULL, + policy_text TEXT NOT NULL, + category TEXT NOT NULL, + embedding vector(1536) -- The `text-embedding-ada-002` model is configured to return 1,536 dimensions, so use that number for the vector column size. + ); + ``` + +1. In your Azure Cloud Shell, use the `COPY` command to load data from CSV files into each table you previously created. Run the following command to populate the `company_policies` table: + + ```sql + \COPY company_policies (title, department, policy_text, category) FROM 'mslearn-postgresql/Allfiles/Labs/Shared/company-policies.csv' WITH (FORMAT csv, HEADER) + ``` + + The command output should be `COPY 108`, indicating that 108 rows were written into the table from the CSV file. + +1. Backfill embeddings for existing rows. + + Run the following command in your **psql** session (Cloud Shell) to compute embeddings for any rows that don’t have them yet. Replace `` with the name of your embedding deployment. + + ```sql + -- Create embeddings for existing rows that currently have no embeddings + UPDATE company_policies + SET embedding = azure_openai.create_embeddings('', policy_text)::vector + WHERE embedding IS NULL; + ``` + + This calls your Azure OpenAI embedding deployment from SQL (via `azure_ai`) and stores the result in the column `embedding`. + +If you successfully backfilled the 108 rows with embeddings, exit *psql* by typing `\q` and skip the following troubleshooting section. Otherwise, continue with the following troubleshooting steps. + +### Troubleshoot 429 errors if encountered + +*Skip this section if your UPDATE statement successfully backfilled 108 embeddings*. + +1. Depending on your Azure OpenAI rate limits, you might experience **429 Too Many Requests** errors if you exceed the allowed number of requests. If that is the case for the previous UPDATE statement, you can run the following command to batch the requests and retry (if needed manually reduce the *batch_size* too): + + ```sql + DO $$ + DECLARE + batch_size int := 50; -- rows per batch + optimistic_pause int := 10; -- seconds to wait after a successful batch + pause_secs int := 10; -- current wait (resets to optimistic on success) + max_pause int := 60; -- cap the backoff + updated int; + BEGIN + LOOP + BEGIN + WITH todo AS ( + SELECT policy_id, policy_text + FROM company_policies + WHERE embedding IS NULL + ORDER BY policy_id + LIMIT batch_size + ) + UPDATE company_policies p + SET embedding = azure_openai.create_embeddings('embedding', t.policy_text)::vector + FROM todo t + WHERE p.policy_id = t.policy_id; + + GET DIAGNOSTICS updated = ROW_COUNT; + + IF updated = 0 THEN + RAISE NOTICE 'No rows left to embed.'; + EXIT; + END IF; + + -- Success: reset to optimistic pause and sleep briefly + pause_secs := optimistic_pause; + RAISE NOTICE 'Updated % rows; sleeping % seconds before next batch.', updated, pause_secs; + PERFORM pg_sleep(pause_secs); + + EXCEPTION WHEN OTHERS THEN + -- Likely throttled (429) or transient error: back off and retry + RAISE NOTICE 'Throttled/transient error; backing off % seconds.', pause_secs; + PERFORM pg_sleep(pause_secs); + pause_secs := LEAST(pause_secs * 2, max_pause); + END; + END LOOP; + END $$; + + ``` + +1. If you successfully backfilled the 108 rows with embeddings, exit *psql* by typing `\q`, otherwise, try reducing the *batch_size* by 10 and run the previous script again. + +### Test the vector table with a similarity query + +Let's make sure everything is working by verifying with a similarity search and simple filtering directly from SQL. + +1. On the Azure Cloud Shell, connect to the *ContosoHelpDesk* database using *psql* as before. + +1. Run the following SQL statement: + + ```sql + -- Best match for a question (cosine) + SELECT policy_id, title, department, policy_text + FROM company_policies + ORDER BY embedding <=> azure_openai.create_embeddings('embedding', + 'How many vacation days do employees get?')::vector + LIMIT 1; + ``` + +1. Add a filter plus a vector search by running the following SQL statement: + + ```sql + -- Filter + vector (hybrid) + SELECT policy_id, title, department, policy_text + FROM company_policies + WHERE department = 'HR' + ORDER BY embedding <=> azure_openai.create_embeddings('embedding', + 'Does the company help me with college expenses')::vector + LIMIT 3; + ``` + +1. Type *\q* and press Enter to exit *psql*. + +While these answers are a good start, they might not be comprehensive enough for more complex queries. To address this problem, you can create a Python RAG (Retrieval-Augmented Generation) application that retrieves relevant passages from our database and uses them as context for generating answers. + +## Create a Python RAG application to retrieve natural language answers + +Now that your embeddings are in place, you can write a short Python script that asks a question, fetches the most relevant policies from PostgreSQL, and prints an answer based only on those passages. + +### Update your environment variables + +Before you look at our Python application, you need to set the correct environment variables for PostgreSQL and Azure OpenAI. + +1. Open your `.env` file: + + ```bash + code "mslearn-postgresql/Allfiles/Labs/14/.env" + ``` + +1. Update your `.env` file with your PostgreSQL and Azure OpenAI credentials: + + ```text + # PostgreSQL connection + PGHOST= + PGUSER=pgAdmin + PGPASSWORD= + PGDATABASE=ContosoHelpDesk + + # Azure OpenAI + AZURE_OPENAI_API_KEY= + AZURE_OPENAI_ENDPOINT= # e.g., https://oai-learn--.openai.azure.com + OPENAI_API_VERSION=2024-02-15-preview + + # Deployment names (match the Bicep resources) + OPENAI_EMBED_DEPLOYMENT=embedding + OPENAI_CHAT_DEPLOYMENT=chat + ``` + +1. Save the file and close the *code* editor. + +> [!NOTE] +> If you can't find the save/exit options, on the *code* editor window, move your mouse to the upper right of the editor. Your icon should change, press your mouse button and you should see the options to save and close. + +### Update your Python RAG application + +On the GitHub repo you cloned, you can find the `app.py` file, which contains the shell for your RAG application. Time to implement the logic to retrieve and answer questions based on the context from PostgreSQL. + +1. Open the `CompanyPolicies.py` to add the RAG logic. + + ```bash + code "mslearn-postgresql/Allfiles/Labs/14/CompanyPolicies.py" + ``` + +1. Review the libraries the application depends on. The main library you use for interacting with Azure OpenAI is `langchain_openai`. + +1. our first function, `get_conn`, just creates a connection to the PostgreSQL database. This one is predefined for you. For the following three functions, replace the comments with actual code provided. + +1. Replace the comment **# Retrieve top-k rows by cosine similarity (embedding must be present)** with the following script: + + ```python + # Retrieve top-k rows by cosine similarity (embedding must be present) + def retrieve_chunks(question, top_k=5): + sql = """ + WITH q AS ( + SELECT azure_openai.create_embeddings(%s, %s)::vector AS qvec + ) + SELECT policy_id, title, policy_text + FROM company_policies, q + WHERE embedding IS NOT NULL + ORDER BY embedding <=> q.qvec + LIMIT %s; + """ + params = (os.getenv("OPENAI_EMBED_DEPLOYMENT"), question, top_k) + with get_conn() as conn, conn.cursor() as cur: + cur.execute(sql, params) + rows = cur.fetchall() + return [{"policy_id": r[0], "title": r[1], "text": r[2]} for r in rows] + ``` + + This function retrieves the top-k relevant chunks from the PostgreSQL database based on the user's question. + +1. Replace the comment **# Format retrieved chunks for the model prompt** with the following script: + + ```python + # Format retrieved chunks for the model prompt + def format_context(chunks): + return "\n\n".join([f"[{c['title']}] {c['text']}" for c in chunks]) + ``` + + This function formats the retrieved chunks into a context string suitable for the model prompt. + +1. Replace the comment **# Call Azure OpenAI to answer using the provided context** with the following script: + + ```python + # Call Azure OpenAI to answer using the provided context + def generate_answer(question, chunks): + llm = AzureChatOpenAI( + azure_deployment=os.getenv("OPENAI_CHAT_DEPLOYMENT"), + api_key=os.getenv("AZURE_OPENAI_API_KEY"), + azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"), + api_version=os.getenv("OPENAI_API_VERSION"), + temperature=0, + ) + messages = [ + {"role": "system", "content": "Answer ONLY from the provided context. If it isn't in the context, say you don’t have enough information. Cite policy titles in square brackets, e.g., [Vacation policy]."}, + {"role": "user", "content": f"Question: {question}\nContext:\n{format_context(chunks)}"}, + ] + return llm.invoke(messages).content + ``` + + This function generates an answer to the user's question using the provided context chunks. + +1. The final section of the application is the main application logic. This part of the code prompts the user for a question, retrieves relevant chunks, generates an answer, and loops until the user decides to quit. + +1. Save the file and close the *code* editor. + +## Run the application + +The last thing you need to do before running the application is to set up the Python environment and install the required packages. Finally, you run the application. + +```bash +# Navigate to the exercise folder +cd ~/mslearn-postgresql/Allfiles/Labs/14 + +# Set up the Python environment +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt + +# When prompted, enter a question (for example, How many vacation days do employees get?) +python CompanyPolicies.py +``` + +Try different questions to see how the model responds. + +- What is the company's policy on remote work? +- I need to visit some local customers, can I use the company car for that visit? +- We're expecting a new child, can I take some time off, and is it paid time off? +- What are the guidelines for employee conduct? + +Or come up with your own questions, maybe they're covered by the existing policies you added to the database. This small Python script is the basis of a RAG application. You search for relevant documents in the database and use them to answer user questions. But for an effective RAG application, you need to ensure that your document retrieval is fast and scalable. To achieve this fast retrieval, you can implement a vector index. + +## Add a vector index (speed at scale) + +Since your company_policies table was small, most likely your queries ran relatively fast. However, as the table grows, you should optimize for performance. The first step to improve query performance is to add a vector index. + +But adding an index to such a small table might not show significant improvements. So let's go ahead and emulate a larger table by adding 50,000 rows to the table. For this lab, to increase the size of the table, just copy the existing rows multiple times. + +1. On the Azure Cloud Shell, connect to the *ContosoHelpDesk* database using *psql* as before. + +1. Run the following SQL statement to insert more rows into the company_policies table: + + ```sql + -- Inflate to ~50k rows (keeps embeddings the same; OK for a demo) + INSERT INTO company_policies (title, department, policy_text, category, embedding) + SELECT title || ' (copy ' || gs || ')', department, policy_text, category, embedding + FROM company_policies + CROSS JOIN generate_series(1, 500) AS gs; + ``` + +Let's review the execution plan for our query with and without the index. + +### Run query without a vector index + +First, let's run the query without the index. + +1. On the Azure Cloud Shell, connect to the *ContosoHelpDesk* database using *psql* as before. + +1. Evaluate the execution plan for your query without the index by running the following SQL statement: + + ```sql + -- Disable pagination for better output readability + \pset pager off + ``` + + ```sql + EXPLAIN (ANALYZE, BUFFERS) + SELECT policy_id, title, department, policy_text + FROM company_policies + ORDER BY embedding <=> azure_openai.create_embeddings('embedding', + 'How many vacation days do employees get?')::vector + LIMIT 1; + ``` + +This query should return a detailed execution plan. Notice that because it doesn't use an index, the *Execution Time*, and *Buffers* metrics could indicate higher resource usage. If you run the query a second time, the query planner should use cached results, potentially improving performance. Take note of these metrics so you can compare them later. + +### Run query with a vector index + +Creating an IVFFlat index keeps top-k similarity fast as the table grows. Start simple and tune later. + +Let's go ahead and create the index. + +1. On the Azure Cloud Shell, connect to the *ContosoHelpDesk* database using *psql* as before. + +1. Create the IVFFlat index: + + ```sql + -- Drop the IVFFlat index + DROP INDEX IF EXISTS company_policies_embedding_ivfflat_idx; + + -- Use cosine distance (vector_cosine_ops) for text embeddings + CREATE INDEX company_policies_embedding_ivfflat_idx + ON company_policies + USING ivfflat (embedding vector_cosine_ops) + WITH (lists = 100); + + ANALYZE company_policies; + ``` + +1. Evaluate the execution plan for your query with the index by running the following SQL statement: + + ```sql + -- Disable pagination for better output readability + \pset pager off + ``` + + ```sql + EXPLAIN (ANALYZE, BUFFERS) + SELECT policy_id, title, department, policy_text + FROM company_policies + ORDER BY embedding <=> azure_openai.create_embeddings('embedding', + 'How many vacation days do employees get?')::vector + LIMIT 1; + ``` + +You notice several improvements in the execution plan, including reduced *Execution Time* and *Buffers* metrics. Additionally, you also notice that the query is now using the IVFFlat index. Even if you run the query multiple times, the performance should remain consistent. + +### Key takeaways + +By completing this exercise, you now know how to build a retrieval augmented application in Python. Your application retrieved the relevant documents and answered user queries intelligently. You scratched the surface of what's possible with RAG applications. With further enhancements, you can improve the accuracy and efficiency of your document retrieval and response generation. + +Additionally, you explored how to optimize query performance using vector indexes, which is crucial for scaling your application as the dataset grows. By using these indexes, you can ensure that your RAG application remains responsive and efficient, even as the volume of data increases. + +Finally, you learned about the importance of monitoring and fine-tuning your application over time. As user queries evolve and the dataset expands, you need to revisit your indexing strategy, prompt design, and overall architecture to maintain optimal performance and accuracy. + diff --git a/Instructions/Labs/14-exercise-implement-graph-rag.md b/Instructions/Labs/14-exercise-implement-graph-rag.md new file mode 100644 index 0000000..692e309 --- /dev/null +++ b/Instructions/Labs/14-exercise-implement-graph-rag.md @@ -0,0 +1,702 @@ +This hands-on exercise adds a lightweight knowledge-graph layer inside the same Azure Database for PostgreSQL you used in earlier units. You create two small tables for **nodes** and **edges**, link them to your existing `company_policies` rows, and then run a **graph-narrowed vector search** that first filters by relationships (Topic/Department) and then ranks with pgvector. The goal is *higher precision* on multi-concept questions without complicating prompts or moving data to a separate store. + +By the end of this exercise, you will: + +- Understand how to create and manage graph structures in PostgreSQL. +- Be able to perform graph-narrowed vector searches. +- Gain experience with integrating knowledge graphs into RAG applications. + +## Before you start + +You need an [Azure subscription](https://azure.microsoft.com/free) with administrative rights, and you must be approved for Azure OpenAI access in that subscription. If you need Azure OpenAI access, apply at the [Azure OpenAI limited access](https://learn.microsoft.com/legal/cognitive-services/openai/limited-access) page. + +### Deploy resources into your Azure subscription + +*If you already have a nonproduction Azure Database for PostgreSQL server and a nonproduction Azure OpenAI resource setup, you can skip this section.* + +This step guides you through using Azure CLI commands from the Azure Cloud Shell to create a resource group and run a Bicep script to deploy the Azure services necessary for completing this exercise into your Azure subscription. + +1. Open a web browser and navigate to the [Azure portal](https://portal.azure.com/). + +1. Select the **Cloud Shell** icon in the Azure portal toolbar to open a new [Cloud Shell](https://learn.microsoft.com/azure/cloud-shell/overview) pane at the bottom of your browser window. + + ![Screenshot of the Azure toolbar with the Cloud Shell icon highlighted by a red box.](media/14-portal-toolbar-cloud-shell.png) + + If prompted, select the required options to open a *Bash* shell. If you previously used a *PowerShell* console, switch it to a *Bash* shell. + +1. At the Cloud Shell prompt, enter the following to clone the GitHub repo containing exercise resources: + + ```bash + git clone --branch "postgresql-ai-update" --single-branch --depth 1 https://github.com/MicrosoftLearning/mslearn-postgresql.git + ``` + +1. Next, you run three commands to define variables to reduce redundant typing when using Azure CLI commands to create Azure resources. The variables represent the name to assign to your resource group (`RG_NAME`), the Azure region (`REGION`) into which resources are deployed, and a randomly generated password for the PostgreSQL administrator sign in (`ADMIN_PASSWORD`). + + In the first command, the region assigned to the corresponding variable is `westus3`, but you can also replace it with a location of your preference. However, if replacing the default, you must select another [Azure region that supports abstractive summarization](https://learn.microsoft.com/azure/ai-services/language-service/summarization/region-support) to ensure you can complete all of the tasks in the modules in this learning path. + + ```bash + REGION=westus3 + ``` + + The following command assigns the name to be used for the resource group that houses all the resources used in this exercise. The resource group name assigned to the corresponding variable is `rg-learn-postgresql-ai-$REGION`, where `$REGION` is the location you previously specified. However, you can change it to any other resource group name that suits your preference. + + ```bash + RG_NAME=rg-learn-postgresql-ai-$REGION + ``` + + The final command randomly generates a password for the PostgreSQL admin sign in. **Make sure you copy it** to a safe place to use later to connect to your PostgreSQL. + + ```bash + a=() + for i in {a..z} {A..Z} {0..9}; + do + a[$RANDOM]=$i + done + ADMIN_PASSWORD=$(IFS=; echo "${a[*]::18}") + echo "Your randomly generated PostgreSQL admin user's password is:" + echo $ADMIN_PASSWORD + ``` + +1. *Only run this command if you want to change your current subscription*. If you have access to more than one Azure subscription, and your default subscription isn't the one in which you want to create the resource group and other resources for this exercise, run this command to set the appropriate subscription, replacing the `` token with either the name or ID of the subscription you want to use: + + ```azurecli + az account set --subscription + ``` + +1. Run the following Azure CLI command to create your resource group: + + ```azurecli + az group create --name $RG_NAME --location $REGION + ``` + +1. Finally, use the Azure CLI to execute Bicep deployment scripts to provision Azure resources in your resource group: + + ```azurecli + #1 Core infra: PostgreSQL + DB + firewall + server param, AOAI account, Language account + az deployment group create \ + --resource-group "$RG_NAME" \ + --template-file "mslearn-postgresql/Allfiles/Labs/Shared/deploy.core.bicep" \ + --parameters restore=false adminLogin=pgAdmin adminLoginPassword="$ADMIN_PASSWORD" databaseName=ContosoHelpDesk + + AOAI=$(az cognitiveservices account list -g "$RG_NAME" --query "[?kind=='OpenAI'].name | [0]" -o tsv) + + #2 Wait for the parent AOAI account to finish provisioning + echo "Waiting for AOAI account to be ready..." + while true; do + STATE=$(az cognitiveservices account show -g "$RG_NAME" -n "$AOAI" --query "properties.provisioningState" -o tsv) + echo "provisioningState=$STATE" + [ "$STATE" = "Succeeded" ] && break + sleep 10 + done + + #3 OpenAI deployments: embedding + chat + az deployment group create \ + --resource-group "$RG_NAME" \ + --template-file "mslearn-postgresql/Allfiles/Labs/Shared/deploy.aoai-deployments.bicep" \ + --parameters azureOpenAIServiceName="$AOAI" + ``` + + The Bicep deployment scripts provisions the Azure services required to complete this exercise into your resource group. The resources deployed include an Azure Database for PostgreSQL server, Azure OpenAI, an Azure AI Language service. The Bicep script also performs some configuration steps, such as adding the `azure_ai` and `vector` extensions to the PostgreSQL server's _allowlist_ (via the `azure.extensions` server parameter), creating a database named `ContosoHelpDesk` on the server, and adding a deployment named `embedding` using the `text-embedding-ada-002` model to your Azure OpenAI service. Finally it adds a deployment named `chat` using the `gpt-4o-mini` model to your Azure OpenAI service. The Bicep file shares all modules in this learning path, so you might only use some of the deployed resources in some exercises. + + The deployment typically takes several minutes to complete. You can monitor it from the Cloud Shell or navigate to the **Deployments** page for the resource group you previously created and observe the deployment progress there. + +1. Take note of the resource names and their corresponding ID, and the PostgreSQL server's fully qualified domain name (FQDN), username, and password, as you need them later. + +### Troubleshooting deployment errors + +You could encounter a few errors when running the Bicep deployment script. *If no errors are encountered, skip this section.* + +- If you previously ran the Bicep deployment script for this learning path and later deleted the resources, you could receive an error message like the following if you're attempting to rerun the script within 48 hours of deleting the resources: + + ```bash + {"code": "InvalidTemplateDeployment", "message": "The template deployment 'deploy' is not valid according to the validation procedure. The tracking id is '4e87a33d-a0ac-4aec-88d8-177b04c1d752'. See inner errors for details."} + + Inner Errors: + {"code": "FlagMustBeSetForRestore", "message": "An existing resource with ID '/subscriptions/{subscriptionId}/resourceGroups/rg-learn-postgresql-ai-eastus/providers/Microsoft.CognitiveServices/accounts/{accountName}' has been soft-deleted. To restore the resource, you must specify 'restore' to be 'true' in the property. If you don't want to restore existing resource, please purge it first."} + ``` + + If you receive this message, modify the `azure deployment group create` command previously to set the `restore` parameter equal to `true` and rerun it. + +- If the selected region is restricted from provisioning specific resources, you must set the `REGION` variable to a different location and rerun the commands to create the resource group and run the Bicep deployment script. + + ```bash + {"status":"Failed","error":{"code":"DeploymentFailed","target":"/subscriptions/{subscriptionId}/resourceGroups/{resourceGrouName}/providers/Microsoft.Resources/deployments/{deploymentName}","message":"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.","details":[{"code":"ResourceDeploymentFailure","target":"/subscriptions/{subscriptionId}/resourceGroups/{resourceGrouName}/providers/Microsoft.DBforPostgreSQL/flexibleServers/{serverName}","message":"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.","details":[{"code":"RegionIsOfferRestricted","message":"Subscriptions are restricted from provisioning in this region. Please choose a different region. For exceptions to this rule please open a support request with Issue type of 'Service and subscription limits'. See https://review.learn.microsoft.com/en-us/azure/postgresql/flexible-server/how-to-request-quota-increase for more details."}]}]}} + ``` + +- If the script is unable to create an AI resource due to the requirement to accept the responsible AI agreement, you get the following error. If you get that error, use the Azure portal user interface to create an Azure AI Services resource, and then rerun the deployment script. + + ```bash + {"code": "InvalidTemplateDeployment", "message": "The template deployment 'deploy' is not valid according to the validation procedure. The tracking id is 'f8412edb-6386-4192-a22f-43557a51ea5f'. See inner errors for details."} + + Inner Errors: + {"code": "ResourceKindRequireAcceptTerms", "message": "This subscription cannot create TextAnalytics until you agree to Responsible AI terms for this resource. You can agree to Responsible AI terms by creating a resource through the Azure Portal then trying again. For more detail go to https://go.microsoft.com/fwlink/?linkid=2164190"} + ``` + +## Connect to your database using psql in the Azure Cloud Shell + +You connect to the `ContosoHelpDesk` database on your Azure Database for PostgreSQL server using the [psql command-line utility](https://www.postgresql.org/docs/current/app-psql.html) from the [Azure Cloud Shell](https://learn.microsoft.com/azure/cloud-shell/overview). + +1. In the [Azure portal](https://portal.azure.com/), navigate to your newly created Azure Database for PostgreSQL server. + +1. In the resource menu, under **Settings**, select **Databases** select **Connect** for the `ContosoHelpDesk` database. Selecting **Connect** doesn't actually connect you to the database; it simply provides instructions for connecting to the database using various methods. Review the instructions to **Connect from browser or locally** and use those instructions to connect using the Azure Cloud Shell. + + ![Screenshot of the Azure Database for PostgreSQL Databases page. Databases and Connect for the ContosoHelpDesk database are highlighted by red boxes.](media/14-postgresql-database-connect.png) + +1. At the "Password for user pgAdmin" prompt in the Cloud Shell, enter the randomly generated password for the **pgAdmin** sign in. + + Once you sign in, the `psql` prompt for the `ContosoHelpDesk` database is displayed. + +1. Throughout the remainder of this exercise, you continue working in the Cloud Shell, so it helps to expand the pane within your browser window by selecting the **Maximize** button at the top right of the pane. + + ![Screenshot of the Azure Cloud Shell pane with the Maximize button highlighted by a red box.](media/14-azure-cloud-shell-pane-maximize.png) + +## Setup: Configure extensions + +To store and query vectors, and to generate embeddings, you need to allowlist and enable two extensions for Azure Database for PostgreSQL: `vector` and `azure_ai`. + +1. To allowlist both extensions, add `vector` and `azure_ai` to the server parameter `azure.extensions`, as per the instructions provided in [How to use PostgreSQL extensions](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-extensions#how-to-use-postgresql-extensions). + +1. Run the following SQL command to enable the `vector` and `azure_ai` extensions. For detailed instructions, read [How to enable and use `pgvector` on Azure Database for PostgreSQL](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/how-to-use-pgvector#enable-extension). + + On *ContosoHelpDesk* prompt, run the following SQL commands: + + ```sql + -- Enable required extensions + CREATE EXTENSION vector; + CREATE EXTENSION azure_ai; + ``` + +1. To enable the `azure_ai` extension, run the following SQL command. You need the endpoint and API key for the Azure OpenAI resource. For detailed instructions, read [Enable the `azure_ai` extension](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/generative-ai-azure-overview#enable-the-azure_ai-extension). + + On the *ContosoHelpDesk* prompt, run the following commands: + + ```sql + -- Configure Azure OpenAI (requires azure_ai_settings_manager role) + SELECT azure_ai.set_setting('azure_openai.endpoint', 'https://.openai.azure.com'); -- e.g., https://YOUR-RESOURCE.openai.azure.com + SELECT azure_ai.set_setting('azure_openai.subscription_key', ''); + ``` + +## Populate the database with sample data + +Before you use the `azure_ai` extension, add a table to the `ContosoHelpDesk` database and populate them with sample data so you have information to work with as you create your application. + +1. On the **ContosoHelpDesk** prompt, run the following commands to create the `company_policies` table for storing company policy data: + + ```sql + -- Create table for policies and embeddings (matches CSV columns) + DROP TABLE IF EXISTS company_policies CASCADE; + + CREATE TABLE company_policies ( + policy_id BIGSERIAL PRIMARY KEY, + title TEXT NOT NULL, + department TEXT NOT NULL, + policy_text TEXT NOT NULL, + category TEXT NOT NULL, + embedding vector(1536) -- The `text-embedding-ada-002` model is configured to return 1,536 dimensions, so use that number for the vector column size. + ); + ``` + +1. In your Azure Cloud Shell, use the `COPY` command to load data from CSV files into each table you previously created. Run the following command to populate the `company_policies` table: + + ```sql + \COPY company_policies (title, department, policy_text, category) FROM 'mslearn-postgresql/Allfiles/Labs/Shared/company-policies.csv' WITH (FORMAT csv, HEADER) + ``` + + The command output should be `COPY 108`, indicating that 108 rows were written into the table from the CSV file. + +1. Backfill embeddings for existing rows. + + Run the following command in your **psql** session (Cloud Shell) to compute embeddings for any rows that don’t have them yet. Replace `` with the name of your embedding deployment. + + ```sql + -- Create embeddings for existing rows that currently have no embeddings + UPDATE company_policies + SET embedding = azure_openai.create_embeddings('', policy_text)::vector + WHERE embedding IS NULL; + ``` + + This calls your Azure OpenAI embedding deployment from SQL (via `azure_ai`) and stores the result in the column `embedding`. + +If you successfully backfilled the 108 rows with embeddings, exit *psql* by typing `\q` and skip the following troubleshooting section. Otherwise, continue with the following troubleshooting steps. + +### Troubleshoot 429 errors if encountered + +*Skip this section if your UPDATE statement successfully backfilled 108 embeddings*. + +1. Depending on your Azure OpenAI rate limits, you might experience **429 Too Many Requests** errors if you exceed the allowed number of requests. If that is the case for the previous UPDATE statement, you can run the following command to batch the requests and retry (if needed manually reduce the *batch_size* too): + + ```sql + DO $$ + DECLARE + batch_size int := 50; -- rows per batch + optimistic_pause int := 10; -- seconds to wait after a successful batch + pause_secs int := 10; -- current wait (resets to optimistic on success) + max_pause int := 60; -- cap the backoff + updated int; + BEGIN + LOOP + BEGIN + WITH todo AS ( + SELECT policy_id, policy_text + FROM company_policies + WHERE embedding IS NULL + ORDER BY policy_id + LIMIT batch_size + ) + UPDATE company_policies p + SET embedding = azure_openai.create_embeddings('embedding', t.policy_text)::vector + FROM todo t + WHERE p.policy_id = t.policy_id; + + GET DIAGNOSTICS updated = ROW_COUNT; + + IF updated = 0 THEN + RAISE NOTICE 'No rows left to embed.'; + EXIT; + END IF; + + -- Success: reset to optimistic pause and sleep briefly + pause_secs := optimistic_pause; + RAISE NOTICE 'Updated % rows; sleeping % seconds before next batch.', updated, pause_secs; + PERFORM pg_sleep(pause_secs); + + EXCEPTION WHEN OTHERS THEN + -- Likely throttled (429) or transient error: back off and retry + RAISE NOTICE 'Throttled/transient error; backing off % seconds.', pause_secs; + PERFORM pg_sleep(pause_secs); + pause_secs := LEAST(pause_secs * 2, max_pause); + END; + END LOOP; + END $$; + + ``` + +1. If you successfully backfilled the 108 rows with embeddings, exit *psql* by typing `\q`, otherwise, try reducing the *batch_size* by 10 and run the previous script again. + +### Add the vector index + +To improve the performance of similarity searches, you can add a vector index to the `embedding` column of the `company_policies` table. + +1. On the Azure Cloud Shell, connect to the *ContosoHelpDesk* database using *psql* as before. + +1. Create the IVFFlat index: + + ```sql + -- Drop the IVFFlat index + DROP INDEX IF EXISTS company_policies_embedding_ivfflat_idx; + + -- Use cosine distance (vector_cosine_ops) for text embeddings + CREATE INDEX company_policies_embedding_ivfflat_idx + ON company_policies + USING ivfflat (embedding vector_cosine_ops) + WITH (lists = 100); + + ANALYZE company_policies; + ``` + +### Test the vector table with a similarity query + +Let's make sure everything is working by verifying with a similarity search and simple filtering directly from SQL. + +1. On the Azure Cloud Shell, connect to the *ContosoHelpDesk* database using *psql* as before. + +1. Run the following SQL statement: + + ```sql + -- Best match for a question (cosine) + SELECT policy_id, title, department, policy_text + FROM company_policies + ORDER BY embedding <=> azure_openai.create_embeddings('embedding', + 'How many vacation days do employees get?')::vector + LIMIT 1; + ``` + +1. Add a filter plus a vector search by running the following SQL statement: + + ```sql + -- Filter + vector (hybrid) + SELECT policy_id, title, department, policy_text + FROM company_policies + WHERE department = 'HR' + ORDER BY embedding <=> azure_openai.create_embeddings('embedding', + 'Does the company help me with college expenses')::vector + LIMIT 3; + ``` + +1. Type *\q* and press Enter to exit *psql*. + +While these answers are a good start, they might not be comprehensive enough for more complex queries. To address this problem, you can create a Python RAG (Retrieval-Augmented Generation) application that retrieves relevant passages from our database and uses them as context for generating answers. + +## Enable the Apache AGE Extension on the Azure portal (temporary step that will be removed and added to the bicep setup file) + +Before you can use the Apache AGE extension, you need to enable it on the Azure portal. + +1. Go to the Azure portal and navigate to your Azure Database for PostgreSQL instance. +1. In the left-hand menu, under *Settings*, select *Server parameters*. +1. Search for **azure.extensions**. +1. Under the **Value** column, add `AGE` to the list of enabled extensions. +1. Do a new search for **shared_preload_libraries**. +1. Under the **Value** column, add `AGE` to the list of enabled extensions. +1. Select **Save** to apply the changes. + +Now you should be able to use the Apache AGE extension in your PostgreSQL database. + +## Build the knowledge graph + +A knowledge graph is a structured representation of information that captures entities and their relationships. In this case, you create a knowledge graph from the company policies data. You start by creating the nodes and edges tables. + +### Enable AGE and create the graph (same session) + +Let's first enable the Apache AGE extension and create a new graph. + +1. On the Azure Cloud Shell, connect to the *ContosoHelpDesk* database using *psql* as before. + +1. Run the following SQL statement: + + ```sql + -- Enable the AGE extension + CREATE EXTENSION IF NOT EXISTS age CASCADE; + + -- Put ag_catalog in the session path + SET search_path = public, ag_catalog; + + -- Create a fresh graph namespace + SELECT ag_catalog.create_graph('company_policies_graph'); + ``` + +Time to create the nodes and edges for the knowledge graph. + +### Create the graph nodes + +Let's create the nodes from the company policies data. + +1. On the Azure Cloud Shell, connect to the *ContosoHelpDesk* database using *psql* as before. + +1. First, let's define the function for upserting the policy nodes. Run the following SQL statement: + + ```sql + DROP FUNCTION IF EXISTS public.policy_graph_upsert(BIGINT, TEXT, TEXT, TEXT, TEXT); + + -- Create a policy node + CREATE OR REPLACE FUNCTION public.policy_graph_upsert( + _id BIGINT, _title TEXT, _dept TEXT, _cat TEXT, _text TEXT + ) RETURNS void + LANGUAGE plpgsql + VOLATILE + AS $BODY$ + BEGIN + -- Use ag_catalog just for this statement (doesn't leak outside the function) + SET LOCAL search_path TO ag_catalog, public; + + EXECUTE format( + 'SELECT * FROM cypher(''company_policies_graph'', $$ + MERGE (p:Policy {policy_id: %s}) + SET p.title = %L, + p.department = %L, + p.category = %L, + p.policy_text = %L + $$) AS (n agtype);', + _id, _title, _dept, _cat, _text + ); + END + $BODY$; + ``` + +1. Next, let's add a function to upsert the department, category, and topic nodes. + + ```sql + DROP FUNCTION IF EXISTS public.create_entity_in_policies_graph(TEXT, TEXT); + + -- Create a new department, category, or topic entity node in the graph + CREATE OR REPLACE FUNCTION public.create_entity_in_policies_graph( + _type TEXT, _name TEXT + ) RETURNS void + LANGUAGE plpgsql + VOLATILE + AS $BODY$ + BEGIN + SET LOCAL search_path TO ag_catalog, public; + + EXECUTE format( + 'SELECT * FROM cypher(''company_policies_graph'', $$ + MERGE (e:Entity {type: %L, name: %L}) + $$) AS (n agtype);', + _type, _name + ); + END + $BODY$; + ``` + +1. Run the following SQL statements to add our nodes: + + ```sql + -- Disable pagination for better output readability + \pset pager off + + -- Upsert the policy nodes + SELECT public.policy_graph_upsert(policy_id, title, department, category, policy_text) + FROM public.company_policies + ORDER BY policy_id; + + -- Departments + SELECT public.create_entity_in_policies_graph('Department', d.department) + FROM (SELECT DISTINCT department FROM public.company_policies) AS d; + + -- Categories + SELECT public.create_entity_in_policies_graph('Category', c.category) + FROM (SELECT DISTINCT category FROM public.company_policies) AS c; + + -- Topics - Group policies by list of common terms that might be mentioned in a policy + WITH topics(name) AS ( + VALUES + ('Employees'), + ('Approval'), + ('Customer'), + ('Meetings'), + ('Exit/Termination'), + ('Legal'), + ('Devices'), + ('Events'), + ('Expense'), + ('New Hires'), + ('Reconciled Monthly'), + ('Remote'), + ('Vendors/Suppliers'), + ('Internet/Social Media'), + ('Onboarding'), + ('Prior Approval'), + ('Products'), + ('Reviewed Quarterly'), + ('Tickets'), + ('Training') + ) + SELECT public.create_entity_in_policies_graph('Topic', name) + FROM topics; + ``` +You should now have all the nodes created, time to create the edges that connect them. + +### Create the graph edges + +So far you added the nodes for policies, departments, categories, and topics. It's time to create the edges that connect them. + +1. On the Azure Cloud Shell, connect to the *ContosoHelpDesk* database using *psql* as before. + +1. Let's create a function to establish the edges between the policy nodes and their respective department, category, and topic nodes. + + ```sql + -- Policy -> Entity edge upsert + -- rel ∈ ('BELONGS_TO','IN_CATEGORY','MENTIONS') + DROP FUNCTION IF EXISTS public.create_policy_link_in_policies_graph(BIGINT, TEXT, TEXT, TEXT); + + CREATE OR REPLACE FUNCTION public.create_policy_link_in_policies_graph( + _policy_id BIGINT, _etype TEXT, _ename TEXT, _rel TEXT + ) RETURNS void + LANGUAGE plpgsql + VOLATILE + AS $BODY$ + BEGIN + SET LOCAL search_path TO ag_catalog, public; + + EXECUTE format( + 'SELECT * FROM cypher(''company_policies_graph'', $$ + MATCH (p:Policy {policy_id: %s}) + MATCH (e:Entity {type: %L, name: %L}) + MERGE (p)-[:%s]->(e) + RETURN 1 + $$) AS (ok agtype);', + _policy_id, _etype, _ename, _rel + ); + END + $BODY$; + ``` + +1. Now, run the following SQL command to create the edges between the policy nodes and their respective department, category, and topic nodes: + + ```sql + -- Disable pagination for better output readability + \pset pager off + + -- BELONGS_TO + SELECT public.create_policy_link_in_policies_graph(policy_id, 'Department', department, 'BELONGS_TO') + FROM public.company_policies; + + -- IN_CATEGORY + SELECT public.create_policy_link_in_policies_graph(policy_id, 'Category', category, 'IN_CATEGORY') + FROM public.company_policies; + + -- MENTIONS - Note that you use some regex patterns to match similar terms + WITH topics(name, pattern) AS ( + VALUES + ('Employees', $$\memployee(s)?\M$$), + ('Approval', $$\mapprov(e|al|ed|als|ing)?\M$$), + ('Customer', $$\mcustomer(s)?\M$$), + ('Meetings', $$\mmeeting(s)?\M$$), + ('Exit/Termination', $$\m(exit|termination)\M$$), + ('Legal', $$\mlegal\M$$), + ('Devices', $$\m(device(s)?|laptop(s)?)\M$$), + ('Events', $$\mevent(s)?\M$$), + ('Expense', $$\mexpense(s)?\M$$), + ('New Hires', $$\mnew\M\s+\mhires\M$$), + ('Reconciled Monthly', $$\mreconciled\M\s+\mmonthly\M$$), + ('Remote', $$\mremote\M(\s+\mwork\M)?$$), + ('Vendors/Suppliers', $$\m(vendor(s)?|supplier(s)?)\M$$), + ('Internet/Social Media', $$\minternet\M|\msocial\M\s+\mmedia\M$$), + ('Onboarding', $$\monboard(ed|ing)?\M|\monboarding\M$$), + ('Prior Approval', $$\mprior\M\s+\mapproval\M$$), + ('Products', $$\mproduct(s)?\M$$), + ('Reviewed Quarterly', $$\mreviewed\M\s+\mquarterly\M$$), + ('Tickets', $$\mticket(s)?\M|\mhelp\M\s*\mdesk\M$$), + ('Training', $$\mtrain(ing|ed|s)?\M$$) + ) + SELECT public.create_policy_link_in_policies_graph(p.policy_id, 'Topic', t.name, 'MENTIONS') + FROM public.company_policies p + JOIN topics t + ON p.policy_text ~* t.pattern + GROUP BY t.name, p.policy_id + ORDER BY t.name, p.policy_id; + ``` + +1. Let's now check the nodes and edges counts for your graph: + + ```sql + -- Total policy nodes + SELECT * FROM cypher('company_policies_graph', + $$ MATCH (p:Policy) RETURN count(p) $$) AS (count agtype); + + -- Entities by type (Department/Category/Topic) + SELECT * FROM cypher('company_policies_graph', + $$ MATCH (e:Entity) RETURN e.type, count(e) ORDER BY e.type $$) AS (type agtype, count agtype); + + -- Edges by relationship type + SELECT * FROM cypher('company_policies_graph', + $$ MATCH ()-[r]->() RETURN type(r), count(r) ORDER BY type(r) $$) AS (rel agtype, count agtype); + ``` + +Now you have a full graph representation of the company policies, including all relevant entities and their relationships. + +## Run a graph-narrowed vector search + +Now that you created policy, department, and topic nodes, and connected them with edges, you can use the graph to **narrow candidates** and then use **pgvector** to **rank** those candidates by semantic similarity to a question. This process keeps the full retrieval flow inside *Azure Database for PostgreSQL*. + +### Use department plus topics to narrow candidates + +Let's narrow our search to policies that belong to the **Finance** department and mention specific topics. + +1. On the Azure Cloud Shell, connect to the *ContosoHelpDesk* database using *psql* as before. + +1. On the *ContosoHelpDesk* prompt, set a variable with the question you want to search for: + + ``` + \set question 'What expenses require prior approval for remote work travel?' + ``` + +1. Run the following SQL statement to retrieve the top five policy passages. The graph picks **Finance** policies that **mention** any of the selected topics. The vector step ranks those candidates by semantic similarity to your question. A small deduplication step keeps one row per policy, which removes duplicates. + + ```sql + /* Graph-narrowed vector search: filter by graph, then rank with pgvector */ + WITH + /* 1) GRAPH FILTER: candidate policy_ids from AGE */ + graph_ids AS ( + SELECT ((pid)::text)::bigint AS policy_id + FROM ag_catalog.cypher('company_policies_graph'::name, $$ + MATCH (p:Policy)-[:BELONGS_TO]->(:Entity {type:'Department', name:'Finance'}) + MATCH (p)-[:MENTIONS]->(t:Entity {type:'Topic'}) + WHERE t.name IN ['Expense','Approval','Remote'] /* adjust topics as needed */ + RETURN p.policy_id AS pid + $$::cstring) AS (pid agtype) + ), + + /* 2) QUESTION EMBEDDING: compute once from the psql 'question' variable */ + q AS ( + SELECT azure_openai.create_embeddings('embedding', :'question')::vector AS qv + ), + + /* 3) VECTOR RANK: smaller cosine distance is better */ + ranked AS ( + SELECT + cp.policy_id, + cp.title, + cp.department, + cp.category, + cp.policy_text, + (cp.embedding <=> q.qv) AS distance + FROM public.company_policies cp + JOIN graph_ids USING (policy_id) + CROSS JOIN q + WHERE cp.embedding IS NOT NULL + ), + + /* 4) DEDUP: keep the best (smallest distance) row per policy */ + dedup AS ( + SELECT *, + ROW_NUMBER() OVER (PARTITION BY policy_id ORDER BY distance) AS rn + FROM ranked + ) + + /* 5) RESULT: unique top 5 */ + SELECT policy_id, title, department, category, policy_text + FROM dedup + WHERE rn = 1 + ORDER BY distance + LIMIT 5; + ``` + +> [!TIP] +> In `psql`, run the `\set question` command on its **own line**, and press **Enter** before you run the SQL query. If you paste both at once, the CTE might not run as expected. + +This query generates a ranked list of policy passages that are relevant to the specified question. It first uses the graph structure to filter candidates by department and topic, then ranks them by semantic similarity to the question. + +Let's try a different filter. + +- **Topic-only (no department filter)** — Previously you filtered by the *Finance* department, now let's include all departments. Run the following query: + + ```sql + WITH graph_ids AS ( + SELECT ((pid)::text)::bigint AS policy_id + FROM ag_catalog.cypher('company_policies_graph'::name, $$ + MATCH (p:Policy)-[:MENTIONS]->(t:Entity {type:'Topic'}) + WHERE t.name IN ['Customer','Meetings','Expense'] + RETURN p.policy_id AS pid + $$::cstring) AS (pid agtype) + ), + q AS (SELECT azure_openai.create_embeddings('embedding', :'question')::vector AS qv), + ranked AS ( + SELECT cp.policy_id, cp.title, cp.department, cp.category, cp.policy_text, + (cp.embedding <=> q.qv) AS distance + FROM public.company_policies cp + JOIN graph_ids USING (policy_id) + CROSS JOIN q + WHERE cp.embedding IS NOT NULL + ), + dedup AS ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY policy_id ORDER BY distance) AS rn + FROM ranked + ) + SELECT policy_id, title, department, category, policy_text + FROM dedup + WHERE rn = 1 + ORDER BY distance + LIMIT 5; + ``` + + Notice how the candidate set changed, given a more inclusive topic filter. + +- **Change the topic list** — Replace `['Expense','Approval','Remote']` in the script with any subset of the 20 topics you created earlier (for example, `['Customer','Meetings','Expense']`). Then rerun the query. Notice how the candidate set changes again based on the topics selected. + +You can also change your question by modifying the `\set question` command in `psql`. Try some of the following ones and rerun the previous queries: + +- What are the policies related to customer interactions? +- How do we handle meeting notes and action items? +- What are the guidelines for remote work and travel? +- How do we ensure compliance with data privacy regulations? + +You can also try your own questions and change the topic list as you see fit. + +Combining your graph and vector search skills allow you to create powerful search applications. The larger the dataset you work with, the more effective your search capabilities become. + +## Key takeaways + +In this exercise, you used a small graph to add structure to retrieval. Instead of relying on look alike text only, you first pull candidates by connections like department and topic, then rank that list with `pgvector` against your question. Because it all runs in one database inside *Azure Database for PostgreSQL*, the flow stays simple to operate and easy to explain since the filters and paths are visible. + +To apply the methods discussed here on your own data, start small. Pick a few entities and relationships that matter, link them to your rows, use a short `openCypher` query to fetch candidate `ids`, then apply vector ranking. Tighten or relax the filters as needed, swap in other concepts, and keep the workflow in SQL so it's straightforward to maintain. + diff --git a/Instructions/Labs/media/14-postgresql-database-connect.png b/Instructions/Labs/media/14-postgresql-database-connect.png new file mode 100644 index 0000000..a30d9a8 Binary files /dev/null and b/Instructions/Labs/media/14-postgresql-database-connect.png differ