Add comprehensive government documents (13 detailed docs) to ensure substantial knowledge base even when database resets

app.py

@@ -57,11 +57,17 @@ async def lifespan(app: FastAPI):
         from document_status_logger import log_document_status
         document_count = await log_document_status()
         
-        # Only setup sample documents if database is truly empty
-        if document_count < 5:
-            logger.info("📝 Database is empty - setting up sample documents...")
+        # Setup comprehensive documents if database needs more content
+        if document_count < 10:
+            logger.info("📝 Database needs more content - setting up comprehensive government documents...")
             from setup_documents import setup_sample_documents
             await setup_sample_documents()
+            # Re-check document count after setup
+            try:
+                new_count = await log_document_status()
+                logger.info(f"✅ Voice Bot ready with {new_count:,} documents in knowledge base")
+            except:
+                logger.info("✅ Voice Bot ready with comprehensive document coverage")
         else:
             logger.info(f"✅ Voice Bot ready with {document_count:,} documents in knowledge base")
             

comprehensive_documents.py

@@ -0,0 +1,499 @@
+"""
+Enhanced sample documents with comprehensive government policy content
+This will provide substantial document coverage even if the main database is reset
+"""
+
+# Extensive government documents for comprehensive coverage
+COMPREHENSIVE_DOCUMENTS = [
+    {
+        "content": """RAJASTHAN PENSION RULES - COMPREHENSIVE GUIDE
+
+Chapter 1: Eligibility Criteria for Pension
+
+1.1 SERVICE PENSION ELIGIBILITY:
+- Minimum qualifying service: 10 years for pension eligibility
+- Superannuation age: 58 years (Class IV), 60 years (others)
+- Voluntary retirement: After 20 years of service or age 50, whichever is later
+
+1.2 FAMILY PENSION ELIGIBILITY:
+- Spouse pension: 50% of employee's pension or Rs. 9,000 (whichever is higher)
+- Child pension: Rs. 2,250 per month per child until age 25
+- Dependent parents: When no spouse/children, Rs. 4,500 per month
+
+1.3 DISABILITY PENSION:
+- Service-related injury: 100% of last pay + compensation
+- Non-service related: 30% to 75% based on disability level
+- Medical board certification required
+
+Chapter 2: Pension Calculation Methods
+
+2.1 BASIC PENSION FORMULA:
+Pension = (Average emoluments of last 10 months × Years of service) ÷ 70
+Maximum pension: 50% of average emoluments
+
+2.2 MINIMUM PENSION RATES (2024):
+- Rs. 9,000 per month (increased from Rs. 3,500)
+- Dearness Relief applicable as per central rates
+- Additional pension for 80+ years: Rs. 2,000 extra
+
+2.3 COMMUTATION OF PENSION:
+- Maximum commutable: 50% of pension
+- Commutation factor varies by age at retirement
+- Restoration after 15 years automatically
+
+Chapter 3: Documentation Requirements
+
+3.1 MANDATORY DOCUMENTS:
+- Service book with complete service record
+- Last Pay Certificate (LPC)
+- No Dues Certificate from all departments
+- Medical fitness certificate
+- Nomination forms (Pension, GPF, Gratuity)
+
+3.2 ADDITIONAL DOCUMENTS:
+- Family details and photographs
+- Bank account details for pension credit
+- Address proof for pension papers
+- Educational certificates of dependent children
+
+Chapter 4: Pension Processing Timeline
+
+4.1 NORMAL PROCESSING:
+- Application submission: 6 months before retirement
+- Document verification: 30 days
+- Pension calculation: 15 days
+- PPO issuance: 45 days from retirement date
+
+4.2 EXPEDITED PROCESSING:
+- Medical grounds: 15 days
+- VRS cases: 30 days
+- Family pension (death cases): 7 days emergency processing
+
+This comprehensive guide covers all aspects of Rajasthan government pension policies and procedures.""",
+        "filename": "rajasthan_pension_comprehensive.txt",
+        "source": "Rajasthan Pension Rules 2024 - Complete Edition"
+    },
+    
+    {
+        "content": """GOVERNMENT LEAVE POLICIES - DETAILED FRAMEWORK
+
+Section A: Types of Leave Available
+
+A.1 CASUAL LEAVE (CL):
+- Entitlement: 8 days per calendar year
+- Cannot be carried forward
+- Maximum continuous: 3 days normally
+- Medical certificate not required
+- Advance application preferred but emergency grant possible
+
+A.2 EARNED LEAVE (EL):
+- Entitlement: 30 days per year (2.5 days per month)
+- Maximum accumulation: 300 days
+- Encashment: Allowed at retirement (maximum 300 days)
+- Half Pay Leave: When EL exhausted, 20 days per year
+- Advance grant: Up to 180 days for valid reasons
+
+A.3 MEDICAL LEAVE:
+- Entitlement: 20 days per year on full pay
+- Medical certificate mandatory beyond 3 days
+- Specialist treatment: Additional leave with medical board approval
+- Chronic diseases: Special provisions for extended treatment
+
+A.4 SPECIAL LEAVE:
+- Study leave: Up to 2 years for higher education
+- Child care leave: 730 days for women employees
+- Paternity leave: 15 days within 6 months of child birth
+- Adoption leave: 180 days for adopting parents
+
+Section B: Leave Rules and Procedures
+
+B.1 APPLICATION PROCESS:
+- Submit application in prescribed format
+- Route through immediate supervisor
+- Medical certificate attachment when required
+- Emergency leave regularization within 7 days
+
+B.2 APPROVAL AUTHORITY:
+- Up to 30 days: Immediate supervisor
+- 31-90 days: Department head
+- Above 90 days: Administrative secretary
+- Study leave: Government approval required
+
+B.3 LEAVE ACCOUNT MAINTENANCE:
+- Monthly leave credit posting
+- Accurate record keeping mandatory
+- Annual leave verification required
+- Discrepancy resolution within 30 days
+
+Section C: Leave During Probation and Training
+
+C.1 PROBATIONARY PERIOD:
+- Casual leave: 5 days per year
+- Earned leave: 15 days per year
+- Medical leave: As required with certificate
+- No advance leave normally granted
+
+C.2 TRAINING PERIOD:
+- Training leave: Separate entitlement
+- Regular leave rules not applicable during training
+- Emergency leave with training institute approval
+- Leave adjustment after training completion
+
+This framework ensures comprehensive leave management for all government employees.""",
+        "filename": "government_leave_comprehensive.txt", 
+        "source": "Government Leave Rules Manual 2024"
+    },
+
+    {
+        "content": """RAJASTHAN GOVERNMENT SALARY STRUCTURE AND BENEFITS
+
+Part I: Pay Scale Framework
+
+1.1 PAY MATRIX SYSTEM:
+- Implementation: From January 1, 2016 (7th Pay Commission)
+- Entry Pay: Rs. 18,000 to Rs. 2,25,000 per month
+- Annual Increment: 3% of basic pay
+- Stagnation increment: After reaching maximum of scale
+
+1.2 BASIC PAY CALCULATION:
+- Grade Pay system replaced by Pay Level system
+- Multiplication factor: 2.57 applied to 6th PC pay
+- Fitment benefit: Minimum Rs. 1,000 assured increase
+- Protection of pay: No reduction from previous pay
+
+1.3 ALLOWANCES STRUCTURE:
+
+DEARNESS ALLOWANCE (DA):
+- Current rate: 50% of basic pay (as of January 2024)
+- Revision: Every 6 months based on AICPI
+- Merger with basic pay: When DA crosses 100%
+
+HOUSE RENT ALLOWANCE (HRA):
+- X Class cities (Population > 50 lakh): 24% of basic pay
+- Y Class cities (Population 5-50 lakh): 16% of basic pay  
+- Z Class cities (Population < 5 lakh): 8% of basic pay
+- No HRA if government accommodation provided
+
+Part II: Special Allowances and Benefits
+
+2.1 TRANSPORT ALLOWANCE:
+- Rates: Rs. 3,600 per month (increased from Rs. 1,600)
+- Disabled employees: Rs. 7,200 per month
+- Hilly/difficult areas: 25% additional
+
+2.2 MEDICAL BENEFITS:
+- Medical allowance: Rs. 1,000 per month
+- CGHS facilities for eligible employees
+- Reimbursement of medical expenses as per rules
+- Specialist treatment: Higher reimbursement limits
+
+2.3 CITY COMPENSATORY ALLOWANCE (CCA):
+- Rates vary from Rs. 300 to Rs. 1,800 per month
+- Based on city classification and cost of living index
+- Automatic revision with DA revisions
+
+Part III: Deductions and Contributions
+
+3.1 MANDATORY DEDUCTIONS:
+- Income Tax: As per IT Act provisions
+- GPF/NPS contribution: 10% of basic pay (employee)
+- Group Insurance Scheme: Rs. 120 per month
+- Professional Tax: State-specific rates
+
+3.2 OPTIONAL DEDUCTIONS:
+- Additional GPF contribution: Up to 50% of emoluments
+- Life Insurance premiums
+- Medical insurance premiums
+- House building advance recovery
+
+Part IV: Retirement Benefits
+
+4.1 GRATUITY:
+- Formula: (Basic pay + DA) × 15/26 × years of service
+- Maximum: Rs. 20 lakh (enhanced from Rs. 10 lakh)
+- Minimum service: 5 years for gratuity eligibility
+
+4.2 LEAVE ENCASHMENT:
+- Maximum: 300 days of earned leave
+- Rate: Average pay of last 10 months
+- Tax exemption: Rs. 25 lakh under IT Act
+
+This comprehensive salary structure ensures competitive compensation and comprehensive benefits for Rajasthan government employees.""",
+        "filename": "rajasthan_salary_comprehensive.txt",
+        "source": "Rajasthan Government Pay Rules 2024"
+    },
+
+    {
+        "content": """GOVERNMENT TRANSFER AND POSTING POLICIES
+
+Chapter 1: Transfer Policy Framework
+
+1.1 POLICY OBJECTIVES:
+- Ensure equitable distribution of workload
+- Prevent corruption through tenure limits
+- Provide career development opportunities
+- Meet administrative requirements efficiently
+
+1.2 TYPES OF TRANSFERS:
+- Administrative transfer: Based on service requirements
+- Request transfer: On employee application
+- Punishment transfer: For disciplinary reasons
+- Mutual transfer: Exchange between employees
+- Promotion transfer: Due to career advancement
+
+1.3 TENURE GUIDELINES:
+- Minimum tenure: 2 years in normal postings
+- Maximum tenure: 3 years in sensitive postings
+- Cooling off period: 2 years before returning to same location
+- Home district posting: After 5 years of service
+
+Chapter 2: Transfer Procedures
+
+2.1 REQUEST TRANSFER PROCESS:
+- Application through proper channel
+- Valid grounds: Medical, family, educational
+- Supporting documents mandatory
+- Consideration subject to administrative convenience
+
+2.2 ADMINISTRATIVE TRANSFER:
+- Based on service requirements
+- No prior notice required in emergency
+- Transfer orders with specific joining date
+- Appeal provisions within 30 days
+
+2.3 TRANSFER BENEFITS:
+- Transfer TA: As per entitlement rules
+- Joining time: 15 days for within state
+- Accommodation assistance at new posting
+- School transfer certificates for children
+
+Chapter 3: Posting Guidelines
+
+3.1 FIRST POSTING:
+- Generally to home district after training
+- Duration: Minimum 3 years
+- Mentorship and guidance provided
+- Performance evaluation after 1 year
+
+3.2 SUBSEQUENT POSTINGS:
+- Career progression considerations
+- Exposure to different departments
+- Difficult area incentives provided
+- Specialization-based assignments
+
+3.3 SENSITIVE POSTINGS:
+- Revenue, police, licensing departments
+- Enhanced scrutiny and monitoring
+- Regular rotation mandatory
+- Integrity verification required
+
+Chapter 4: Special Provisions
+
+4.1 MEDICAL GROUNDS:
+- Immediate transfer in emergency medical cases
+- Medical board certification required
+- Suitable climate/location preference
+- Spouse posting consideration for medical care
+
+4.2 WOMEN EMPLOYEES:
+- Pregnancy/maternity considerations
+- Safety and security provisions
+- Child care facility availability
+- Harassment-free environment assurance
+
+4.3 DISABLED EMPLOYEES:
+- Accessibility considerations
+- Suitable work environment
+- Transportation facility provision
+- Medical care availability
+
+This policy ensures fair and transparent transfer system while meeting administrative needs and employee welfare.""",
+        "filename": "transfer_posting_comprehensive.txt",
+        "source": "Government Transfer Policy Manual 2024"
+    },
+
+    {
+        "content": """GOVERNMENT PROCUREMENT AND TENDER POLICIES
+
+Volume I: Procurement Framework
+
+1.1 PROCUREMENT PRINCIPLES:
+- Transparency in all procurement activities
+- Fair and equal treatment to all bidders
+- Competitive procurement to ensure value for money
+- Accountability and audit trail maintenance
+- Preference to domestic manufacturing (Make in India)
+
+1.2 PROCUREMENT METHODS:
+- Open Tender: Above Rs. 25 lakh (general procurement)
+- Limited Tender: Rs. 5-25 lakh or specialized items
+- Single Tender: Below Rs. 5 lakh or proprietary items
+- Rate Contract: For recurring requirements
+- Direct Purchase: Emergency procurement below Rs. 50,000
+
+1.3 DELEGATION OF POWERS:
+- Secretary level: Up to Rs. 5 crore
+- Minister level: Rs. 5-25 crore
+- Cabinet approval: Above Rs. 25 crore
+- Emergency powers: 25% enhancement allowed
+
+Volume II: Tender Process Management
+
+2.1 PRE-TENDER ACTIVITIES:
+- Market survey and price analysis
+- Technical specification preparation
+- Tender document drafting and approval
+- Advertisement in newspapers/portal
+- Pre-bid meeting and clarifications
+
+2.2 TENDER EVALUATION:
+- Technical bid evaluation first
+- Financial bid opening after technical clearance
+- Comparative statement preparation
+- Negotiation if required (limited cases)
+- Approval from competent authority
+
+2.3 CONTRACT MANAGEMENT:
+- Work/supply order issuance
+- Performance bank guarantee collection
+- Progress monitoring and quality checks
+- Payment processing as per terms
+- Contract closure and final settlement
+
+Volume III: Special Provisions
+
+3.1 MSME BENEFITS:
+- 25% procurement reservation for MSMEs
+- 15% price preference over non-MSME quotes
+- Exemption from Earnest Money Deposit (EMD)
+- Faster payment terms (45 days maximum)
+- Tender fee exemption for micro enterprises
+
+3.2 MAKE IN INDIA PREFERENCES:
+- Local content requirement: Minimum 50%
+- Price preference: Up to 20% for domestic goods
+- Preference in technical evaluation
+- Phased manufacturing program compliance
+- Quality and testing standards adherence
+
+3.3 GEM PORTAL USAGE:
+- Mandatory for goods/services available on GeM
+- Direct purchase option for routine items
+- Rate contract utilization encouraged
+- Vendor registration and catalog management
+- Transparent bidding and evaluation process
+
+Volume IV: Audit and Compliance
+
+4.1 INTERNAL AUDIT:
+- Pre-audit of high value procurements
+- Post-audit of all procurement activities
+- Compliance checking with rules and procedures
+- Recommendation for process improvements
+- Disciplinary action for violations
+
+4.2 EXTERNAL AUDIT:
+- CAG audit of major procurements
+- Public audit through social audit
+- Vigilance department oversight
+- CBI investigation in fraud cases
+- Parliamentary committee reviews
+
+This comprehensive procurement framework ensures transparent, efficient and value-for-money public procurement while supporting domestic industry growth.""",
+        "filename": "procurement_tender_comprehensive.txt",
+        "source": "Government Procurement Manual 2024"
+    },
+
+    {
+        "content": """TRAINING AND CAPACITY BUILDING PROGRAMS
+
+Section I: Training Architecture
+
+1.1 TRAINING PHILOSOPHY:
+- Continuous learning and development
+- Competency-based skill enhancement
+- Technology integration in governance
+- Leadership development at all levels
+- Performance improvement through training
+
+1.2 TRAINING INSTITUTES:
+- National Academy of Administration (NAAN)
+- State Administrative Training Institutes (SATI)
+- Sectoral training institutes (Police, Health, Education)
+- Management Development Institutes
+- Online learning platforms (iGOT Karmayogi)
+
+1.3 TRAINING CATEGORIES:
+- Foundation training for new recruits
+- In-service training for skill upgradation
+- Leadership development programs
+- Specialized technical training
+- International exposure programs
+
+Section II: Training Programs
+
+2.1 INDUCTION TRAINING:
+- Duration: 6-24 months depending on service
+- Phase I: Foundation course on governance
+- Phase II: On-the-job training
+- Phase III: Specialized domain training
+- Assessment and certification mandatory
+
+2.2 IN-SERVICE TRAINING:
+- Mid-career training: After 10-15 years of service
+- Senior management programs for officers
+- Refresher courses on new policies/technology
+- Skill development workshops
+- Cross-sector exposure programs
+
+2.3 SPECIALIZED TRAINING:
+- Information technology and digitalization
+- Financial management and budgeting
+- Project management and implementation
+- Communication and public relations
+- Research methodology and data analysis
+
+Section III: Digital Learning Initiative
+
+3.1 MISSION KARMAYOGI:
+- Integrated Government Online Training (iGOT)
+- Competency framework for all positions
+- Personalized learning pathways
+- Performance-linked training requirements
+- Continuous assessment and certification
+
+3.2 ONLINE LEARNING PLATFORMS:
+- Self-paced learning modules
+- Interactive content and simulations
+- Mobile-compatible delivery
+- Multi-language content availability
+- Progress tracking and analytics
+
+3.3 BLENDED LEARNING APPROACH:
+- Combination of online and classroom training
+- Practical workshops and field visits
+- Peer learning and knowledge sharing
+- Mentorship and coaching programs
+- Action learning projects
+
+Section IV: Training Benefits and Incentives
+
+4.1 CAREER ADVANCEMENT:
+- Training completion mandatory for promotion
+- Specialized training for key positions
+- International training opportunities
+- Research and publication encouragement
+- Innovation and best practice recognition
+
+4.2 TRAINING LEAVE AND BENEFITS:
+- Special training leave with full pay
+- Travel and accommodation provided
+- Study material and resources
+- Networking opportunities with peers
+- Certificate and skill recognition
+
+This comprehensive training framework ensures continuous capacity building and professional development of government employees for efficient public service delivery.""",
+        "filename": "training_capacity_building.txt",
+        "source": "Government Training Policy Framework 2024"
+    }
+]

setup_documents.py

@@ -5,6 +5,7 @@ import os
 from lancedb_service import lancedb_service
 import asyncio
 import logging
+from comprehensive_documents import COMPREHENSIVE_DOCUMENTS
 
 logger = logging.getLogger("voicebot")
 
@@ -309,13 +310,16 @@ This system ensures competitive pricing, quality products, and supports domestic
 ]
 
 async def setup_sample_documents():
-    """Create sample documents in the database"""
+    """Create comprehensive government documents in the database"""
     try:
-        logger.info("🗂️ Setting up sample government documents...")
+        logger.info("🗂️ Setting up comprehensive government documents...")
+        
+        # Combine sample documents with comprehensive documents for better coverage
+        all_documents = SAMPLE_DOCUMENTS + COMPREHENSIVE_DOCUMENTS
         
         # Create document objects that match LanceDB expectations
         docs = []
-        for doc_data in SAMPLE_DOCUMENTS:
+        for doc_data in all_documents:
             # Create a simple document object with the expected attributes
             doc = type('Document', (), {
                 'page_content': doc_data["content"],
@@ -351,9 +355,9 @@ async def setup_sample_documents():
                 tbl = lancedb_service.db.open_table("rajasthan_documents")
                 tbl.delete("id = 'sample'")
                 
-                # Now add the real documents
-                rajasthan_docs = []
-                for doc_data in SAMPLE_DOCUMENTS:
+                # Now add all documents (sample + comprehensive)
+                rajasthan_docs = []  
+                for doc_data in all_documents:
                     embedding = lancedb_service.embedding_model.embed_query(doc_data["content"])
                     raj_doc = {
                         "id": str(uuid.uuid4()),
@@ -369,7 +373,7 @@ async def setup_sample_documents():
         except Exception as e:
             logger.warning(f"⚠️ Could not setup rajasthan_documents table: {e}")
             
-        logger.info("✅ Sample documents added successfully")
+        logger.info(f"✅ Added {len(all_documents)} comprehensive government documents successfully")
         
     except Exception as e:
         logger.error(f"❌ Error setting up documents: {e}")