# migrations

dir: embeddings_queue
filename: 00001-embeddings.sqlite.sql
hash: d3755dfd232be8e8301f4d7fcfb3a486
sql:
CREATE TABLE embeddings_queue (
    seq_id INTEGER PRIMARY KEY,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    operation INTEGER NOT NULL,
    topic TEXT NOT NULL,
    id TEXT NOT NULL,
    vector BLOB,
    encoding TEXT,
    metadata TEXT
);


dir: sysdb
filename: 00001-collections.sqlite.sql
hash: 38352d725ad1c16074fac420b22b4633
sql:
CREATE TABLE collections (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    topic TEXT NOT NULL,
    UNIQUE (name)
);

CREATE TABLE collection_metadata (
    collection_id TEXT REFERENCES collections(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (collection_id, key)
);


dir: sysdb
filename: 00002-segments.sqlite.sql
hash: 2913cb6a503055a95f625448037e8912
sql:
CREATE TABLE segments (
    id TEXT PRIMARY KEY,
    type TEXT NOT NULL,
    scope TEXT NOT NULL,
    topic TEXT,
    collection TEXT REFERENCES collection(id)
);

CREATE TABLE segment_metadata (
    segment_id TEXT  REFERENCES segments(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (segment_id, key)
);


dir: sysdb
filename: 00004-tenants-databases.sqlite.sql
hash: 048867ce8fcdefe4023c7110e4433591
sql:
CREATE TABLE IF NOT EXISTS tenants (
    id TEXT PRIMARY KEY,
    UNIQUE (id)
);

CREATE TABLE IF NOT EXISTS databases (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per tenant
    tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
    UNIQUE (tenant_id, name) -- Ensure that a tenant has only one database with a given name
);

CREATE TABLE IF NOT EXISTS collections_tmp (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per database
    topic TEXT NOT NULL,
    dimension INTEGER,
    database_id TEXT NOT NULL REFERENCES databases(id) ON DELETE CASCADE,
    UNIQUE (name, database_id)
);

-- Create default tenant and database
INSERT OR REPLACE INTO tenants (id) VALUES ('default_tenant'); -- The default tenant id is 'default_tenant' others are UUIDs
INSERT OR REPLACE INTO databases (id, name, tenant_id) VALUES ('00000000-0000-0000-0000-000000000000', 'default_database', 'default_tenant');

INSERT OR REPLACE INTO collections_tmp (id, name, topic, dimension, database_id)
    SELECT id, name, topic, dimension, '00000000-0000-0000-0000-000000000000' FROM collections;
DROP TABLE collections;
ALTER TABLE collections_tmp RENAME TO collections;


dir: sysdb
filename: 00005-remove-topic.sqlite.sql
hash: b1367c826b8fba5f96f27befdc1d42d2
sql:
-- Remove the topic column from the Collections and Segments tables

ALTER TABLE collections DROP COLUMN topic;
ALTER TABLE segments DROP COLUMN topic;


dir: sysdb
filename: 00006-collection-segment-metadata.sqlite.sql
hash: 4eea7468935bf25d4604a0fed2366116
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE collection_metadata ADD COLUMN bool_value INTEGER;
ALTER TABLE segment_metadata ADD COLUMN bool_value INTEGER;


dir: sysdb
filename: 00007-collection-config.sqlite.sql
hash: 1c7e63bba346a42a18b6ab7f1c989bed
sql:
-- Stores collection configuration dictionaries.
ALTER TABLE collections ADD COLUMN config_json_str TEXT;


dir: sysdb
filename: 00008-maintenance-log.sqlite.sql
hash: 0a0e7e93111a01789addf64961c6127c
sql:
-- Records when database maintenance operations are performed.
-- At time of creation, this table is only used to record vacuum operations.
CREATE TABLE maintenance_log (
  id INT PRIMARY KEY,
  timestamp INT NOT NULL,
  operation TEXT NOT NULL
);


dir: sysdb
filename: 00009-segment-collection-not-null.sqlite.sql
hash: 054355aef9e63702bf54ea29e61563f1
sql:
-- This makes segments.collection non-nullable.
CREATE TABLE segments_temp (
    id TEXT PRIMARY KEY,
    type TEXT NOT NULL,
    scope TEXT NOT NULL,
    collection TEXT REFERENCES collection(id) NOT NULL
);

INSERT INTO segments_temp SELECT * FROM segments;
DROP TABLE segments;
ALTER TABLE segments_temp RENAME TO segments;


dir: metadb
filename: 00001-embedding-metadata.sqlite.sql
hash: 2b4cf52c4bb2676e21d6860a4409f856
sql:
CREATE TABLE embeddings (
    id INTEGER PRIMARY KEY,
    segment_id TEXT NOT NULL,
    embedding_id TEXT NOT NULL,
    seq_id BLOB NOT NULL,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    UNIQUE (segment_id, embedding_id)
);

CREATE TABLE embedding_metadata (
    id INTEGER REFERENCES embeddings(id),
    key TEXT NOT NULL,
    string_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (id, key)
);

CREATE TABLE max_seq_id (
    segment_id TEXT PRIMARY KEY,
    seq_id BLOB NOT NULL
);

CREATE VIRTUAL TABLE embedding_fulltext USING fts5(id, string_value);


dir: metadb
filename: 00002-embedding-metadata.sqlite.sql
hash: 12a570f7121b3a8ce750a2a7c36da20f
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE embedding_metadata ADD COLUMN bool_value INTEGER


dir: metadb
filename: 00003-full-text-tokenize.sqlite.sql
hash: f97ad6334aeaa8f419f01110b648b97a
sql:
CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value, tokenize='trigram');
INSERT INTO embedding_fulltext_search (rowid, string_value) SELECT rowid, string_value FROM embedding_metadata;
DROP TABLE embedding_fulltext;


dir: metadb
filename: 00004-metadata-indices.sqlite.sql
hash: fb36603a45ee2cd0254cef3ef86585e8
sql:
CREATE INDEX IF NOT EXISTS embedding_metadata_int_value ON embedding_metadata (key, int_value) WHERE int_value IS NOT NULL;
CREATE INDEX IF NOT EXISTS embedding_metadata_float_value ON embedding_metadata (key, float_value) WHERE float_value IS NOT NULL;
CREATE INDEX IF NOT EXISTS embedding_metadata_string_value ON embedding_metadata (key, string_value) WHERE string_value IS NOT NULL;


# embeddings_queue

created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: 198bc61f-560a-4790-a7b0-f9c3e72b8ec8
encoding: FLOAT32
metadata:
{"source": "pdf/temp_hume_documentation.pdf", "chroma:document": "Hume.ai\n\nHume.ai is an AI voice system designed to respond empathetically. It has applications in healthcare for monitoring patients and mental health conditions, as well as in schools for monitoring students to help them focus on classes and create emotionally intelligent chatbots. Hume.ai measures 53 expressions through emotional language and 48 expressions through facial cues, vocal bursts, and speech prosody.\n\nEmotional Language: The words we say and the emotions they convey. \u2022 Facial Cues: Facial expressions. \u2022 Vocal Bursts: Non-linguistic vocal utterances like sighs, ohs, ahs, umms, etc. \u2022 Speech Prosody: The way we use words while speaking.\n\nFeatures:\n\n1. Empathic Voice Interface (EVI) 2. Expression Measurement API 3. Custom Model API\n\n1. Empathic Voice Interface (EVI)\n\nHume's Empathic Voice Interface (EVI) is an emotionally intelligent AI that takes audio input and returns both audio and a transcript. Based on tone and rhythm, EVI generates empathetic responses."}


created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: 14047b8b-b3dc-45b2-92a2-ffc7830a60ef
encoding: FLOAT32
metadata:
{"source": "pdf/temp_hume_documentation.pdf", "chroma:document": "Hume's Empathic Voice Interface (EVI) is an emotionally intelligent AI that takes audio input and returns both audio and a transcript. Based on tone and rhythm, EVI generates empathetic responses.\n\nConnection: A WebSocket connection is established using an API key and sometimes a WSS URI. Once connected, users can start speaking through their device. Audio is sent to EVI via WebSocket, and EVI returns transcribed text, vocal expression measurements, generated text, and audio responses.\n\nResponses:\n\nThe text of EVI\u2019s reply \u2022 EVI\u2019s expressive audio response \u2022 A transcript of the user\u2019s message with vocal expression measures \u2022 Messages if the user interrupts EVI \u2022 Notifications when EVI finishes responding \u2022 Error messages if issues arise\n\n2. Expression Measurement API\n\nHume's Expression Measurement API is a cutting-edge tool for capturing and analyzing human expressions across different modalities.\n\n\n\nInput Processing: The API can analyze various types of media:"}


created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: ca249cb6-a83d-4b95-871c-fbd6c4b298ea
encoding: FLOAT32
metadata:
{"source": "pdf/temp_hume_documentation.pdf", "chroma:document": "Hume's Expression Measurement API is a cutting-edge tool for capturing and analyzing human expressions across different modalities.\n\n\n\nInput Processing: The API can analyze various types of media:\n\nAudio files for vocal expressions \u2022 Video files for facial expressions \u2022 Images for facial expressions \u2022 Text for linguistic emotional content\n\nMeasurement Capabilities:\n\nFacial Expression: Captures 48 dimensions of emotional meaning, including subtle facial movements. \u2022 Speech Prosody: Analyzes the tone, rhythm, and timbre of speech across 48 dimensions. \u2022 Vocal Burst: Identifies and analyzes vocal sounds (e.g., laughs, sighs, cries) along 48 dimensions. \u2022 Emotional Language: Evaluates the emotional tone of transcribed text across 53 dimensions.\n\nProcessing Methods:\n\nBatch Processing: For analyzing large volumes of files asynchronously. \u2022 Real-time Streaming: For continuous, real-time analysis of data streams.\n\n3. Custom Model API"}


created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: e6e4358c-c27a-4e07-95e2-6394b1efc24e
encoding: FLOAT32
metadata:
{"source": "pdf/temp_hume_documentation.pdf", "chroma:document": "Processing Methods:\n\nBatch Processing: For analyzing large volumes of files asynchronously. \u2022 Real-time Streaming: For continuous, real-time analysis of data streams.\n\n3. Custom Model API\n\nThe Custom Model API integrates patterns of language, vocal expression, and facial expression to predict human preferences and needs more accurately than traditional language models. This API allows users to create custom multimodal models tailored to specific applications, such as predicting well-being, satisfaction, mental health, and more.\n\nWork Summary:\n\n1. Integrating EVI with Expression Measurement API:"}


created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: 4bb94cba-c0e2-447e-b2c0-f4b267139a07
encoding: FLOAT32
metadata:
{"source": "pdf/temp_hume_documentation.pdf", "chroma:document": "Work Summary:\n\n1. Integrating EVI with Expression Measurement API:\n\nIntegrated EVI with the Expression Measurement API to enable EVI to understand both facial and vocal expressions. The camera detected facial expressions while EVI detected vocal expressions simultaneously. We explored features in Hume.ai but found no built-in functionality for this integration. Attempts to connect them through prompts were unsuccessful, as prompts are only sent at the start of a conversation and do not update with each new expression. Ultimately, we concluded that EVI and the Expression Measurement API cannot work together as intended. 2. Connecting RAG with EVI:"}


created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: e814990c-a490-4448-9f83-d7146e12c32d
encoding: FLOAT32
metadata:
{"source": "pdf/temp_hume_documentation.pdf", "chroma:document": "Attempted to connect RAG to EVI. During this process, it became clear that audio is converted to bytes and sent to EVI via WebSocket, making it impossible to add RAG in between this process. After consulting with the Hume.ai team on Discord, we learned that to make EVI work according to our requirements, we would need to train our own model using the Custom Model API."}


created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: 5b641159-bb3f-43b9-bdb3-5888f5bbcfb5
encoding: FLOAT32
metadata:
{"source": "pdf/chromadb.pdf", "chroma:document": "Chromadb Documentation\n\nOverview\n\nChromadb consists of two primary databases:\n\n1. Correction Chromadb: Stores doctor instructions or corrections for patient records. 2. Books Chromadb: Manages segmented documents from books or PDFs for medical analysis.\n\nCorrection Chromadb\n\nInput Specifications\n\nDocuments: Contains patient information and specific sections needing corrections or\n\ninstructions from doctors.\n\nEmbeddings: Utilizes HuggingfaceEmbeddings model \"sentence-transformers/all-mpnet-\n\nbase-v2\" for generating document embeddings.\n\nOutput Specifications\n\nRetrieves the latest entered doctor instructions or corrections based on patient ID.\n\nProcess\n\nDocuments submitted to Correction Chromadb are saved with embeddings generated using\n\nthe specified Huggingface model.\n\nReturns the most recent instruction or correction entered.\n\nImprovements\n\nDuplicate Prevention: Ensures no duplicate entries by checking existing page content in the\n\ncollection before saving."}


created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: 8f49db94-1749-418c-8dba-c89c51101420
encoding: FLOAT32
metadata:
{"source": "pdf/chromadb.pdf", "chroma:document": "Returns the most recent instruction or correction entered.\n\nImprovements\n\nDuplicate Prevention: Ensures no duplicate entries by checking existing page content in the\n\ncollection before saving.\n\nSeparate Collections: Each doctor has a dedicated collection in Chromadb identified by\n\ndoctor ID, ensuring organizational clarity and data segregation.\n\nBooks Chromadb\n\nInput Specifications\n\nDocuments: Segmented chunks of PDFs or documents, with a chunk size of 1000 and\n\noverlap of 200.\n\nEmbeddings: Uses HuggingfaceEmbeddings model \"sentence-transformers/all-mpnet-base-\n\nv2\" for embedding generation.\n\nOutput Specifications\n\nDetermines whether a document indicates a Urinary Tract Infection (UTI) and identifies the\n\ntype.\n\nProvides medication recommendations based on the identified UTI type.\n\nProcess\n\nStores segmented PDFs/documents in Chromadb for retrieval and analysis. \u2022 Retrieves data directly from stored books in the system."}


created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: 84efc23a-66c9-4737-a92f-329877ba12a1
encoding: FLOAT32
metadata:
{"source": "pdf/GPUs.pdf", "chroma:document": "RAG\n\nFor a task that requires Retrieval-Augmented Generation (RAG) to provide responses within 10 seconds, you need a GPU that offers high performance for both retrieval and generation tasks. Based on the need for rapid response times and efficient processing, here are some suitable GPUs:\n\n### Recommended GPUs\n\n1. **NVIDIA A100**: - **Pros**: - Exceptional performance for both AI training and inference. - Large memory capacity (up to 80 GB HBM2e) which is beneficial for handling large datasets and complex models. - Superior compute power with Tensor Cores optimized for AI workloads. - **Cons**: - High cost. - Requires a compatible data center environment.\n\n2. **NVIDIA V100**: - **Pros**: - High performance with up to 32 GB of HBM2 memory. - Good for AI training and inference tasks. - **Cons**: - Still expensive but less so than the A100. - Older architecture compared to A100."}


created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: d50a05aa-586d-426a-905e-4fa3f09bf3d7
encoding: FLOAT32
metadata:
{"source": "pdf/GPUs.pdf", "chroma:document": "3. **NVIDIA RTX 3090**: - **Pros**: - Excellent performance for both AI tasks and general computing. - 24 GB of GDDR6X memory, which is substantial for many RAG applications. - More accessible for non-data center environments. - **Cons**: - Less memory and AI-specific optimizations compared to A100 and V100. - Still relatively high cost.\n\n4. **NVIDIA RTX 3080 Ti**: - **Pros**: - High performance with a more reasonable cost compared to the 3090. - 12 GB of GDDR6X memory, sufficient for many tasks. - **Cons**: - Lower memory capacity compared to the 3090 and higher-end GPUs.\n\n5. **NVIDIA T4**: - **Pros**: - Designed for AI inference with good performance and energy efficiency. - 16 GB of GDDR6 memory. - Suitable for data center environments with lower power consumption. - **Cons**: - Less powerful compared to the A100 and V100, but still capable for many inference tasks. ### Conclusion"}


created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: cb20da9b-2a88-418c-badc-a84423790b92
encoding: FLOAT32
metadata:
{"source": "pdf/GPUs.pdf", "chroma:document": "**Best Choice for 10-Second RAG Response**: - **NVIDIA A100**: If budget allows and you require the best performance, the A100 is the top choice for ensuring that RAG operations complete within 10 seconds, especially for complex and large-scale tasks.\n\n**Balanced Choice**: - **NVIDIA RTX 3090**: If you need a balance between cost and performance, the RTX 3090 offers substantial power and memory, making it suitable for fast RAG operations within the desired response time.\n\n**Cost-Effective Choice**: - **NVIDIA T4**: If budget and energy efficiency are major concerns and the RAG workload is less demanding, the T4 can be a cost-effective solution while still providing good performance for inference tasks."}


created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: 2f001991-fc04-4d6b-9e85-3705b53e314e
encoding: FLOAT32
metadata:
{"source": "pdf/GPUs.pdf", "chroma:document": "Choosing the right GPU depends on the specific details of your RAG workload, including the size of your models, the complexity of the queries, and the scale of your deployment. For the most demanding scenarios, the A100 would be ideal, but for many practical applications, the RTX 3090 or even the T4 might be sufficient.\n\nCost Summary\n\nNVIDIA A100: $10,000 - $15,000 \u2022 NVIDIA V100: $8,000 - $12,000 \u2022 NVIDIA RTX 3090: $1,500 - $2,000 \u2022 NVIDIA RTX 3080 Ti: $1,200 - $1,500 \u2022 NVIDIA T4: $1,000 - $2,000\n\nTo use GPUs like the NVIDIA A100 in cloud platforms such as Azure, AWS, and Google Cloud for Retrieval- Augmented Generation (RAG) operations, here are the approximate costs:\n\n### AWS - **NVIDIA A100**: Around $4.10 per hour for on-demand instances. - **NVIDIA T4**: Approximately $0.35 per hour for on-demand instances."}


created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: 7d4e5891-c2c2-4719-b3ba-74b1e6a8b585
encoding: FLOAT32
metadata:
{"source": "pdf/GPUs.pdf", "chroma:document": "### AWS - **NVIDIA A100**: Around $4.10 per hour for on-demand instances. - **NVIDIA T4**: Approximately $0.35 per hour for on-demand instances.\n\n### Azure - **NVIDIA A100**: The pricing for Azure NVv4-series, which includes A100 GPUs, starts at about $3.86 per hour. - **NVIDIA T4**: Azure's pricing for NCasT4_v3-series, which uses T4 GPUs, is around $0.40 per hour.\n\n### Google Cloud - **NVIDIA A100**: - **a2-highgpu-1g (1 A100 40GB GPU)**: Around $2.75 to $3.75 per hour depending on the region for on-demand instances. - **a2-highgpu-8g (8 A100 40GB GPUs)**: Around $29.39 to $34.60 per hour for on-demand instances, with cheaper rates available for preemptible instances."}


created_at: 2024-10-10 16:52:44
topic: persistent://default/default/7c6c78ed-e18b-4d98-8086-afedae1c70c3
id: cae82282-7384-4376-84a7-d0046249a654
encoding: FLOAT32
metadata:
{"source": "pdf/GPUs.pdf", "chroma:document": "These costs are estimates and can vary based on the region and specific configurations. For the most demanding RAG tasks that need responses within 10 seconds, the NVIDIA A100 is the best choice due to its high performance. If cost efficiency is a priority and your workload is less intensive, the NVIDIA T4 is a more affordable option while still providing good performance."}


# collections

id: 7c6c78ed-e18b-4d98-8086-afedae1c70c3
name: langchain
database_id: 00000000-0000-0000-0000-000000000000
config_json_str:
{"hnsw_configuration": {"space": "l2", "ef_construction": 100, "ef_search": 10, "num_threads": 20, "M": 16, "resize_factor": 1.2, "batch_size": 100, "sync_threshold": 1000, "_type": "HNSWConfigurationInternal"}, "_type": "CollectionConfigurationInternal"}


# embedding_metadata

key: chroma:document
string_value:
Hume.ai

Hume.ai is an AI voice system designed to respond empathetically. It has applications in healthcare for monitoring patients and mental health conditions, as well as in schools for monitoring students to help them focus on classes and create emotionally intelligent chatbots. Hume.ai measures 53 expressions through emotional language and 48 expressions through facial cues, vocal bursts, and speech prosody.

Emotional Language: The words we say and the emotions they convey. • Facial Cues: Facial expressions. • Vocal Bursts: Non-linguistic vocal utterances like sighs, ohs, ahs, umms, etc. • Speech Prosody: The way we use words while speaking.

Features:

1. Empathic Voice Interface (EVI) 2. Expression Measurement API 3. Custom Model API

1. Empathic Voice Interface (EVI)

Hume's Empathic Voice Interface (EVI) is an emotionally intelligent AI that takes audio input and returns both audio and a transcript. Based on tone and rhythm, EVI generates empathetic responses.


key: chroma:document
string_value:
Hume's Empathic Voice Interface (EVI) is an emotionally intelligent AI that takes audio input and returns both audio and a transcript. Based on tone and rhythm, EVI generates empathetic responses.

Connection: A WebSocket connection is established using an API key and sometimes a WSS URI. Once connected, users can start speaking through their device. Audio is sent to EVI via WebSocket, and EVI returns transcribed text, vocal expression measurements, generated text, and audio responses.

Responses:

The text of EVI’s reply • EVI’s expressive audio response • A transcript of the user’s message with vocal expression measures • Messages if the user interrupts EVI • Notifications when EVI finishes responding • Error messages if issues arise

2. Expression Measurement API

Hume's Expression Measurement API is a cutting-edge tool for capturing and analyzing human expressions across different modalities.



Input Processing: The API can analyze various types of media:


key: chroma:document
string_value:
Hume's Expression Measurement API is a cutting-edge tool for capturing and analyzing human expressions across different modalities.



Input Processing: The API can analyze various types of media:

Audio files for vocal expressions • Video files for facial expressions • Images for facial expressions • Text for linguistic emotional content

Measurement Capabilities:

Facial Expression: Captures 48 dimensions of emotional meaning, including subtle facial movements. • Speech Prosody: Analyzes the tone, rhythm, and timbre of speech across 48 dimensions. • Vocal Burst: Identifies and analyzes vocal sounds (e.g., laughs, sighs, cries) along 48 dimensions. • Emotional Language: Evaluates the emotional tone of transcribed text across 53 dimensions.

Processing Methods:

Batch Processing: For analyzing large volumes of files asynchronously. • Real-time Streaming: For continuous, real-time analysis of data streams.

3. Custom Model API


key: chroma:document
string_value:
Processing Methods:

Batch Processing: For analyzing large volumes of files asynchronously. • Real-time Streaming: For continuous, real-time analysis of data streams.

3. Custom Model API

The Custom Model API integrates patterns of language, vocal expression, and facial expression to predict human preferences and needs more accurately than traditional language models. This API allows users to create custom multimodal models tailored to specific applications, such as predicting well-being, satisfaction, mental health, and more.

Work Summary:

1. Integrating EVI with Expression Measurement API:


key: chroma:document
string_value:
Work Summary:

1. Integrating EVI with Expression Measurement API:

Integrated EVI with the Expression Measurement API to enable EVI to understand both facial and vocal expressions. The camera detected facial expressions while EVI detected vocal expressions simultaneously. We explored features in Hume.ai but found no built-in functionality for this integration. Attempts to connect them through prompts were unsuccessful, as prompts are only sent at the start of a conversation and do not update with each new expression. Ultimately, we concluded that EVI and the Expression Measurement API cannot work together as intended. 2. Connecting RAG with EVI:


key: chroma:document
string_value:
Attempted to connect RAG to EVI. During this process, it became clear that audio is converted to bytes and sent to EVI via WebSocket, making it impossible to add RAG in between this process. After consulting with the Hume.ai team on Discord, we learned that to make EVI work according to our requirements, we would need to train our own model using the Custom Model API.


key: chroma:document
string_value:
Chromadb Documentation

Overview

Chromadb consists of two primary databases:

1. Correction Chromadb: Stores doctor instructions or corrections for patient records. 2. Books Chromadb: Manages segmented documents from books or PDFs for medical analysis.

Correction Chromadb

Input Specifications

Documents: Contains patient information and specific sections needing corrections or

instructions from doctors.

Embeddings: Utilizes HuggingfaceEmbeddings model "sentence-transformers/all-mpnet-

base-v2" for generating document embeddings.

Output Specifications

Retrieves the latest entered doctor instructions or corrections based on patient ID.

Process

Documents submitted to Correction Chromadb are saved with embeddings generated using

the specified Huggingface model.

Returns the most recent instruction or correction entered.

Improvements

Duplicate Prevention: Ensures no duplicate entries by checking existing page content in the

collection before saving.


key: chroma:document
string_value:
Returns the most recent instruction or correction entered.

Improvements

Duplicate Prevention: Ensures no duplicate entries by checking existing page content in the

collection before saving.

Separate Collections: Each doctor has a dedicated collection in Chromadb identified by

doctor ID, ensuring organizational clarity and data segregation.

Books Chromadb

Input Specifications

Documents: Segmented chunks of PDFs or documents, with a chunk size of 1000 and

overlap of 200.

Embeddings: Uses HuggingfaceEmbeddings model "sentence-transformers/all-mpnet-base-

v2" for embedding generation.

Output Specifications

Determines whether a document indicates a Urinary Tract Infection (UTI) and identifies the

type.

Provides medication recommendations based on the identified UTI type.

Process

Stores segmented PDFs/documents in Chromadb for retrieval and analysis. • Retrieves data directly from stored books in the system.


key: chroma:document
string_value:
RAG

For a task that requires Retrieval-Augmented Generation (RAG) to provide responses within 10 seconds, you need a GPU that offers high performance for both retrieval and generation tasks. Based on the need for rapid response times and efficient processing, here are some suitable GPUs:

### Recommended GPUs

1. **NVIDIA A100**: - **Pros**: - Exceptional performance for both AI training and inference. - Large memory capacity (up to 80 GB HBM2e) which is beneficial for handling large datasets and complex models. - Superior compute power with Tensor Cores optimized for AI workloads. - **Cons**: - High cost. - Requires a compatible data center environment.

2. **NVIDIA V100**: - **Pros**: - High performance with up to 32 GB of HBM2 memory. - Good for AI training and inference tasks. - **Cons**: - Still expensive but less so than the A100. - Older architecture compared to A100.


key: chroma:document
string_value:
3. **NVIDIA RTX 3090**: - **Pros**: - Excellent performance for both AI tasks and general computing. - 24 GB of GDDR6X memory, which is substantial for many RAG applications. - More accessible for non-data center environments. - **Cons**: - Less memory and AI-specific optimizations compared to A100 and V100. - Still relatively high cost.

4. **NVIDIA RTX 3080 Ti**: - **Pros**: - High performance with a more reasonable cost compared to the 3090. - 12 GB of GDDR6X memory, sufficient for many tasks. - **Cons**: - Lower memory capacity compared to the 3090 and higher-end GPUs.

5. **NVIDIA T4**: - **Pros**: - Designed for AI inference with good performance and energy efficiency. - 16 GB of GDDR6 memory. - Suitable for data center environments with lower power consumption. - **Cons**: - Less powerful compared to the A100 and V100, but still capable for many inference tasks. ### Conclusion


key: chroma:document
string_value:
**Best Choice for 10-Second RAG Response**: - **NVIDIA A100**: If budget allows and you require the best performance, the A100 is the top choice for ensuring that RAG operations complete within 10 seconds, especially for complex and large-scale tasks.

**Balanced Choice**: - **NVIDIA RTX 3090**: If you need a balance between cost and performance, the RTX 3090 offers substantial power and memory, making it suitable for fast RAG operations within the desired response time.

**Cost-Effective Choice**: - **NVIDIA T4**: If budget and energy efficiency are major concerns and the RAG workload is less demanding, the T4 can be a cost-effective solution while still providing good performance for inference tasks.


key: chroma:document
string_value:
Choosing the right GPU depends on the specific details of your RAG workload, including the size of your models, the complexity of the queries, and the scale of your deployment. For the most demanding scenarios, the A100 would be ideal, but for many practical applications, the RTX 3090 or even the T4 might be sufficient.

Cost Summary

NVIDIA A100: $10,000 - $15,000 • NVIDIA V100: $8,000 - $12,000 • NVIDIA RTX 3090: $1,500 - $2,000 • NVIDIA RTX 3080 Ti: $1,200 - $1,500 • NVIDIA T4: $1,000 - $2,000

To use GPUs like the NVIDIA A100 in cloud platforms such as Azure, AWS, and Google Cloud for Retrieval- Augmented Generation (RAG) operations, here are the approximate costs:

### AWS - **NVIDIA A100**: Around $4.10 per hour for on-demand instances. - **NVIDIA T4**: Approximately $0.35 per hour for on-demand instances.


key: chroma:document
string_value:
### AWS - **NVIDIA A100**: Around $4.10 per hour for on-demand instances. - **NVIDIA T4**: Approximately $0.35 per hour for on-demand instances.

### Azure - **NVIDIA A100**: The pricing for Azure NVv4-series, which includes A100 GPUs, starts at about $3.86 per hour. - **NVIDIA T4**: Azure's pricing for NCasT4_v3-series, which uses T4 GPUs, is around $0.40 per hour.

### Google Cloud - **NVIDIA A100**: - **a2-highgpu-1g (1 A100 40GB GPU)**: Around $2.75 to $3.75 per hour depending on the region for on-demand instances. - **a2-highgpu-8g (8 A100 40GB GPUs)**: Around $29.39 to $34.60 per hour for on-demand instances, with cheaper rates available for preemptible instances.


key: chroma:document
string_value:
These costs are estimates and can vary based on the region and specific configurations. For the most demanding RAG tasks that need responses within 10 seconds, the NVIDIA A100 is the best choice due to its high performance. If cost efficiency is a priority and your workload is less intensive, the NVIDIA T4 is a more affordable option while still providing good performance.


