Harnessing Practical Ecommerce Data Scenarios with OpenAI API and ChromaDB Integration

%load_ext dotenv
%dotenv
import os
OPENAI_TOKEN = os.environ.get("OPENAI_TOKEN")
print('')

from openai import OpenAI
import json

def prettyprint(input_dict):
    return print(json.dumps(input_dict, indent=4))

# Define a create_embeddings function
def create_embeddings(texts):
    response = client.embeddings.create(
    model="text-embedding-ada-002",
    input=texts
    )
    response_dict = response.model_dump()
  
    return [data['embedding'] for data in response_dict['data']]

# Define a function to combine the relevant features into a single string
def create_sample_product_text(product):
    return f"""Title: {product['title']}
    Description: {product['description']}
    Category: {product['category']}
    Subcategory: {product['subcategory']}
    Features: {product['features']}"""

# Generate sample product data for home & garden retailers
import json
import random

# Generate fantasy brand names
def generate_fantasy_brand_names(num_names):
    fantasy_brand_names = set()
    prefixes = ["Aero", "Aqua", "Nova", "Vortex", "Frost", "Eclipse", "Celestial", "Serenity", "Mythic", "Zenith"]
    suffixes = ["Tech", "Glo", "Luxe", "Co", "Works", "Innovations", "Industries", "Solutions", "Studios", "Designs"]
    
    while len(fantasy_brand_names) < num_names:
        brand_name = random.choice(prefixes) + random.choice(suffixes)
        fantasy_brand_names.add(brand_name)
    
    return list(fantasy_brand_names)


def generate_product_data(num_products):
    product_data = []
    product_names = set()
    # Sample product categories and subcategories for home & garden
    categories = ["Home Decor", "Furniture", "Kitchen & Dining", "Outdoor Living", "Tools & Home Improvement"]
    subcategories = {
        "Home Decor": ["Wall Art", "Throw Pillows", "Rugs", "Candles & Holders"],
        "Furniture": ["Sofas & Couches", "Tables", "Chairs", "Bedroom Furniture"],
        "Kitchen & Dining": ["Cookware", "Dinnerware", "Cutlery", "Appliances"],
        "Outdoor Living": ["Patio Furniture", "Grills & Outdoor Cooking", "Outdoor Decor", "Gardening"],
        "Tools & Home Improvement": ["Power Tools", "Hand Tools", "Lighting", "Storage & Organization"]
    }

    for i in range(num_products):
        # Generate unique product name
        product_name = ""
        #while product_name in product_names:
        category = random.choice(categories)
        subcategory = random.choice(subcategories[category])
        adjective = random.choice(["Premium", "Stylish", "Modern", "Classic", "Robust"])
        noun = random.choice(["Sofa", "Table", "Chair", "Cookware Set", "Patio Furniture", "Power Drill"])
        brand = generate_fantasy_brand_names(1)[0]
        product_name = f"{brand} {adjective} {subcategory} {noun}"
        
        product_names.add(product_name)

        # Generate product description
        description = " ".join([random.choice(["High-quality", "Durable", "Fashionable", "Functional", "Innovative"]) 
                                for _ in range(10)])
        description += f" {random.choice(['This', 'That', 'The', 'A'])} {noun} is perfect for "
        description += " ".join([random.choice(["everyday use", "enhancing your home decor", "outdoor entertaining", 
                                                 "DIY projects"]) for _ in range(5)])

        # Generate product features
        features = {
            "Material": random.choice(["Wood", "Metal", "Glass", "Plastic", "Stainless Steel"]),
            "Color": random.choice(["White", "Black", "Gray", "Brown", "Blue", "Red"]),
            "Size": random.choice(["Small", "Medium", "Large"]),
            "Weight": round(random.uniform(5, 50), 2),
            "Dimensions": f"{round(random.uniform(10, 200), 2)} x {round(random.uniform(10, 200), 2)} x {round(random.uniform(10, 200), 2)} cm"
        }

        # Generate other properties (e.g., price, category)
        #price = round(random.uniform(20, 500), 2)
        #category = category

        # Store product data
        product_data.append({
            "title": product_name,
            "description": description,
            "category": category,
            "subcategory": subcategory,
            "features": features
        })

    return product_data

# Generate sample product data with 200 products
sample_product_data = generate_product_data(200)

# Save the sample product data as a dictionary
with open("sample_product_data_home_garden.json", "w") as file:
    json.dump(sample_product_data, file)

print("Sample product data generated and saved.")

sample_product_data[0:2]

import scipy
def find_n_closest(query_vector, embeddings, n=3):
    distances = []
    for index, embedding in enumerate(embeddings):
    
        # Calculate the cosine distance between the query vector and embedding
        dist = scipy.spatial.distance.cosine(query_vector, embedding)
        # Append the distance and index to distances
        distances.append({"distance": dist, "index": index})
        # Sort distances by the distance key
        distances_sorted = sorted(distances, key=lambda x: x["distance"])
        # Return the first n elements in distances_sorted
    return distances_sorted[0:n]

import openai

# Set your API key
client = OpenAI(api_key = OPENAI_TOKEN)

# Load sample product data
with open("sample_product_data_home_garden.json", "r") as file:
    sample_product_data = json.load(file)

# Combine the features for each product
product_texts = [create_sample_product_text(product) for product in sample_product_data]

# Create the embeddings from product_texts
product_embeddings = create_embeddings(product_texts)

# Create the query vector from query_text
query_text = "Looking for robust Garden BBQ Furniture"
#query_text = "Looking for a stylish sofa to add elegance to my living room"

query_vector = create_embeddings(query_text)[0]

# Find the five closest distances
hits = find_n_closest(query_vector, product_embeddings, n=5)
hits

print(f'Search results for query "{query_text}":\n')
for hit in hits:
    # Extract the product at each index in hits
    product = sample_product_data[hit['index']]
    print(product["title"])

import seaborn as sns
sns.set_style('darkgrid')
import matplotlib.pyplot as plt

distances = [item['distance'] for item in hits ]
titles = [sample_product_data[i]['title'] for i in [item['index'] for item in hits ]]
sns.barplot(y=titles, x=distances)
plt.xlabel('cosine distances (lower is better)')
plt.ylabel('Products ranked')
plt.title(f'Most similar products of the search query "{query_text}"')
plt.show()

last_product = {'title': 'Building Blocks Deluxe Set',
 'short_description': 'Unleash your creativity with this deluxe set of building blocks for endless fun.',
 'price': 34.99,
 'category': 'Toys',
 'features': ['Includes 500+ colorful building blocks',
  'Promotes STEM learning and creativity',
  'Compatible with other major brick brands',
  'Comes with a durable storage container',
  'Ideal for children ages 3 and up']}

# Combine the features for last_product and each product in products
last_product_text = create_product_text(last_product)
product_texts = [create_product_text(product) for product in products]

# Embed last_product_text and product_texts
last_product_embeddings = create_embeddings(last_product_text)[0]
product_embeddings = create_embeddings(product_texts)

# Find the three smallest cosine distances and their indexes
hits = find_n_closest(last_product_embeddings, product_embeddings)

for hit in hits:
    product = products[hit['index']]
    print(product['title'])

user_history = [{'title': 'Remote-Controlled Dinosaur Toy',
  'short_description': 'Roar into action with this remote-controlled dinosaur toy with lifelike movements.',
  'price': 49.99,
  'category': 'Toys',
  'features': ['Realistic dinosaur sound effects',
   'Walks and roars like a real dinosaur',
   'Remote control included',
   'Educational and entertaining']},
 {'title': 'Building Blocks Deluxe Set',
  'short_description': 'Unleash your creativity with this deluxe set of building blocks for endless fun.',
  'price': 34.99,
  'category': 'Toys',
  'features': ['Includes 500+ colorful building blocks',
   'Promotes STEM learning and creativity',
   'Compatible with other major brick brands',
   'Comes with a durable storage container',
   'Ideal for children ages 3 and up']}]

import numpy as np

# Prepare and embed the user_history, and calculate the mean embeddings
history_texts = [create_product_text(product) for product in user_history]
history_embeddings = create_embeddings(history_texts)
mean_history_embeddings = np.mean(history_embeddings, axis=0)

# Filter products to remove any in user_history
products_filtered = [product for product in products if product not in user_history]

# Combine product features and embed the resulting texts
product_texts = [create_product_text(product) for product in products_filtered]
product_embeddings = create_embeddings(product_texts)

hits = find_n_closest(mean_history_embeddings, product_embeddings)

for hit in hits:
    product = products_filtered[hit['index']]
    print(product['title'])

sentiments = [{'label': 'Positive'},
              {'label': 'Neutral'},
              {'label': 'Negative'}]

reviews = ["The food was delicious!",
           "The service was a bit slow but the food was good",
           "Never going back!"]

# Create a list of class descriptions from the sentiment labels
class_descriptions = [topic['label'] for topic in sentiments]

# Embed the class_descriptions and reviews
class_embeddings = create_embeddings(class_descriptions)
review_embeddings = create_embeddings(reviews)

# Define a function to return the minimum distance and its index
def find_closest(query_vector, embeddings):
    distances = []
    for index, embedding in enumerate(embeddings):
        dist = scipy.spatial.distance.cosine(query_vector, embedding)
        distances.append({"distance": dist, "index": index})
    return min(distances, key=lambda x: x["distance"])

for index, review in enumerate(reviews):
    # Find the closest distance and its index using find_closest()
    closest = find_closest(review_embeddings[index], class_embeddings)
    # Subset sentiments using the index from closest
    label = sentiments[closest['index']]['label']
    print(f'"{review}" was classified as {label}')

sentiments = [{'label': 'Positive',
               'description': 'A positive restaurant review'},
              {'label': 'Neutral',
               'description':'A neutral restaurant review'},
              {'label': 'Negative',
               'description': 'A negative restaurant review'}]

reviews = ["The food was delicious!",
           "The service was a bit slow but the food was good",
           "Never going back!"]

# Extract and embed the descriptions from sentiments
class_descriptions = [sentiment['description'] for sentiment in sentiments]
class_embeddings = create_embeddings(class_descriptions)
review_embeddings = create_embeddings(reviews)

def find_closest(query_vector, embeddings):
    distances = []
    for index, embedding in enumerate(embeddings):
        dist = scipy.spatial.distance.cosine(query_vector, embedding)
        distances.append({"distance": dist, "index": index})
    return min(distances, key=lambda x: x["distance"])

for index, review in enumerate(reviews):
    closest = find_closest(review_embeddings[index], class_embeddings)
    label = sentiments[closest['index']]['label']
    print(f'"{review}" was classified as {label}')

import chromadb
#import chromadb.utils.embedding_functions as OpenAIEmbedding
import chromadb.utils.embedding_functions as embedding_functions

client = chromadb.PersistentClient()

client.delete_collection("netflix_titles")

# Create a netflix_title collection using the OpenAI Embedding function
collection = client.create_collection(
    name="netflix_titles",
    embedding_function = embedding_functions.OpenAIEmbeddingFunction(api_key = OPENAI_TOKEN)
)

# List the collections
print(client.list_collections())

# importing DictReader class  from csv module
from csv import DictReader

ids = []
documents = []
metas = []

with open('netflix_titles_1000.csv', encoding="utf8") as csvfile:
    reader = DictReader(csvfile)
    for i, row in enumerate(reader):
        ids.append(row['show_id'])
        
        text = f"Title: {row['title']} ({row['type']})\nDescription: {row['description']}\nCategories: {row['listed_in']}"
        documents.append(text)
        
        metatags = {'rating': row["rating"] , 'release_year': int(row["release_year"])}
        metas.append(metatags)
        
        
"""{'ids': ['s999'],
 'embeddings': None,
 'metadatas': [{'rating': 'TV-14', 'release_year': 2021}],
 'documents': ['Title: Searching For Sheela (Movie)\nDescription: Journalists and fans await Ma Anand Sheela as the infamous former Rajneesh commune’s spokesperson returns to India after decades for an interview tour.\nCategories: Documentaries, International Movies'],
 'uris': None,
 'data': None}"""

metas[0:2]

prettyprint(metas[0:2])

import tiktoken

# Load the encoder for the OpenAI text-embedding-ada-002 model
enc = tiktoken.encoding_for_model("text-embedding-ada-002")

# Encode each text in documents and calculate the total tokens
total_tokens = sum(len(enc.encode(text)) for text in documents)

cost_per_1k_tokens = 0.0001

# Display number of tokens and cost
print('Total tokens:', total_tokens)
print('Cost:', cost_per_1k_tokens * total_tokens/1000)

collection.add(ids=ids, documents=documents, metadatas=metas)

# Print the collection size and first ten items
print(f"No. of documents: {collection.count()}")
print(f"First ten documents: {collection.peek()}")

# Retrieve the netflix_titles collection
collection = client.get_collection(
  name = "netflix_titles",
  embedding_function = embedding_functions.OpenAIEmbeddingFunction(api_key = OPENAI_TOKEN)
)

# Query the collection for "films about dogs"
result = collection.query(query_texts=["series about business"], n_results=3)

prettyprint(result)

# Retrieve the netflix_titles collection
collection = client.get_collection(
  name = "netflix_titles",
  embedding_function = embedding_functions.OpenAIEmbeddingFunction(api_key = OPENAI_TOKEN)
) 

reference_ids = ['s999', 's1000']

# Retrieve the documents for the reference_ids
reference_texts = collection.get(ids=reference_ids)["documents"]

# Query using reference_texts
result = collection.query(  query_texts=reference_texts,  n_results=3)


prettyprint(result['documents'])

# Retrieve the netflix_titles collection
collection = client.get_collection(
  name="netflix_titles",
  embedding_function = embedding_functions.OpenAIEmbeddingFunction(api_key = OPENAI_TOKEN)
)

reference_texts = ["children's story about a car", "lions"]

# Query two results using reference_texts
result = collection.query(
  query_texts=reference_texts, 
  n_results=2,
  # Filter for titles with a G rating released before 2019
  where={
    "$and": [
        {"rating": 
        	{"$eq": "G"}
        },
        {"release_year": 
         	{"$lt": 2019}
        }
    ]
  }
)

prettyprint(result['documents'])