Non Fungible Token (NFT) Rarity Analysis with Jaccard Distance

# api market macro stats
# for collection officialtimelessapeclub
# on opensea market
import requests, json

url_stats = "https://api.opensea.io/api/v1/collection/officialtimelessapeclub/stats"

r = requests.get(url_stats)

print(f"floor price:", r.json()["stats"]["floor_price"],f"ETH")
print(f"24h Volume:", r.json()["stats"]["one_day_volume"],f"ETH")
print(f"24h sales:", r.json()["stats"]["one_day_sales"])

print(f"Number of sales in last 30 days", r.json()["stats"]["thirty_day_sales"])
print(f"24h AVG price:", r.json()["stats"]["one_day_average_price"],f"ETH")
print(f"Unique owners:", r.json()["stats"]["num_owners"],f"/", r.json()["stats"]["count"] )

import pandas as pd
asset_traits = pd.read_pickle('asset_traits.pkl') # load prepared df with trait variants of tokens 0 to 4094

asset_traits.head()

# Update 1 - 03-Mar-22
# map trait variants to short number codes before number crunching to save 40%+ processing time

dftest = asset_traits.copy()

# all unique trait variants
u =set()
for col in dftest.iloc[:,1:].columns:
    dftest[col] = dftest[col].fillna(0)
    v = set(dftest[col].unique())
    u = u.union(v)

# map unique code across all cols to all trait variants
mapping = {item:i for i, item in enumerate(u)}

for col in dftest.iloc[:,1:].columns:
    dftest[col] = dftest[col].apply(lambda x: mapping[x])
    
dftest.head()

%%time
# Update 1 - 03-Mar-22
# multiprocessing pool for the cpu intensive jaccard distances calculations

import multiprocessing
#from multiprocessing import Pool
from itertools import repeat
import pandas as pd
import jaccard # moved jaccard distance calculations code as jaccard function to jaccard.py in order to make multiproc work

collection_slug = 'officialtimelessapeclub'

# Divide asset dataframe into n chunks
n = 6 # define the number of processes (cpu cores)
chunk_size = int(asset_traits.shape[0]/n)+1
chunks = [asset_traits.index[i:i + chunk_size] for i in range(0, asset_traits.shape[0], chunk_size)]

# multiprocess pool - sending chunk indices and collection info as args, df opened directly by jaccard.py
with multiprocessing.Pool() as pool:
    result = pool.starmap(jaccard.jaccard, zip(chunks, repeat(collection_slug)))
    
# Concatenate all chunks back to one
jdist = pd.concat(result)

%%time
# original single thread solution

import numpy as np
from scipy.spatial.distance import jaccard
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
# https://stackoverflow.com/questions/40659212/futurewarning-elementwise-comparison-failed-returning-scalar-but-in-the-futur

df2 = asset_traits.copy()

jdist = pd.DataFrame(columns=['token_id','jd_mean'])
    
# calc jaccard dist for each row
for row1 in df2.itertuples():
    jdlist = []
    # calc against all other rows
    for row2 in df2.itertuples():
        if row1.token_id != row2.token_id:
            jd = jaccard( np.array(row1[2:]), np.array(row2[2:]) ) # jaccard distance
            jdlist.append(jd)     
    jdist.loc[row1.token_id] = [row1.token_id, np.mean(jdlist)] # save the mean of jd of row1

# normalize jaccard distances for a score 0 to 1
from sklearn import preprocessing
from sklearn.preprocessing import MinMaxScaler

scaler = MinMaxScaler()
arr = jdist['jd_mean'].to_numpy()
normalized = scaler.fit_transform(arr.reshape(-1, 1))
# write to df
score = pd.DataFrame(columns=['token_id','score'])
score['token_id'] = jdist['token_id']
score['score'] = normalized

from scipy import stats
zscores = stats.zscore(arr)

import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('darkgrid')

plt.figure(figsize=(15,4))
sns.histplot(data=score.score)
plt.title('Normalized scores 0 to 1')
plt.show()
plt.figure(figsize=(15,4))
sns.histplot(data=zscores)
plt.title('z-scores of Jaccard distance means')
plt.show()

# normalized scores as score
# and the arithmetic mean of the original jaccard dissimilarities (distances)
score['jdist_mean'] = jdist.jd_mean
score.head()

score.to_pickle('score.pkl')    # save score df to file

# best score 1
score.sort_values('score', ascending=False)

# let's rank the scores with pandas rank
score['rank'] = score['score'].rank(method='min', ascending=False)
score.sort_values('rank', ascending=True)

#let's check a specific token for its rank
score.loc[['1081']]
# token 1081 is pretty average:

# let us check the current market what rarities are offered at which prices
# for that we have to scrape due to lack of immeadiately available api key 
# which is required for the 'Retrieving assets' api call
# https://docs.opensea.io/reference/getting-assets

import requests
import cloudscraper # can scrape cloudflare sites

scraper = cloudscraper.create_scraper(
    browser={
        'browser': 'firefox',
        'platform': 'windows',
        'mobile': False
    }
)
from bs4 import BeautifulSoup as bs
url = "https://opensea.io/collection/officialtimelessapeclub?search[sortAscending]=true&search[sortBy]=PRICE&search[toggles][0]=BUY_NOW"
r = scraper.get(url).text

# todo: change to selenium to better deal with opensea uses infinity scrolling

import re
market = pd.DataFrame(columns=['token_id', 'price', 'rank', 'price_rank_product'])

html_code = bs(r, 'html.parser')
cards = html_code.find_all('article', class_=re.compile('^AssetSearchList--asset'))

for c in cards:
    price = c.find('div', class_=re.compile('^Price--amount')).getText(strip=True).split("#",1)[0] if c.find('div', class_=re.compile('^Price--amount')) else "none"
    token = c.find('div', class_=re.compile('^AssetCardFooter--name')).getText(strip=True).split("#",1)[1] if c.find('div', class_=re.compile('^AssetCardFooter--name')) else "none"
    
    token_score = tmp.loc[tmp.token_id == token]['rank'].to_numpy()
    
    # todo: change to selenium to better deal with opensea uses infinity scrolling
    if len(token_score) == 1:
        grade = round(float(price) * token_score[0],1)
    else: #lazy loaded cards
        token_score = str(token_score)
        grade = '-----'
    market.loc[len(market.index)] = [token, price, token_score, grade]

market[0:32].sort_values('price')

# voila, pick the best deals
market[0:32].sort_values('price_rank_product').head()
# metric still need calibration and weighting as required