embedding

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

import openai
import os

# 加载 .env 文件
from dotenv import load_dotenv, find_dotenv
_ = load_dotenv(find_dotenv(),verbose=True)

# 从环境变量中获得你的 OpenAI Key
openai.api_key  = os.getenv('OPENAI_API_KEY')
print(openai.api_key)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

import numpy as np
from numpy import dot
from numpy.linalg import norm
from langchain.embeddings import OpenAIEmbeddings

def cos_sim(a,b):
    return dot(a, b)/(norm(a)*norm(b))

def l2(a,b):
    x = np.asarray(a)-np.asarray(b)
    return norm(x)

# model = OpenAIEmbeddings(model='text-embedding-ada-002')
model = OpenAIEmbeddings(model='text-embedding-3-small')
#oc_embeddings = OpenAIEmbeddings(model='text-embedding-ada-002')

query = "国际争端"
documents = [
    "联合国就苏丹达尔富尔地区大规模暴力事件发出警告",
    "土耳其、芬兰、瑞典与北约代表将继续就瑞典“入约”问题进行谈判",
    "日本岐阜市陆上自卫队射击场内发生枪击事件 3人受伤",
    "国家游泳中心（水立方）：恢复游泳、嬉水乐园等水上项目运营",
    "我国首次在空间站开展舱外辐射生物学暴露实验",
]

query_vec = model.embed_query(query)
doc_vecs = model.embed_documents(documents)

print("Cosine distance:") #越大越相似
print(cos_sim(query_vec,query_vec))
for vec in doc_vecs:
    print(cos_sim(query_vec,vec))

print("\nEuclidean distance:") #越小越相似
print(l2(query_vec,query_vec))
for vec in doc_vecs:
    print(l2(query_vec,vec))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

Cosine distance:
1.0000000000000002
0.3455584312568389
0.2816726969540693
0.1980973032260824
0.18847459412724374
0.17086387335398848

Euclidean distance:
0.0
1.1440643065345244
1.1986052753479193
1.2664143846102804
1.2739901144614556
1.2877392023589338

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

from langchain.embeddings import OpenAIEmbeddings
from sklearn.cluster import KMeans, DBSCAN
import numpy as np

texts = [
    "这个多少钱",
    "啥价",
    "给我报个价",
    "我要红色的",
    "不要了",
    "算了",
    "来红的吧",
    "作罢",
    "价格介绍一下",
    "红的这个给我吧"
]
model = OpenAIEmbeddings(model='text-embedding-ada-002')
X = []
for t in texts:
    embedding = model.embed_query(t)
    X.append(embedding)
    
#clusters = KMeans(n_clusters=3, random_state=42, n_init="auto").fit(X)
clusters = DBSCAN(eps=0.55, min_samples=2).fit(X)
for i,t in enumerate(texts):
    print("{}\t{}".format(clusters.labels_[i],t))