chore: init monorepo snapshot
This commit is contained in:
liaibo
@@ -0,0 +1,69 @@
|
||||
import os
|
||||
import json
|
||||
import glob
|
||||
import argparse
|
||||
|
||||
|
||||
def extract_unique_contexts(input_directory, output_directory):
|
||||
os.makedirs(output_directory, exist_ok=True)
|
||||
|
||||
jsonl_files = glob.glob(os.path.join(input_directory, "*.jsonl"))
|
||||
print(f"Found {len(jsonl_files)} JSONL files.")
|
||||
|
||||
for file_path in jsonl_files:
|
||||
filename = os.path.basename(file_path)
|
||||
name, ext = os.path.splitext(filename)
|
||||
output_filename = f"{name}_unique_contexts.json"
|
||||
output_path = os.path.join(output_directory, output_filename)
|
||||
|
||||
unique_contexts_dict = {}
|
||||
|
||||
print(f"Processing file: {filename}")
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as infile:
|
||||
for line_number, line in enumerate(infile, start=1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
json_obj = json.loads(line)
|
||||
context = json_obj.get("context")
|
||||
if context and context not in unique_contexts_dict:
|
||||
unique_contexts_dict[context] = None
|
||||
except json.JSONDecodeError as e:
|
||||
print(
|
||||
f"JSON decoding error in file {filename} at line {line_number}: {e}"
|
||||
)
|
||||
except FileNotFoundError:
|
||||
print(f"File not found: {filename}")
|
||||
continue
|
||||
except Exception as e:
|
||||
print(f"An error occurred while processing file {filename}: {e}")
|
||||
continue
|
||||
|
||||
unique_contexts_list = list(unique_contexts_dict.keys())
|
||||
print(
|
||||
f"There are {len(unique_contexts_list)} unique `context` entries in the file {filename}."
|
||||
)
|
||||
|
||||
try:
|
||||
with open(output_path, "w", encoding="utf-8") as outfile:
|
||||
json.dump(unique_contexts_list, outfile, ensure_ascii=False, indent=4)
|
||||
print(f"Unique `context` entries have been saved to: {output_filename}")
|
||||
except Exception as e:
|
||||
print(f"An error occurred while saving to the file {output_filename}: {e}")
|
||||
|
||||
print("All files have been processed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-i", "--input_dir", type=str, default="../datasets")
|
||||
parser.add_argument(
|
||||
"-o", "--output_dir", type=str, default="../datasets/unique_contexts"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
extract_unique_contexts(args.input_dir, args.output_dir)
|
||||
@@ -0,0 +1,51 @@
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import asyncio
|
||||
|
||||
from lightrag import LightRAG
|
||||
from lightrag.kg.shared_storage import initialize_pipeline_status
|
||||
|
||||
|
||||
def insert_text(rag, file_path):
|
||||
with open(file_path, mode="r") as f:
|
||||
unique_contexts = json.load(f)
|
||||
|
||||
retries = 0
|
||||
max_retries = 3
|
||||
while retries < max_retries:
|
||||
try:
|
||||
rag.insert(unique_contexts)
|
||||
break
|
||||
except Exception as e:
|
||||
retries += 1
|
||||
print(f"Insertion failed, retrying ({retries}/{max_retries}), error: {e}")
|
||||
time.sleep(10)
|
||||
if retries == max_retries:
|
||||
print("Insertion failed after exceeding the maximum number of retries")
|
||||
|
||||
|
||||
cls = "agriculture"
|
||||
WORKING_DIR = f"../{cls}"
|
||||
|
||||
if not os.path.exists(WORKING_DIR):
|
||||
os.mkdir(WORKING_DIR)
|
||||
|
||||
|
||||
async def initialize_rag():
|
||||
rag = LightRAG(working_dir=WORKING_DIR)
|
||||
|
||||
await rag.initialize_storages()
|
||||
await initialize_pipeline_status()
|
||||
|
||||
return rag
|
||||
|
||||
|
||||
def main():
|
||||
# Initialize RAG instance
|
||||
rag = asyncio.run(initialize_rag())
|
||||
insert_text(rag, f"../datasets/unique_contexts/{cls}_unique_contexts.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,86 @@
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import asyncio
|
||||
import numpy as np
|
||||
|
||||
from lightrag import LightRAG
|
||||
from lightrag.utils import EmbeddingFunc
|
||||
from lightrag.llm.openai import openai_complete_if_cache, openai_embed
|
||||
from lightrag.kg.shared_storage import initialize_pipeline_status
|
||||
|
||||
|
||||
## For Upstage API
|
||||
# please check if embedding_dim=4096 in lightrag.py and llm.py in lightrag direcotry
|
||||
async def llm_model_func(
|
||||
prompt, system_prompt=None, history_messages=[], **kwargs
|
||||
) -> str:
|
||||
return await openai_complete_if_cache(
|
||||
"solar-mini",
|
||||
prompt,
|
||||
system_prompt=system_prompt,
|
||||
history_messages=history_messages,
|
||||
api_key=os.getenv("UPSTAGE_API_KEY"),
|
||||
base_url="https://api.upstage.ai/v1/solar",
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def embedding_func(texts: list[str]) -> np.ndarray:
|
||||
return await openai_embed(
|
||||
texts,
|
||||
model="solar-embedding-1-large-query",
|
||||
api_key=os.getenv("UPSTAGE_API_KEY"),
|
||||
base_url="https://api.upstage.ai/v1/solar",
|
||||
)
|
||||
|
||||
|
||||
## /For Upstage API
|
||||
|
||||
|
||||
def insert_text(rag, file_path):
|
||||
with open(file_path, mode="r") as f:
|
||||
unique_contexts = json.load(f)
|
||||
|
||||
retries = 0
|
||||
max_retries = 3
|
||||
while retries < max_retries:
|
||||
try:
|
||||
rag.insert(unique_contexts)
|
||||
break
|
||||
except Exception as e:
|
||||
retries += 1
|
||||
print(f"Insertion failed, retrying ({retries}/{max_retries}), error: {e}")
|
||||
time.sleep(10)
|
||||
if retries == max_retries:
|
||||
print("Insertion failed after exceeding the maximum number of retries")
|
||||
|
||||
|
||||
cls = "mix"
|
||||
WORKING_DIR = f"../{cls}"
|
||||
|
||||
if not os.path.exists(WORKING_DIR):
|
||||
os.mkdir(WORKING_DIR)
|
||||
|
||||
|
||||
async def initialize_rag():
|
||||
rag = LightRAG(
|
||||
working_dir=WORKING_DIR,
|
||||
llm_model_func=llm_model_func,
|
||||
embedding_func=EmbeddingFunc(embedding_dim=4096, func=embedding_func),
|
||||
)
|
||||
|
||||
await rag.initialize_storages()
|
||||
await initialize_pipeline_status()
|
||||
|
||||
return rag
|
||||
|
||||
|
||||
def main():
|
||||
# Initialize RAG instance
|
||||
rag = asyncio.run(initialize_rag())
|
||||
insert_text(rag, f"../datasets/unique_contexts/{cls}_unique_contexts.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,78 @@
|
||||
import json
|
||||
from openai import OpenAI
|
||||
from transformers import GPT2Tokenizer
|
||||
|
||||
|
||||
def openai_complete_if_cache(
|
||||
model="gpt-4o", prompt=None, system_prompt=None, history_messages=[], **kwargs
|
||||
) -> str:
|
||||
openai_client = OpenAI()
|
||||
|
||||
messages = []
|
||||
if system_prompt:
|
||||
messages.append({"role": "system", "content": system_prompt})
|
||||
messages.extend(history_messages)
|
||||
messages.append({"role": "user", "content": prompt})
|
||||
|
||||
response = openai_client.chat.completions.create(
|
||||
model=model, messages=messages, **kwargs
|
||||
)
|
||||
return response.choices[0].message.content
|
||||
|
||||
|
||||
tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
|
||||
|
||||
|
||||
def get_summary(context, tot_tokens=2000):
|
||||
tokens = tokenizer.tokenize(context)
|
||||
half_tokens = tot_tokens // 2
|
||||
|
||||
start_tokens = tokens[1000 : 1000 + half_tokens]
|
||||
end_tokens = tokens[-(1000 + half_tokens) : 1000]
|
||||
|
||||
summary_tokens = start_tokens + end_tokens
|
||||
summary = tokenizer.convert_tokens_to_string(summary_tokens)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
clses = ["agriculture"]
|
||||
for cls in clses:
|
||||
with open(f"../datasets/unique_contexts/{cls}_unique_contexts.json", mode="r") as f:
|
||||
unique_contexts = json.load(f)
|
||||
|
||||
summaries = [get_summary(context) for context in unique_contexts]
|
||||
|
||||
total_description = "\n\n".join(summaries)
|
||||
|
||||
prompt = f"""
|
||||
Given the following description of a dataset:
|
||||
|
||||
{total_description}
|
||||
|
||||
Please identify 5 potential users who would engage with this dataset. For each user, list 5 tasks they would perform with this dataset. Then, for each (user, task) combination, generate 5 questions that require a high-level understanding of the entire dataset.
|
||||
|
||||
Output the results in the following structure:
|
||||
- User 1: [user description]
|
||||
- Task 1: [task description]
|
||||
- Question 1:
|
||||
- Question 2:
|
||||
- Question 3:
|
||||
- Question 4:
|
||||
- Question 5:
|
||||
- Task 2: [task description]
|
||||
...
|
||||
- Task 5: [task description]
|
||||
- User 2: [user description]
|
||||
...
|
||||
- User 5: [user description]
|
||||
...
|
||||
"""
|
||||
|
||||
result = openai_complete_if_cache(model="gpt-4o", prompt=prompt)
|
||||
|
||||
file_path = f"../datasets/questions/{cls}_questions.txt"
|
||||
with open(file_path, "w") as file:
|
||||
file.write(result)
|
||||
|
||||
print(f"{cls}_questions written to {file_path}")
|
||||
@@ -0,0 +1,66 @@
|
||||
import re
|
||||
import json
|
||||
from lightrag import LightRAG, QueryParam
|
||||
from lightrag.utils import always_get_an_event_loop
|
||||
|
||||
|
||||
def extract_queries(file_path):
|
||||
with open(file_path, "r") as f:
|
||||
data = f.read()
|
||||
|
||||
data = data.replace("**", "")
|
||||
|
||||
queries = re.findall(r"- Question \d+: (.+)", data)
|
||||
|
||||
return queries
|
||||
|
||||
|
||||
async def process_query(query_text, rag_instance, query_param):
|
||||
try:
|
||||
result = await rag_instance.aquery(query_text, param=query_param)
|
||||
return {"query": query_text, "result": result}, None
|
||||
except Exception as e:
|
||||
return None, {"query": query_text, "error": str(e)}
|
||||
|
||||
|
||||
def run_queries_and_save_to_json(
|
||||
queries, rag_instance, query_param, output_file, error_file
|
||||
):
|
||||
loop = always_get_an_event_loop()
|
||||
|
||||
with (
|
||||
open(output_file, "a", encoding="utf-8") as result_file,
|
||||
open(error_file, "a", encoding="utf-8") as err_file,
|
||||
):
|
||||
result_file.write("[\n")
|
||||
first_entry = True
|
||||
|
||||
for query_text in queries:
|
||||
result, error = loop.run_until_complete(
|
||||
process_query(query_text, rag_instance, query_param)
|
||||
)
|
||||
|
||||
if result:
|
||||
if not first_entry:
|
||||
result_file.write(",\n")
|
||||
json.dump(result, result_file, ensure_ascii=False, indent=4)
|
||||
first_entry = False
|
||||
elif error:
|
||||
json.dump(error, err_file, ensure_ascii=False, indent=4)
|
||||
err_file.write("\n")
|
||||
|
||||
result_file.write("\n]")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cls = "agriculture"
|
||||
mode = "hybrid"
|
||||
WORKING_DIR = f"../{cls}"
|
||||
|
||||
rag = LightRAG(working_dir=WORKING_DIR)
|
||||
query_param = QueryParam(mode=mode)
|
||||
|
||||
queries = extract_queries(f"../datasets/questions/{cls}_questions.txt")
|
||||
run_queries_and_save_to_json(
|
||||
queries, rag, query_param, f"{cls}_result.json", f"{cls}_errors.json"
|
||||
)
|
||||
@@ -0,0 +1,103 @@
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
from lightrag import LightRAG, QueryParam
|
||||
from lightrag.llm.openai import openai_complete_if_cache, openai_embed
|
||||
from lightrag.utils import EmbeddingFunc, always_get_an_event_loop
|
||||
import numpy as np
|
||||
|
||||
|
||||
## For Upstage API
|
||||
# please check if embedding_dim=4096 in lightrag.py and llm.py in lightrag direcotry
|
||||
async def llm_model_func(
|
||||
prompt, system_prompt=None, history_messages=[], **kwargs
|
||||
) -> str:
|
||||
return await openai_complete_if_cache(
|
||||
"solar-mini",
|
||||
prompt,
|
||||
system_prompt=system_prompt,
|
||||
history_messages=history_messages,
|
||||
api_key=os.getenv("UPSTAGE_API_KEY"),
|
||||
base_url="https://api.upstage.ai/v1/solar",
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def embedding_func(texts: list[str]) -> np.ndarray:
|
||||
return await openai_embed(
|
||||
texts,
|
||||
model="solar-embedding-1-large-query",
|
||||
api_key=os.getenv("UPSTAGE_API_KEY"),
|
||||
base_url="https://api.upstage.ai/v1/solar",
|
||||
)
|
||||
|
||||
|
||||
## /For Upstage API
|
||||
|
||||
|
||||
def extract_queries(file_path):
|
||||
with open(file_path, "r") as f:
|
||||
data = f.read()
|
||||
|
||||
data = data.replace("**", "")
|
||||
|
||||
queries = re.findall(r"- Question \d+: (.+)", data)
|
||||
|
||||
return queries
|
||||
|
||||
|
||||
async def process_query(query_text, rag_instance, query_param):
|
||||
try:
|
||||
result = await rag_instance.aquery(query_text, param=query_param)
|
||||
return {"query": query_text, "result": result}, None
|
||||
except Exception as e:
|
||||
return None, {"query": query_text, "error": str(e)}
|
||||
|
||||
|
||||
def run_queries_and_save_to_json(
|
||||
queries, rag_instance, query_param, output_file, error_file
|
||||
):
|
||||
loop = always_get_an_event_loop()
|
||||
|
||||
with (
|
||||
open(output_file, "a", encoding="utf-8") as result_file,
|
||||
open(error_file, "a", encoding="utf-8") as err_file,
|
||||
):
|
||||
result_file.write("[\n")
|
||||
first_entry = True
|
||||
|
||||
for query_text in queries:
|
||||
result, error = loop.run_until_complete(
|
||||
process_query(query_text, rag_instance, query_param)
|
||||
)
|
||||
|
||||
if result:
|
||||
if not first_entry:
|
||||
result_file.write(",\n")
|
||||
json.dump(result, result_file, ensure_ascii=False, indent=4)
|
||||
first_entry = False
|
||||
elif error:
|
||||
json.dump(error, err_file, ensure_ascii=False, indent=4)
|
||||
err_file.write("\n")
|
||||
|
||||
result_file.write("\n]")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cls = "mix"
|
||||
mode = "hybrid"
|
||||
WORKING_DIR = f"../{cls}"
|
||||
|
||||
rag = LightRAG(working_dir=WORKING_DIR)
|
||||
rag = LightRAG(
|
||||
working_dir=WORKING_DIR,
|
||||
llm_model_func=llm_model_func,
|
||||
embedding_func=EmbeddingFunc(embedding_dim=4096, func=embedding_func),
|
||||
)
|
||||
query_param = QueryParam(mode=mode)
|
||||
|
||||
base_dir = "../datasets/questions"
|
||||
queries = extract_queries(f"{base_dir}/{cls}_questions.txt")
|
||||
run_queries_and_save_to_json(
|
||||
queries, rag, query_param, f"{base_dir}/result.json", f"{base_dir}/errors.json"
|
||||
)
|
||||
@@ -0,0 +1,112 @@
|
||||
import re
|
||||
import json
|
||||
import jsonlines
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
|
||||
def batch_eval(query_file, result1_file, result2_file, output_file_path):
|
||||
client = OpenAI()
|
||||
|
||||
with open(query_file, "r") as f:
|
||||
data = f.read()
|
||||
|
||||
queries = re.findall(r"- Question \d+: (.+)", data)
|
||||
|
||||
with open(result1_file, "r") as f:
|
||||
answers1 = json.load(f)
|
||||
answers1 = [i["result"] for i in answers1]
|
||||
|
||||
with open(result2_file, "r") as f:
|
||||
answers2 = json.load(f)
|
||||
answers2 = [i["result"] for i in answers2]
|
||||
|
||||
requests = []
|
||||
for i, (query, answer1, answer2) in enumerate(zip(queries, answers1, answers2)):
|
||||
sys_prompt = """
|
||||
---Role---
|
||||
You are an expert tasked with evaluating two answers to the same question based on three criteria: **Comprehensiveness**, **Diversity**, and **Empowerment**.
|
||||
"""
|
||||
|
||||
prompt = f"""
|
||||
You will evaluate two answers to the same question based on three criteria: **Comprehensiveness**, **Diversity**, and **Empowerment**.
|
||||
|
||||
- **Comprehensiveness**: How much detail does the answer provide to cover all aspects and details of the question?
|
||||
- **Diversity**: How varied and rich is the answer in providing different perspectives and insights on the question?
|
||||
- **Empowerment**: How well does the answer help the reader understand and make informed judgments about the topic?
|
||||
|
||||
For each criterion, choose the better answer (either Answer 1 or Answer 2) and explain why. Then, select an overall winner based on these three categories.
|
||||
|
||||
Here is the question:
|
||||
{query}
|
||||
|
||||
Here are the two answers:
|
||||
|
||||
**Answer 1:**
|
||||
{answer1}
|
||||
|
||||
**Answer 2:**
|
||||
{answer2}
|
||||
|
||||
Evaluate both answers using the three criteria listed above and provide detailed explanations for each criterion.
|
||||
|
||||
Output your evaluation in the following JSON format:
|
||||
|
||||
{{
|
||||
"Comprehensiveness": {{
|
||||
"Winner": "[Answer 1 or Answer 2]",
|
||||
"Explanation": "[Provide explanation here]"
|
||||
}},
|
||||
"Diversity": {{
|
||||
"Winner": "[Answer 1 or Answer 2]",
|
||||
"Explanation": "[Provide explanation here]"
|
||||
}},
|
||||
"Empowerment": {{
|
||||
"Winner": "[Answer 1 or Answer 2]",
|
||||
"Explanation": "[Provide explanation here]"
|
||||
}},
|
||||
"Overall Winner": {{
|
||||
"Winner": "[Answer 1 or Answer 2]",
|
||||
"Explanation": "[Summarize why this answer is the overall winner based on the three criteria]"
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
|
||||
request_data = {
|
||||
"custom_id": f"request-{i + 1}",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [
|
||||
{"role": "system", "content": sys_prompt},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
requests.append(request_data)
|
||||
|
||||
with jsonlines.open(output_file_path, mode="w") as writer:
|
||||
for request in requests:
|
||||
writer.write(request)
|
||||
|
||||
print(f"Batch API requests written to {output_file_path}")
|
||||
|
||||
batch_input_file = client.files.create(
|
||||
file=open(output_file_path, "rb"), purpose="batch"
|
||||
)
|
||||
batch_input_file_id = batch_input_file.id
|
||||
|
||||
batch = client.batches.create(
|
||||
input_file_id=batch_input_file_id,
|
||||
endpoint="/v1/chat/completions",
|
||||
completion_window="24h",
|
||||
metadata={"description": "nightly eval job"},
|
||||
)
|
||||
|
||||
print(f"Batch {batch.id} has been created.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
batch_eval()
|
||||
Reference in New Issue
Block a user