Cross-Pollination for Creativity Leveraging LLMs

numpy==1.25.2
torch==2.0.1
annoy==1.17.3
langchain==0.0.237
sentence-transformers==2.2.2

import json 
from sentence_transformers import SentenceTransformer, util
import numpy as np
import torch
import time
from annoy import AnnoyIndex

file_name = "arxiv-metadata-oai-snapshot.json"

'''
Preprocess the data and build a list
'''
def preprocess(path):
    data = []
    start = time.time()
    with open(path,'r') as f:
        for line in f:
            data.append(json.loads(line))
    start = time.time()
    sents = []
    for i in range(len(data)):
        sents.append(data[i]['title']+'[SEP]'+data[i]['abstract'])
    end = time.time()
    print("Time taken: ",end-start)  
    return sents


'''
Use Sentence BERT Specter embedding to build a numpy array
'''
def generate_embeddings(sents,model):
    embeddings = model.encode(sents,batch_size=400,show_progress_bar=True,device='cuda',convert_to_numpy=True)
    np.save("embeddings.npy",embeddings)
    return embeddings


'''
Generate an annoy index. 
Instead of choosing angular as distance metric, other options such as euclidean are also available
'''
def generate_annoy(embeddings):
    n_trees = 256           
    embedding_size = 768   
    top_k_hits = 5        
    annoy_index = AnnoyIndex(embedding_size, 'angular')
    for i in range(len(embeddings)):
        annoy_index.add_item(i, embeddings[i])
    annoy_index.build(n_trees)
    annoy_index.save("vector_index.ann")
    return annoy_index

sents = preprocess(file_name)
model = SentenceTransformer('sentence-transformers/allenai-specter', device='cuda')
embeddings = generate_embeddings(sents,model)
annoy_index = generate_annoy(embeddings)

model = SentenceTransformer('sentence-transformers/allenai-specter', device='cpu')
OPENAI_API_KEY = getpass("Please enter your OPEN AI API KEY to continue:")
## load the OPENAI LLM model
open_ai_key = OPENAI_API_KEY
sents_from_path = preprocess("<Path to your arxiv.json>")
## index from Annoy
an = generate_annoy("<name of the index.ann>")
llm = OpenAI(openai_api_key=open_ai_key, model_name= "gpt-3.5-turbo-16k")

def search(query,annoy_index,model, topK):
    query_embedding = model.encode(query,convert_to_numpy=True)
    tokens = query.strip().split()
    hit_dict = {}
    hits = annoy_index.get_nns_by_vector(query_embedding, topK, include_distances=True)
    return [hits, hits]

    
def collect_results(hits,sents, path, prefix=""):
    response = ""
    for i in range(len(hits[0])):
        response += "result " + prefix + str(i) + "\n"
        response += "Title:" + sents[hits[0][i]].split('[SEP]')[0] + "\n"
        response += "Abstract: " + sents[hits[0][i]].split('[SEP]')[1] + "\n"
    return response

fieldA = input("Describe the field (or subfield) as a query:")
query = fieldA
response = collect_results(search(query,an,model,30),sents_from_path1, path1)

template = """ Take a deep breath. 
    You are a researcher in the FIELD of {query}. You are tasked with identifying research gaps and opportunities. User entered the FIELD. The TOP ARTICLES from the search engine are {response}. Analyze the results and follow the instructions below to identify research gaps in the field of {query}.
    TASK DESCRIPTION 
    In this task, your goal is to recognize areas within a specified FIELD where further research is needed. 
    By analyzing existing literature and TOP ARTICLES , data, and perhaps the trajectories of ongoing research, the AI can help point out topics or questions that haven't been adequately explored or resolved. This is essential for advancing knowledge and finding new directions that could lead to significant discoveries or developments
    For example, consider the domain of renewable energy. one might analyze a vast number of research papers, patents, and ongoing projects in this domain. It could then identify that while there's abundant research on solar and wind energy, there might be a relative lack of research on harnessing tidal energy or integrating different types of renewable energy systems for more consistent power generation.
    By identifying these research gaps, you will assisint in pinpointing areas where researchers can focus their efforts to potentially make novel contributions. Moreover, it can help funding agencies and institutions to direct resources and support towards these identified areas of opportunity, fostering advancements that might otherwise be overlooked.
   
    INSTRUCTIONS:
    1. Analyze each article returned by the search engine.
    2. Identify the research gaps and opportunities for each article
    3. Summarize the research gaps and opportunities in the field of {query} across ARTICLES. The research gap should BE GROUNDED on the articles returned by the search engine.
    3. Identify an orthogonal field and generate keyword to get articles from that field that can solve the research gaps.
    4. Identify a related field and generate keyword to get articles from that field that can solve the research gaps
    5. Generate a JSON Response that STRICTLY follow the TYPESCRIPT SCHEMA of class Response below
    6. Output MUST BE a JSON  with the following fields in the SCHEMA.
    SCHEMA:
    class Response:
        orthogonal_field: String
        keyword_orthogonal_field: String
        related_field: String
        keyword_related_field: String
        research_gaps: String
    
"""
prompt = PromptTemplate(template=template, input_variables=["query", "response"])
llm_chain = LLMChain(prompt=prompt, llm=llm)

output = ""

try:
	output = llm_chain.run({'query': query, 'response':response})
except Exception as e:
  	print (e)