upload csv to leaderboard_results

.gitignore

@@ -13,6 +13,7 @@ eval-results/
 auto_evals/
 eval-queue-bk/
 eval-results-bk/
+eval-results-bk_hhem21/
 
 src/assets/model_counts.html
 

src/backend/evaluate_model.py

@@ -112,13 +112,13 @@ class Evaluator:
 
         #update leaderboard_summaries.csv
         #first remove previous results for the current model
-        # existing_df = pd.read_csv(os.path.join(working_path, 'leaderboard_summaries.csv'), encoding='utf-8')
-        # mask = existing_df['model'] == self.model
-        # existing_df = existing_df[~mask]
-        # print(existing_df.shape)
-        # summary_doc = set(existing_df['model'].values.tolist())
-        # print(summary_doc)
-        # # get new result
+        existing_df = pd.read_csv(os.path.join(working_path, 'leaderboard_summaries.csv'), encoding='utf-8')
+        mask = existing_df['model'] == self.model
+        existing_df = existing_df[~mask]
+        print(existing_df.shape)
+        summary_doc = set(existing_df['model'].values.tolist())
+        print(summary_doc)
+        # get new result
         leaderboard_summaries_df = source_summary_df
         leaderboard_summaries_df.insert(2, "model", [self.model]*leaderboard_summaries_df.shape[0])
         leaderboard_summaries_df.to_csv(os.path.join(working_path, 'leaderboard_summaries.csv'), mode='a', index=False, header=False)
@@ -126,23 +126,17 @@ class Evaluator:
 
         # update leaderboard_summaries_with_scores.csv
         # BUG: get error when opening the file
-        # existing_df = pd.read_csv(os.path.join(working_path, 'leaderboard_summaries_with_scores.csv'), 
-        #                         encoding='utf-8', sep=",", quotechar='"', quoting=2)
-        # print(existing_df.shape)
-        # score_doc = set(existing_df['model'].values.tolist())
-        # print(score_doc)
-        # mask = existing_df['model'] == self.model
-        # existing_df = existing_df[~mask]
-        # # get new result
+        existing_df = pd.read_csv(os.path.join(working_path, 'leaderboard_summaries_with_scores.csv'), 
+                                encoding='utf-8', sep=",", quotechar='"', quoting=2)
+        print(existing_df.shape)
+        score_doc = set(existing_df['model'].values.tolist())
+        print(score_doc)
+        mask = existing_df['model'] == self.model
+        existing_df = existing_df[~mask]
+        # get new result
         leaderboard_summaries_with_scores_df = pd.DataFrame.from_dict(self.eval_results)
         leaderboard_summaries_with_scores_df.insert(3, "model", [self.model]*leaderboard_summaries_with_scores_df.shape[0])
         leaderboard_summaries_with_scores_df.to_csv(os.path.join(working_path, 'leaderboard_summaries_with_scores.csv'), mode='a', index=False, header=False)
         print('leaderboard_summaries_with_scores.csv has been updated')
 
-        # for model in summary_doc:
-        #     if model not in score_doc:
-        #         print(f"{model} records missing in leaderboard_summaries_with_scores.csv")
-
-        # for model in score_doc:
-        #     if model not in summary_doc:
-        #         print(f"{model} records missing in leaderboard_summaries.csv")
+        

src/backend/model_operations.py

@@ -27,7 +27,7 @@ import google.generativeai as genai
 import src.backend.util as util
 import src.envs as envs
 
-litellm.set_verbose=True
+litellm.set_verbose=False
 
 # Set up basic configuration for logging
 logging.basicConfig(level=logging.INFO,
@@ -95,6 +95,7 @@ class SummaryGenerator:
         self.answer_rate = None
         self.exceptions = None
         self.local_model = None
+        self.local_pipeline = None
 
     def generate_summaries(self, df, save_path=None):
         """Generate summaries for a given DataFrame of source docs.
@@ -118,8 +119,9 @@ class SummaryGenerator:
 
                 system_prompt = envs.SYSTEM_PROMPT
                 user_prompt = f"{envs.USER_PROMPT}\nPassage:\n{_source}"
+                _summary = None
 
-                while True:
+                while not _summary:
                     try:
                         _summary = self.generate_summary(system_prompt, user_prompt)
                         # print(f"Finish index {index}")
@@ -169,11 +171,22 @@ class SummaryGenerator:
     def generate_summary(self, system_prompt: str, user_prompt: str):
         # Using Together AI API
         using_together_api = False
-        together_ai_api_models = ['mixtral', 'dbrx', 'wizardlm', 'llama-3', 'qwen'] #, 'mistralai'
-        for together_ai_api_model in together_ai_api_models:
-            if together_ai_api_model in self.model_id.lower():
-                using_together_api = True
+        together_ai_api_models = ['mixtral', 'dbrx', 'wizardlm', 'llama-3-', 'qwen'] #, 'mistralai'
+        using_replicate_api = False
+        replicate_api_models = ['snowflake', 'llama-3.1-405b']
+
+        for replicate_api_model in replicate_api_models:
+            if replicate_api_model in self.model_id.lower():
+                using_replicate_api = True
                 break
+
+        if not using_replicate_api:
+            for together_ai_api_model in together_ai_api_models:
+                if together_ai_api_model in self.model_id.lower():
+                    using_together_api = True
+                    break
+
+        
         # if 'mixtral' in self.model_id.lower() or 'dbrx' in self.model_id.lower() or 'wizardlm' in self.model_id.lower(): # For mixtral and dbrx models, use Together AI API
         if using_together_api:
             # print('using together api')
@@ -269,24 +282,33 @@ class SummaryGenerator:
             print(result)
             return result
 
-        elif 'snowflake' in self.model_id.lower():
+        elif using_replicate_api:
             print("using replicate")
-            input = {
-                "prompt": user_prompt,
-                "temperature": 0,
-                "max_new_tokens": 250,
-                "stop_sequences": "<|im_end|>",
-                "prompt_template": f"<|im_start|>system\n{system_prompt}<|im_end|>\n" + "<|im_start|>user\n{prompt}<|im_end|>\n\n<|im_start|>assistant\n",
-            }
+            if 'snowflake' in self.model_id.lower():
+                input = {
+                    "prompt": user_prompt,
+                    "temperature": 0,
+                    "max_new_tokens": 250,
+                    "stop_sequences": "<|im_end|>",
+                    "prompt_template": f"<|im_start|>system\n{system_prompt}<|im_end|>\n" + "<|im_start|>user\n{prompt}<|im_end|>\n\n<|im_start|>assistant\n",
+                }
+            else:
+                input = {
+                    "prompt": user_prompt,
+                    "system_prompt": system_prompt,
+                    "temperature": 0,
+                    "max_new_tokens": 250
+                }
             response = replicate.run(
-                self.model_id.lower(),
+                self.model_id,
                 input=input
             )
+            # print(response)
             if isinstance(response, list):
                 response = ''.join(response)
-                print(response)
-                print()
-            
+                # print(response)
+                # print()
+            print(response)
             return response
 
         elif 'claude' in self.model_id.lower(): # using anthropic api
@@ -313,22 +335,11 @@ class SummaryGenerator:
             return result
                 
         # Using HF API or download checkpoints
-        elif self.local_model is None:
-            # response = litellm.completion(
-            #     model='command-r-plus' if 'command' in self.model else self.model,
-            #     messages=[{"role": "system", "content": system_prompt},
-            #                 {"role": "user", "content": user_prompt}],
-            #     temperature=0.0,
-            #     max_tokens=256,
-            #     api_base=self.api_base,
-            # )
-            # result = response['choices'][0]['message']['content']
-            # print(result)
-            # return result
+        elif self.local_model is None and self.local_pipeline is None:
             try: # try use HuggingFace API
-                print('using huggingface api')
+                print('** using huggingface api')
                 response = litellm.completion(
-                    model='command-r-plus' if 'command' in self.model_id else self.model_id,
+                    model=self.model,
                     messages=[{"role": "system", "content": system_prompt},
                                 {"role": "user", "content": user_prompt}],
                     temperature=0.0,
@@ -345,13 +356,35 @@ class SummaryGenerator:
                     print(f"Rate limit hit at {current_time}. Waiting for 5 minutes before retrying...")
                     time.sleep(wait_time)
                 else:
-                    self.tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf" if 'openelm' in self.model_id.lower() else self.model_id, trust_remote_code=True)
-                    print("Tokenizer loaded")
-                    self.local_model = AutoModelForCausalLM.from_pretrained(self.model_id, trust_remote_code=True, device_map="auto", torch_dtype="auto", cache_dir='/home/paperspace/cache')
-                    print("Local model loaded")
+                    try:
+                        self.local_pipeline = pipeline(
+                            "text-generation",
+                            model=self.model_id,
+                            model_kwargs={"torch_dtype": torch.bfloat16},
+                            device_map="auto",
+                        )
+                    except:
+                        self.tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf" if 'openelm' in self.model_id.lower() else self.model_id, trust_remote_code=True)
+                        print("Tokenizer loaded")
+                        self.local_model = AutoModelForCausalLM.from_pretrained(self.model_id, trust_remote_code=True, device_map="auto", torch_dtype="auto")
+                        print("Local model loaded")
         
-        # Using local model
-        if self.local_model: # cannot call API. using local model
+            
+        # Using local model/pipeline
+        if self.local_pipeline:
+            messages=[
+                {"role": "system", "content": system_prompt}, 
+                {"role": "user", "content": user_prompt}
+            ]
+            outputs = self.local_pipeline(
+                messages,
+                max_new_tokens=250,
+            )
+            result = outputs[0]["generated_text"][-1]['content']
+            print(result)
+            return result
+
+        elif self.local_model: # cannot call API. using local model / pipeline
             if 'gemma' in self.model_id.lower() or 'mistral-7b' in self.model_id.lower():
                 messages=[
                     # gemma-1.1, mistral-7b does not accept system role
@@ -361,10 +394,10 @@ class SummaryGenerator:
             
             elif 'phi-2' in self.model_id.lower():
                 prompt = system_prompt + '\n' + user_prompt
-            
+
             else:
                 messages=[
-                    {"role": "system", "content": system_prompt}, # gemma-1.1, mistral-7b does not accept system role
+                    {"role": "system", "content": system_prompt}, 
                     {"role": "user", "content": user_prompt}
                 ]
                 prompt = self.tokenizer.apply_chat_template(messages,add_generation_prompt=True, tokenize=False)

src/backend/run_eval_suite.py

@@ -50,6 +50,14 @@ def run_evaluation(eval_request: EvalRequest, batch_size, device,
         results = evaluator.evaluate()
         if write_results:
             evaluator.write_results()
+            # upload leaderboard_summaries.csv to HF
+            envs.API.upload_file(
+                path_or_fileobj=envs.LEADERBOARD_DATASET_PATH,
+                path_in_repo=envs.LEADERBOARD_DATASET_PATH.split('/')[-1],
+                repo_id=envs.LEADERBOARD_DATASET_REPO,
+                repo_type="dataset",
+            )
+            
     except Exception as e:
         logging.error(f"Error during evaluation: {e}")
         raise

src/envs.py

@@ -10,6 +10,7 @@ OWNER = "vectara"
 REPO_ID = f"{OWNER}/leaderboard"
 QUEUE_REPO = f"{OWNER}/requests"
 RESULTS_REPO = f"{OWNER}/results"
+LEADERBOARD_DATASET_REPO = f"{OWNER}/leaderboard_results"
 
 CACHE_PATH=os.getenv("HF_HOME", ".")
 
@@ -22,6 +23,7 @@ EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu') #"cpu"
 API = HfApi(token=TOKEN)
 
+LEADERBOARD_DATASET_PATH = "Hallucination Leaderboard Results/leaderboard_summaries.csv"
 DATASET_PATH = "src/datasets/leaderboard_dataset.csv"
 SAMPLE_DATASET_PATH = "src/datasets/sample_dataset.csv"
 HEM_PATH = 'vectara/hallucination_evaluation_model'