Update app.py

app.py

@@ -50,7 +50,7 @@ def load_demo_refresh_model_list(request: gr.Request):
     state = default_conversation.copy()
     dropdown_update = gr.Dropdown.update(
         choices=models,
-        value=models[0] if len(models) > 0 else ""
+        value=models[0]+"-4bit" if len(models) > 0 else ""
     )
     return state, dropdown_update
 
@@ -183,27 +183,27 @@ def http_bot(state, model_selector, temperature, top_p, max_new_tokens, request:
     yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
 
 
-    # try:
+    try:
         # Stream output
-    response = chat.generate_stream_gate(pload)
-    for chunk in response:
-        if chunk:
-            data = json.loads(chunk.decode())
-            if data["error_code"] == 0:
-                output = data["text"][len(prompt):].strip()
-                state.messages[-1][-1] = output + "▌"
-                yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
-            else:
-                output = data["text"] + f" (error_code: {data['error_code']})"
-                state.messages[-1][-1] = output
-                yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
-                return
-            time.sleep(0.03)
-    # except Exception:
-    #     gr.Warning(server_error_msg)
-    #     state.messages[-1][-1] = server_error_msg
-    #     yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
-    #     return
+        response = chat.generate_stream_gate(pload)
+        for chunk in response:
+            if chunk:
+                data = json.loads(chunk.decode())
+                if data["error_code"] == 0:
+                    output = data["text"][len(prompt):].strip()
+                    state.messages[-1][-1] = output + "▌"
+                    yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
+                else:
+                    output = data["text"] + f" (error_code: {data['error_code']})"
+                    state.messages[-1][-1] = output
+                    yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
+                    return
+                time.sleep(0.03)
+    except Exception:
+        gr.Warning(server_error_msg)
+        state.messages[-1][-1] = server_error_msg
+        yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
+        return
 
     state.messages[-1][-1] = state.messages[-1][-1][:-1]
     yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5
@@ -372,6 +372,8 @@ if __name__ == "__main__":
         model_name = args.model_name
 
     models = [model_name]
+    args.load_4bit = True
+    
     chat = Chat(
         args.model_path,
         args.model_base,