ChatGLM2-6B

具体代码如下：

import traceback

from langchain.prompts import PromptTemplate
from langchain.llms.chatglm import ChatGLM
from langchain.chains import LLMChain

template = """{question}"""
prompt = PromptTemplate(template=template, input_variables=["question"])
llm = ChatGLM(
    endpoint_url="http://127.0.0.1:8899",
    max_token=80000,
    history=[],
    top_p=0.9,
    model_kwargs={"sample_model_args": False},
)
llm_chain = LLMChain(prompt=prompt, llm=llm)

try:
    question1 = "你的名字是什么啊"
    print("问：" + question1)
    answer1 = llm_chain.run(question1)
    print("答：" + str(answer1))

    question2 = "你知道 xiaodu114 吗"
    print("问：" + question2)
    answer2 = llm_chain.run(question2)
    print("答：" + str(answer2))
except Exception as ex:
    print(ex.args)
    print("="*24+">")
    print(traceback.format_exc())
                

具体代码如下：

import traceback

from langchain.llms.chatglm import ChatGLM
from langchain.chains.question_answering import load_qa_chain

llm = ChatGLM(
    endpoint_url="http://127.0.0.1:8899",
    max_token=80000,
    history=[],
    top_p=0.9,
    model_kwargs={"sample_model_args": False},
)
chain = load_qa_chain(llm, chain_type="stuff")

question = "你知道 xiaodu114 吗？"

try:
    print("问：" + question)
    answer = chain.run(input_documents=[], question=question)
    print("答：" + str(answer))
except Exception as ex:
    print(ex.args)
    print("="*24+">")
    print(traceback.format_exc())
                

具体代码如下：

const btnOpenAiAPIEle = document.getElementById("btnOpenAiAPI");
btnOpenAiAPIEle.addEventListener("click", () => {
    let myHeaders = new Headers();
    myHeaders.append("Content-Type", "application/json");
    myHeaders.append("Accept", "text/event-stream");

    const url = "http://127.0.0.1:8899/v1/chat/completions";
    fetch(url, {
        method: "POST",
        headers: myHeaders,
        body: JSON.stringify({
            model: "xxx",
            stream: true,
            messages: [{ role: "user", content: "你好" }]
        })
    }).then(
        (response) => {
            if (!(response && response.ok)) {
                console.error("异常：" + response.status);
                return;
            }
            let reader = response.body.getReader();
            reader.read().then(function processResult(result) {
                if (result.done) {
                    console.log("Event stream ended.");
                    return;
                }
                const decoder = new TextDecoder();
                const decodedString = decoder.decode(result.value);
                console.log(decodedString);
                // 继续读取下一个事件流数据
                reader.read().then(processResult);
            });
        },
        (error) => {
            console.error(`异常：${JSON.stringify(error)}`);
        }
    );
});
                

import datetime
import json
import traceback
import uvicorn
from transformers import AutoTokenizer, AutoModel
from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from sse_starlette.sse import EventSourceResponse

app = FastAPI()

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"], 
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

@app.get("/GetDateTime")
def get_server_datetime():
    return datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

async def process(prompt, max_length, top_p, temperature, history):
    """
    发文字消息
    """
    # 不能是空消息
    if not prompt:
        yield "prompt 不能为空"
        return

    try:
        current_length = 0
        #   经测试发现，历史对话也就是 history 参数需要的格式如下：
        #       [('中国的首都是哪里', '中国的首都是北京。'), ('他的面积是多少', '北京市的总面积大约为16,800平方公里。')]
        for response, history in model.stream_chat(tokenizer,
                                                   prompt, 
                                                   history=history,
                                                   max_length=max_length if max_length else 999999,
                                                   top_p=top_p if top_p else 0.7,
                                                   temperature=temperature if temperature else 0.95):
            if len(response) == current_length:
                continue
            new_text = response[current_length:]
            current_length = len(response)
            yield json.dumps({"text":str(new_text)},ensure_ascii=False)
    except Exception as ex:
        yield ex.args +"\n" + traceback.format_exc()
        return

@app.post("/ChatStream")
async def create_item(request: Request):
    global model, tokenizer
    json_post_raw = await request.json()
    json_post = json.dumps(json_post_raw)
    json_post_list = json.loads(json_post)

    prompt = json_post_list.get('prompt')
    max_length = json_post_list.get('max_length')
    top_p = json_post_list.get('top_p')
    temperature = json_post_list.get('temperature')
    history = json_post_list.get('history')
    #   先这样定一下客户端传入 history 的格式，如下：
    #       [["中国的首都是哪里", "中国的首都是北京。"],["他的面积是多少", "北京市的总面积大约为16,800平方公里。"]]
    #   这里在转成 model.stream_chat 需要的格式
    history = [tuple(h) for h in history] 

    answer_text = process(prompt, max_length, top_p, temperature,history)
    return EventSourceResponse(answer_text, media_type="text/event-stream")


if __name__ == '__main__':
    model_path = "D:\llm\THUDM\chatglm2-6b-int4"
    tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
    model = AutoModel.from_pretrained(model_path, trust_remote_code=True).float()

    model = model.eval()
    uvicorn.run(app, host='127.0.0.1', port=8899, workers=1)
                

const btnTest1Ele = document.getElementById("btnTest1");
btnTest1Ele.addEventListener("click", () => {
    let myHeaders = new Headers();
    myHeaders.append("Content-Type", "application/json");
    myHeaders.append("Accept", "text/event-stream");

    //  陆续询问下面这几个问题：
    //  中国的首都是哪里
    //  他的面积是多少
    //  他有机场吗？几座？
    //  下辖哪些行政区域
    const url = "http://127.0.0.1:8899/ChatStream";
    fetch(url, {
        method: "POST",
        headers: myHeaders,
        // body: JSON.stringify({
        //     prompt: "中国的首都是哪里",
        //     history: []
        // })
        // body: JSON.stringify({
        //     prompt: "他的面积是多少",
        //     history: [["中国的首都是哪里", "中国的首都是北京。"]]
        // })
        body: JSON.stringify({
            prompt: "他有机场吗？几座？",
            history: [
                ["中国的首都是哪里", "中国的首都是北京。"],
                ["他的面积是多少", "北京市的总面积大约为16,800平方公里。"]
            ]
        })
    }).then(
        (response) => {
            if (!(response && response.ok)) {
                console.error("异常：" + response.status);
                return;
            }
            let answer = "";
            let reader = response.body.getReader();
            const decoder = new TextDecoder();
            reader.read().then(function processResult(result) {
                if (result.done) {
                    console.log("Event stream ended.");
                    console.log(answer);
                    return;
                }
                try {
                    let decodedString = decoder.decode(result.value);
                    console.log(decodedString);
                    if (decodedString.startsWith("data:")) {
                        let data = JSON.parse(decodedString.slice("data:".length).trim());
                        answer += data.text;
                    }
                } catch (error) {}

                // 继续读取下一个事件流数据
                reader.read().then(processResult);
            });
        },
        (error) => {
            console.error(`异常：${JSON.stringify(error)}`);
        }
    );
});