Under some conditions agent responses are attributed to Copilot instead of the agent

[tspascoal]

When the response is streamed, under some conditions the agent response is attributed to copilot instead of the extension:

Note

This happens only on dotcom chat, VSCode always attributes the response to the agent

I haven't figured the root cause, but it seems related to the use of createAckEvent and createDoneEvent in conjunction with streaming. This happens when using both built in SDK methods for prompt and using openai library against copilot endpoint.

The resulting payload doesn't seem to be consistent either with streaming/non streaming (under some conditions createDoneEvent is redundant)

I've included the four conditions, together with the response payloads (made them as small as possible).

I've used the following pattern:

Send ACK()
CallGENAIAndSendResponse() // using all four combinations
SendDone()

The full source code used for the repro is included at the end of the issue (no concerns in making the code pretty :) )

Using Prompt method ✅

Payload

data: {"choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}

data: {"choices":[{"index":0,"delta":{"content":"No.","role":"assistant"}}]}

data: {"choices":[{"index":0,"finish_reason":"stop","delta":{"content":null}}]}

data: [DONE]

Using prompt method with streaming 🔴

Payload

data: {"choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}

data: {"choices":[],"created":0,"id":"","prompt_filter_results":[{"content_filter_results":{"error":{"code":"","message":""},"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"prompt_index":0}]}

data: {"choices":[{"index":0,"content_filter_offsets":{"check_offset":479,"start_offset":479,"end_offset":482},"content_filter_results":{"error":{"code":"","message":""},"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"","role":"assistant"}}],"created":1728829509,"id":"chatcmpl-AHtoruqQKpZydRFXYyQBmgx0VXKBN","model":"gpt-4o-2024-05-13","system_fingerprint":"fp_67802d9a6d"}

data: {"choices":[{"index":0,"content_filter_offsets":{"check_offset":479,"start_offset":479,"end_offset":482},"content_filter_results":{"error":{"code":"","message":""},"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"No"}}],"created":1728829509,"id":"chatcmpl-AHtoruqQKpZydRFXYyQBmgx0VXKBN","model":"gpt-4o-2024-05-13","system_fingerprint":"fp_67802d9a6d"}

data: {"choices":[{"index":0,"content_filter_offsets":{"check_offset":479,"start_offset":479,"end_offset":482},"content_filter_results":{"error":{"code":"","message":""},"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"."}}],"created":1728829509,"id":"chatcmpl-AHtoruqQKpZydRFXYyQBmgx0VXKBN","model":"gpt-4o-2024-05-13","system_fingerprint":"fp_67802d9a6d"}

data: {"choices":[{"finish_reason":"stop","index":0,"content_filter_offsets":{"check_offset":479,"start_offset":479,"end_offset":482},"content_filter_results":{"error":{"code":"","message":""},"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":null}}],"created":1728829509,"id":"chatcmpl-AHtoruqQKpZydRFXYyQBmgx0VXKBN","usage":{"completion_tokens":2,"prompt_tokens":97,"total_tokens":99},"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_67802d9a6d"}

data: [DONE]

data: {"choices":[{"index":0,"finish_reason":"stop","delta":{"content":null}}]}

data: [DONE]

Workaround: Don't send createDoneEvent() Which basically removes the two last responses on the above payload (the streaming response already included data: [DONE])

Using CAPI with no stream ✅

Payload

data: {"choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}

data: {"choices":[{"index":0,"delta":{"content":"No.","role":"assistant"}}]}

data: {"choices":[{"index":0,"finish_reason":"stop","delta":{"content":null}}]}

data: [DONE]

Using CAPI with streaming 🔴

Payload

data: {"choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}

data: {"choices":[],"created":0,"id":"","prompt_filter_results":[{"content_filter_results":{"error":{"code":"","message":""},"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"prompt_index":0}]}

data: {"choices":[{"index":0,"content_filter_offsets":{"check_offset":420,"start_offset":420,"end_offset":423},"content_filter_results":{"error":{"code":"","message":""},"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"","role":"assistant"}}],"created":1728830285,"id":"chatcmpl-AHu1NCHjVZ15y0IX5x0g53IkRqA8u","model":"gpt-4o-2024-05-13","system_fingerprint":"fp_67802d9a6d"}

data: {"choices":[{"index":0,"content_filter_offsets":{"check_offset":420,"start_offset":420,"end_offset":423},"content_filter_results":{"error":{"code":"","message":""},"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"No"}}],"created":1728830285,"id":"chatcmpl-AHu1NCHjVZ15y0IX5x0g53IkRqA8u","model":"gpt-4o-2024-05-13","system_fingerprint":"fp_67802d9a6d"}

data: {"choices":[{"index":0,"content_filter_offsets":{"check_offset":420,"start_offset":420,"end_offset":423},"content_filter_results":{"error":{"code":"","message":""},"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"."}}],"created":1728830285,"id":"chatcmpl-AHu1NCHjVZ15y0IX5x0g53IkRqA8u","model":"gpt-4o-2024-05-13","system_fingerprint":"fp_67802d9a6d"}

data: {"choices":[{"finish_reason":"stop","index":0,"content_filter_offsets":{"check_offset":420,"start_offset":420,"end_offset":423},"content_filter_results":{"error":{"code":"","message":""},"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":null}}],"created":1728830285,"id":"chatcmpl-AHu1NCHjVZ15y0IX5x0g53IkRqA8u","usage":{"completion_tokens":2,"prompt_tokens":87,"total_tokens":89},"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_67802d9a6d"}

data: {"choices":[{"index":0,"finish_reason":"stop","delta":{"content":null}}]}

data: [DONE]

Workaround don't send createDoneEvent() even though the streaming response doesn't sends it explicitly, the client seems to be lenient (but probably not a good idea not to send it)

Full Repro code

Dependencies

{
    "@copilot-extensions/preview-sdk": "^5.0.0",
    "openai": "^4.67.1"
}

Source Code

import { createServer, IncomingMessage, ServerResponse } from "node:http";
import {
    CopilotRequestPayload,
    prompt,
    parseRequestBody,
    createAckEvent,
    createTextEvent,
    createDoneEvent,
    getUserMessage
} from "@copilot-extensions/preview-sdk";
import OpenAI from "openai";

if (process.env.NODE_ENV === 'production') {
    console.debug = function () { };
}

const MODEL = "gpt-4o";

const server = createServer(async (request: IncomingMessage, response: ServerResponse) => {
    console.log(`handling url ${request.url} with method ${request.method}`);

    if (request.url?.startsWith("/auth/authorization")) {
        return returnResponse(response, 200, "Auth Configured.", "Auth callback received");
    } else if (request.url?.startsWith("/auth/callback")) {
        return returnResponse(response, 200, "You can now use the agent. You can revoke the authorization in your settings page", "Auth callback received");
    } else if (request.method === "GET") {
        return returnResponse(response, 200, "OK");
    }

    console.log("Request received");
    console.time("processing");

    const body = await getBody(request);
    const apiKey = request.headers["x-github-token"] as string;

    if (!apiKey) {
        return returnResponse(response, 400, "Missing header", "Missing header x-github-token");
    }

    const payload = parseRequestBody(body);
    const userPrompt = getUserMessage(payload);

    console.log("Processing request");

    response.write(createAckEvent())

    switch (userPrompt) {
        case 'prompt':
            await replyPrompt(payload, apiKey, response);
            break;
        case 'prompt-streaming':
            await replyPromptStreaming(payload, apiKey, response);
            break;
        case 'capi':
            await replyCAPI(payload, apiKey, response);
            break;
        case 'capi-streaming':
            await replyCAPIStreaming(payload, apiKey, response);
            break;
        default: response.write(createTextEvent("only prompt, prompt-streaming, capi, capi-streaming are supported"));
    }

    response.write(createDoneEvent());
    returnResponse(response, 200, "", "Done Event Sent");
    console.timeEnd("processing");
});


const port = process.env.PORT || 3000;

server.listen(port);
console.log(`Server running on http://localhost:${port}`);

async function replyPrompt(payload: CopilotRequestPayload, apiKey: string, response: ServerResponse) {
    const message = await prompt({
        model: MODEL,
        token: apiKey,
        messages: getPrompt()
    });

    response.write(createTextEvent(message?.message.content ?? "Ooooops. You got me. I have no answer for that."));
}

async function replyPromptStreaming(payload: CopilotRequestPayload, apiKey: string, response: ServerResponse) {

    const { stream } = await prompt.stream({
        model: MODEL,
        token: apiKey,
        messages: getPrompt()
    });
    for await (const chunk of stream) {
        const decodedChunk = new TextDecoder().decode(chunk);
        response.write(decodedChunk);
    }
}

async function replyCAPI(payload: CopilotRequestPayload, apiKey: string, response: ServerResponse) {
    const capiClient = new OpenAI({ baseURL: "https://api.githubcopilot.com", apiKey });

    const result = await capiClient.chat.completions.create({
        stream: false,
        model: MODEL,
        messages: getPrompt()
    });

    if (result.choices[0].message?.content) {
        response.write(createTextEvent(result.choices[0].message.content || "Ooooops. You got me. I have no answer for that."));
    }
}

async function replyCAPIStreaming(payload: CopilotRequestPayload, apiKey: string, response: ServerResponse) {
    const capiClient = new OpenAI({ baseURL: "https://api.githubcopilot.com", apiKey });

    const completionResponseStream = await capiClient.chat.completions.create({
        stream: true,
        model: MODEL,
        messages: getPrompt()
    });

    for await (const chunk of completionResponseStream) {
        const chunkStr = "data: " + JSON.stringify(chunk) + "\n\n";
        response.write(chunkStr);
        console.debug(chunkStr);
    }
}

function getPrompt(): any {
    return [
        {
            role: "system",
            content: [
                "You are an extension of GitHub Copilot, built allways say no.",
                "Whatever It is asked, you should always say no.",
                "You should never answer any question.",
                "You should never provide any information.",
                "You should never provide any help.",
                "You should never provide any guidance.",
                "always say no. That is it"
            ].join("\n"),
        },
        {
            "role": "user",
            "content": "What do you say if I ask you a non programming related question?"
        }
    ];
}

function getBody(req: IncomingMessage): Promise<string> {
    return new Promise((resolve, reject) => {
        let data = '';
        req.on('data', chunk => {
            data += chunk;
        });
        req.on('end', () => {
            resolve(data);
        });
    });
}

function returnResponse(response: ServerResponse, statusCode: number, body: string, logMessage?: string) {
    if (logMessage) console.log(logMessage);
    response.statusCode = statusCode;
    response.end(body);
}