Skip to content

Commit

Permalink
server : (web UI) add copy button for code block, fix api key (#10242)
Browse files Browse the repository at this point in the history 
* server : (web ui) add copy btn for code blocks

* fix problem with api key

* use settings-modal-short-input component

* always show copy btn for code snippet
  • Loading branch information
@ngxson
ngxson authored Nov 15, 2024
1 parent 231f936 commit 9901068
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 36 deletions.
62 changes: 42 additions & 20 deletions examples/server/public/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
.markdown {
h1, h2, h3, h4, h5, h6, ul, ol, li { all: revert; }
pre {
@apply whitespace-pre-wrap my-4 rounded-lg p-2;
@apply whitespace-pre-wrap rounded-lg p-2;
border: 1px solid currentColor;
}
/* TODO: fix markdown table */
Expand All @@ -25,8 +25,11 @@
.bg-base-200 {background-color: var(--fallback-b2,oklch(var(--b2)/1))}
.bg-base-300 {background-color: var(--fallback-b3,oklch(var(--b3)/1))}
.text-base-content {color: var(--fallback-bc,oklch(var(--bc)/1))}
.show-on-hover {
@apply opacity-0 group-hover:opacity-100;
}
.btn-mini {
@apply cursor-pointer opacity-0 group-hover:opacity-100 hover:shadow-md;
@apply cursor-pointer hover:shadow-md;
}
.chat-screen { max-width: 900px; }
/* because the default bubble color is quite dark, we will make a custom one using bg-base-300 */
Expand Down Expand Up @@ -152,14 +155,14 @@ <h2 class="font-bold mb-4 ml-4">Conversations</h2>
<!-- actions for each message -->
<div :class="{'text-right': msg.role === 'user'}" class="mx-4 mt-2 mb-2">
<!-- user message -->
<button v-if="msg.role === 'user'" class="badge btn-mini" @click="editingMsg = msg" :disabled="isGenerating">
<button v-if="msg.role === 'user'" class="badge btn-minishow-on-hover " @click="editingMsg = msg" :disabled="isGenerating">
✍️ Edit
</button>
<!-- assistant message -->
<button v-if="msg.role === 'assistant'" class="badge btn-mini mr-2" @click="regenerateMsg(msg)" :disabled="isGenerating">
<button v-if="msg.role === 'assistant'" class="badge btn-mini show-on-hover mr-2" @click="regenerateMsg(msg)" :disabled="isGenerating">
🔄 Regenerate
</button>
<button v-if="msg.role === 'assistant'" class="badge btn-mini mr-2" @click="copyMsg(msg)" :disabled="isGenerating">
<button v-if="msg.role === 'assistant'" class="badge btn-mini show-on-hover mr-2" @click="copyMsg(msg)" :disabled="isGenerating">
📋 Copy
</button>
</div>
Expand Down Expand Up @@ -196,20 +199,21 @@ <h2 class="font-bold mb-4 ml-4">Conversations</h2>
<h3 class="text-lg font-bold mb-6">Settings</h3>
<div class="h-[calc(90vh-12rem)] overflow-y-auto">
<p class="opacity-40 mb-6">Settings below are saved in browser's localStorage</p>
<settings-modal-short-input :config-key="'apiKey'" :config-default="configDefault" :config-info="configInfo" v-model="config.apiKey"></settings-modal-short-input>
<label class="form-control mb-2">
<div class="label">System Message</div>
<textarea class="textarea textarea-bordered h-24" :placeholder="'Default: ' + configDefault.systemMessage" v-model="config.systemMessage"></textarea>
</label>
<template v-for="configKey in ['temperature', 'top_k', 'top_p', 'min_p', 'max_tokens']">
<settings-modal-numeric-input :config-key="configKey" :config-default="configDefault" :config-info="configInfo" v-model="config[configKey]" />
<settings-modal-short-input :config-key="configKey" :config-default="configDefault" :config-info="configInfo" v-model="config[configKey]" />
</template>
<!-- TODO: add more sampling-related configs, please regroup them into different "collapse" sections -->
<!-- Section: Other sampler settings -->
<details class="collapse collapse-arrow bg-base-200 mb-2 overflow-visible">
<summary class="collapse-title font-bold">Other sampler settings</summary>
<div class="collapse-content">
<template v-for="configKey in ['dynatemp_range', 'dynatemp_exponent', 'typical_p', 'xtc_probability', 'xtc_threshold']">
<settings-modal-numeric-input :config-key="configKey" :config-default="configDefault" :config-info="configInfo" v-model="config[configKey]" />
<settings-modal-short-input :config-key="configKey" :config-default="configDefault" :config-info="configInfo" v-model="config[configKey]" />
</template>
</div>
</details>
Expand All @@ -218,7 +222,7 @@ <h3 class="text-lg font-bold mb-6">Settings</h3>
<summary class="collapse-title font-bold">Penalties settings</summary>
<div class="collapse-content">
<template v-for="configKey in ['repeat_last_n', 'repeat_penalty', 'presence_penalty', 'frequency_penalty', 'dry_multiplier', 'dry_base', 'dry_allowed_length', 'dry_penalty_last_n']">
<settings-modal-numeric-input :config-key="configKey" :config-default="configDefault" :config-info="configInfo" v-model="config[configKey]" />
<settings-modal-short-input :config-key="configKey" :config-default="configDefault" :config-info="configInfo" v-model="config[configKey]" />
</template>
</div>
</details>
Expand All @@ -245,7 +249,7 @@ <h3 class="text-lg font-bold mb-6">Settings</h3>
</div>

<!-- Template to be used by settings modal -->
<template id="settings-modal-numeric-input">
<template id="settings-modal-short-input">
<label class="input input-bordered join-item grow flex items-center gap-2 mb-2">
<!-- Show help message on hovering on the input label -->
<div class="dropdown dropdown-hover">
Expand All @@ -264,9 +268,13 @@ <h3 class="text-lg font-bold mb-6">Settings</h3>
import { createApp, defineComponent, shallowRef, computed, h } from './deps_vue.esm-browser.js';
import { llama } from './completion.js';

// utility functions
const isString = (x) => !!x.toLowerCase;
const isNumeric = (n) => !isString(n) && !isNaN(n);
const escapeAttr = (str) => str.replace(/>/g, '&gt;').replace(/"/g, '&quot;');
const copyStr = (str) => navigator.clipboard.writeText(str);

// constants
const BASE_URL = localStorage.getItem('base') // for debugging
|| (new URL('.', document.baseURI).href).toString(); // for production
const CONFIG_DEFAULT = {
Expand Down Expand Up @@ -295,7 +303,7 @@ <h3 class="text-lg font-bold mb-6">Settings</h3>
custom: '', // custom json-stringified object
};
const CONFIG_INFO = {
apiKey: '',
apiKey: 'Set the API Key if you are using --api-key option for the server.',
systemMessage: 'The starting message that defines how model should behave.',
temperature: 'Controls the randomness of the generated text by affecting the probability distribution of the output tokens. Higher = more random, lower = more focused.',
dynatemp_range: 'Addon for the temperature sampler. The added value to the range of dynamic temperature, which adjusts probabilities by entropy of tokens.',
Expand Down Expand Up @@ -325,19 +333,28 @@ <h3 class="text-lg font-bold mb-6">Settings</h3>
// markdown support
const VueMarkdown = defineComponent(
(props) => {
const md = shallowRef(new markdownit(props.options ?? { breaks: true }));
for (const plugin of props.plugins ?? []) {
md.value.use(plugin);
}
const md = shallowRef(new markdownit({ breaks: true }));
const origFenchRenderer = md.value.renderer.rules.fence;
md.value.renderer.rules.fence = (tokens, idx, ...args) => {
const content = tokens[idx].content;
const origRendered = origFenchRenderer(tokens, idx, ...args);
return `<div class="relative my-4">
<div class="text-right sticky top-4 mb-2 mr-2 h-0">
<button class="badge btn-mini" onclick="copyStr(${escapeAttr(JSON.stringify(content))})">📋 Copy</button>
</div>
${origRendered}
</div>`;
};
window.copyStr = copyStr;
const content = computed(() => md.value.render(props.source));
return () => h("div", { innerHTML: content.value });
},
{ props: ["source", "options", "plugins"] }
{ props: ["source"] }
);

// inout field to be used by settings modal
const SettingsModalNumericInput = defineComponent({
template: document.getElementById('settings-modal-numeric-input').innerHTML,
const SettingsModalShortInput = defineComponent({
template: document.getElementById('settings-modal-short-input').innerHTML,
props: ['configKey', 'configDefault', 'configInfo', 'modelValue'],
});

Expand Down Expand Up @@ -390,7 +407,11 @@ <h3 class="text-lg font-bold mb-6">Settings</h3>
if (!conv) return;
const msg = conv.messages.pop();
conv.lastModified = Date.now();
localStorage.setItem(convId, JSON.stringify(conv));
if (conv.messages.length === 0) {
StorageUtils.remove(convId);
} else {
localStorage.setItem(convId, JSON.stringify(conv));
}
return msg;
},

Expand Down Expand Up @@ -431,7 +452,7 @@ <h3 class="text-lg font-bold mb-6">Settings</h3>
const mainApp = createApp({
components: {
VueMarkdown,
SettingsModalNumericInput,
SettingsModalShortInput,
},
data() {
return {
Expand Down Expand Up @@ -587,6 +608,7 @@ <h3 class="text-lg font-bold mb-6">Settings</h3>
this.isGenerating = false;
this.stopGeneration = () => {};
this.fetchMessages();
chatScrollToBottom();
},

// message actions
Expand All @@ -600,7 +622,7 @@ <h3 class="text-lg font-bold mb-6">Settings</h3>
this.generateMessage(currConvId);
},
copyMsg(msg) {
navigator.clipboard.writeText(msg.content);
copyStr(msg.content);
},
editUserMsgAndRegenerate(msg) {
if (this.isGenerating) return;
Expand Down
42 changes: 26 additions & 16 deletions examples/server/server.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ struct server_task_result {
bool error;
};

struct server_static_file {
const unsigned char * data;
unsigned int size;
const char * mime_type;
};

struct slot_params {
bool stream = true;
bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt
Expand Down Expand Up @@ -2259,6 +2265,16 @@ int main(int argc, char ** argv) {
LOG_INF("%s\n", common_params_get_system_info(params).c_str());
LOG_INF("\n");

// static files
std::map<std::string, server_static_file> static_files = {
{ "/", { index_html, index_html_len, "text/html; charset=utf-8" }},
{ "/completion.js", { completion_js, completion_js_len, "text/javascript; charset=utf-8" }},
{ "/deps_daisyui.min.css", { deps_daisyui_min_css, deps_daisyui_min_css_len, "text/css; charset=utf-8" }},
{ "/deps_markdown-it.js", { deps_markdown_it_js, deps_markdown_it_js_len, "text/javascript; charset=utf-8" }},
{ "/deps_tailwindcss.js", { deps_tailwindcss_js, deps_tailwindcss_js_len, "text/javascript; charset=utf-8" }},
{ "/deps_vue.esm-browser.js", { deps_vue_esm_browser_js, deps_vue_esm_browser_js_len, "text/javascript; charset=utf-8" }},
};

std::unique_ptr<httplib::Server> svr;
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
if (params.ssl_file_key != "" && params.ssl_file_cert != "") {
Expand Down Expand Up @@ -2339,7 +2355,7 @@ int main(int argc, char ** argv) {
// Middlewares
//

auto middleware_validate_api_key = [&params, &res_error](const httplib::Request & req, httplib::Response & res) {
auto middleware_validate_api_key = [&params, &res_error, &static_files](const httplib::Request & req, httplib::Response & res) {
static const std::unordered_set<std::string> public_endpoints = {
"/health",
"/models",
Expand All @@ -2351,8 +2367,8 @@ int main(int argc, char ** argv) {
return true;
}

// If path is public, skip validation
if (public_endpoints.find(req.path) != public_endpoints.end()) {
// If path is public or is static file, skip validation
if (public_endpoints.find(req.path) != public_endpoints.end() || static_files.find(req.path) != static_files.end()) {
return true;
}

Expand Down Expand Up @@ -3096,13 +3112,6 @@ int main(int argc, char ** argv) {
res.status = 200; // HTTP OK
};

auto handle_static_file = [](unsigned char * content, size_t len, const char * mime_type) {
return [content, len, mime_type](const httplib::Request &, httplib::Response & res) {
res.set_content(reinterpret_cast<const char*>(content), len, mime_type);
return false;
};
};

//
// Router
//
Expand All @@ -3117,12 +3126,13 @@ int main(int argc, char ** argv) {
}
} else {
// using embedded static files
svr->Get("/", handle_static_file(index_html, index_html_len, "text/html; charset=utf-8"));
svr->Get("/completion.js", handle_static_file(completion_js, completion_js_len, "text/javascript; charset=utf-8"));
svr->Get("/deps_daisyui.min.css", handle_static_file(deps_daisyui_min_css, deps_daisyui_min_css_len, "text/css; charset=utf-8"));
svr->Get("/deps_markdown-it.js", handle_static_file(deps_markdown_it_js, deps_markdown_it_js_len, "text/javascript; charset=utf-8"));
svr->Get("/deps_tailwindcss.js", handle_static_file(deps_tailwindcss_js, deps_tailwindcss_js_len, "text/javascript; charset=utf-8"));
svr->Get("/deps_vue.esm-browser.js", handle_static_file(deps_vue_esm_browser_js, deps_vue_esm_browser_js_len, "text/javascript; charset=utf-8"));
for (const auto & it : static_files) {
const server_static_file & static_file = it.second;
svr->Get(it.first.c_str(), [&static_file](const httplib::Request &, httplib::Response & res) {
res.set_content(reinterpret_cast<const char*>(static_file.data), static_file.size, static_file.mime_type);
return false;
});
}
}

// register API routes
Expand Down

0 comments on commit 9901068

Please sign in to comment.