Escape characters in Regex evaluations, minor UI fixes (#56)

* Fix ScenariosHeader stickiness

* Move meta tag from _app.tsx to _document.tsx

* Show spinner when saving variant

* Escape quotes and regex in evaluations

src/server/utils/fillTemplate.ts

@@ -2,6 +2,16 @@ import { type JSONSerializable } from "../types";
 
 export type VariableMap = Record<string, string>;
 
+// Escape quotes to match the way we encode JSON
+export function escapeQuotes(str: string) {
+  return str.replace(/(\\")|"/g, (match, p1) => (p1 ? match : '\\"'));
+}
+
+// Escape regex special characters
+export function escapeRegExp(str: string) {
+  return str.replace(/[.*+\-?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
+}
+
 export function fillTemplate(template: string, variables: VariableMap): string {
   return template.replace(/{{\s*(\w+)\s*}}/g, (_, key: string) => variables[key] || "");
 }

src/server/utils/runOneEval.ts

@@ -1,6 +1,6 @@
 import { type Evaluation, type ModelOutput, type TestScenario } from "@prisma/client";
 import { type ChatCompletion } from "openai/resources/chat";
-import { type VariableMap, fillTemplate } from "./fillTemplate";
+import { type VariableMap, fillTemplate, escapeRegExp, escapeQuotes } from "./fillTemplate";
 import { openai } from "./openai";
 import dedent from "dedent";
 
@@ -80,7 +80,7 @@ export const runOneEval = async (
 
   const stringifiedMessage = message.content ?? JSON.stringify(message.function_call);
 
-  const matchRegex = fillTemplate(evaluation.value, scenario.variableValues as VariableMap);
+  const matchRegex = escapeRegExp(fillTemplate(escapeQuotes(evaluation.value), scenario.variableValues as VariableMap));
 
   switch (evaluation.evalType) {
     case "CONTAINS":