Compare commits
16 Commits
bump-model
...
gh-btn
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5945f0ed6b | ||
|
|
6bc7d76d15 | ||
|
|
e9ed173e34 | ||
|
|
75d58d7021 | ||
|
|
896c8c5c57 | ||
|
|
ec5547d0b0 | ||
|
|
77e4e3b8c3 | ||
|
|
a1b03ddad1 | ||
|
|
6be32bea4c | ||
|
|
72c70e2a55 | ||
|
|
026532f2c2 | ||
|
|
f88538336f | ||
|
|
3c7178115e | ||
|
|
292aaf090a | ||
|
|
d9915dc41b | ||
|
|
3560bcff14 |
41
README.md
41
README.md
@@ -6,40 +6,47 @@ OpenPipe is a flexible playground for comparing and optimizing LLM prompts. It l
|
||||
|
||||
## Sample Experiments
|
||||
|
||||
These are simple experiments users have created that show how OpenPipe works.
|
||||
These are simple experiments users have created that show how OpenPipe works. Feel free to fork them and start experimenting yourself.
|
||||
|
||||
- [Country Capitals](https://app.openpipe.ai/experiments/11111111-1111-1111-1111-111111111111)
|
||||
- [Twitter Sentiment Analysis](https://app.openpipe.ai/experiments/62c20a73-2012-4a64-973c-4b665ad46a57)
|
||||
- [Reddit User Needs](https://app.openpipe.ai/experiments/22222222-2222-2222-2222-222222222222)
|
||||
- [OpenAI Function Calls](https://app.openpipe.ai/experiments/2ebbdcb3-ed51-456e-87dc-91f72eaf3e2b)
|
||||
- [Activity Classification](https://app.openpipe.ai/experiments/3950940f-ab6b-4b74-841d-7e9dbc4e4ff8)
|
||||
|
||||
<img src="https://github.com/openpipe/openpipe/assets/176426/fc7624c6-5b65-4d4d-82b7-4a816f3e5678" alt="demo" height="400px">
|
||||
<img src="https://github.com/openpipe/openpipe/assets/41524992/219a844e-3f4e-4f6b-8066-41348b42977b" alt="demo">
|
||||
|
||||
You can use our hosted version of OpenPipe at [https://openpipe.ai]. You can also clone this repository and [run it locally](#running-locally).
|
||||
You can use our hosted version of OpenPipe at https://openpipe.ai. You can also clone this repository and [run it locally](#running-locally).
|
||||
|
||||
## High-Level Features
|
||||
|
||||
**Configure Multiple Prompts**
|
||||
Set up multiple prompt configurations and compare their output side-by-side. Each configuration can be configured independently.
|
||||
|
||||
**Visualize Responses**
|
||||
Inspect prompt completions side-by-side.
|
||||
|
||||
**Test Many Inputs**
|
||||
OpenPipe lets you _template_ a prompt. Use the templating feature to run the prompts you're testing against many potential inputs for broader coverage of your problem space than you'd get with manual testing.
|
||||
<br>
|
||||
|
||||
**Test Many Inputs**
|
||||
OpenPipe lets you _template_ a prompt. Use the templating feature to run the prompts you're testing against many potential inputs for broad coverage of your problem space.
|
||||
|
||||
<br>
|
||||
|
||||
**Translate between Model APIs**
|
||||
Write your prompt in one format and automatically convert it to work with any other model.
|
||||
|
||||
<img width="480" alt="Screenshot 2023-08-01 at 11 55 38 PM" src="https://github.com/OpenPipe/OpenPipe/assets/41524992/1e19ccf2-96b6-4e93-a3a5-1449710d1b5b" alt="translate between models">
|
||||
|
||||
<br><br>
|
||||
**Refine your prompts automatically**
|
||||
Use a growing database of best-practice refinements to improve your prompts automatically.
|
||||
|
||||
<img width="480" alt="Screenshot 2023-08-01 at 11 55 38 PM" src="https://github.com/OpenPipe/OpenPipe/assets/41524992/87a27fe7-daef-445c-a5e2-1c82b23f9f99" alt="add function call">
|
||||
|
||||
<br><br>
|
||||
**🪄 Auto-generate Test Scenarios**
|
||||
OpenPipe includes a tool to generate new test scenarios based on your existing prompts and scenarios. Just click "Autogenerate Scenario" to try it out!
|
||||
|
||||
**Prompt Validation and Typeahead**
|
||||
We use OpenAI's OpenAPI spec to automatically provide typeahead and validate prompts.
|
||||
<img width="600" src="https://github.com/openpipe/openpipe/assets/41524992/219a844e-3f4e-4f6b-8066-41348b42977b" alt="auto-generate">
|
||||
|
||||
<img alt="typeahead" src="https://github.com/openpipe/openpipe/assets/176426/acc638f8-d851-4742-8d01-fe6f98890840" height="300px">
|
||||
|
||||
**Function Call Support**
|
||||
Natively supports [OpenAI function calls](https://openai.com/blog/function-calling-and-other-api-updates) on supported models.
|
||||
|
||||
<img height="300px" alt="function calls" src="https://github.com/openpipe/openpipe/assets/176426/48ad13fe-af2f-4294-bf32-62015597fd9b">
|
||||
<br><br>
|
||||
|
||||
## Supported Models
|
||||
|
||||
|
||||
@@ -25,8 +25,10 @@
|
||||
"@apidevtools/json-schema-ref-parser": "^10.1.0",
|
||||
"@babel/preset-typescript": "^7.22.5",
|
||||
"@babel/standalone": "^7.22.9",
|
||||
"@chakra-ui/anatomy": "^2.2.0",
|
||||
"@chakra-ui/next-js": "^2.1.4",
|
||||
"@chakra-ui/react": "^2.7.1",
|
||||
"@chakra-ui/styled-system": "^2.9.1",
|
||||
"@emotion/react": "^11.11.1",
|
||||
"@emotion/server": "^11.11.0",
|
||||
"@emotion/styled": "^11.11.0",
|
||||
@@ -63,7 +65,7 @@
|
||||
"next-auth": "^4.22.1",
|
||||
"next-query-params": "^4.2.3",
|
||||
"nextjs-routes": "^2.0.1",
|
||||
"openai": "4.0.0-beta.2",
|
||||
"openai": "4.0.0-beta.7",
|
||||
"pluralize": "^8.0.0",
|
||||
"posthog-js": "^1.68.4",
|
||||
"prettier": "^3.0.0",
|
||||
@@ -71,7 +73,9 @@
|
||||
"react": "18.2.0",
|
||||
"react-diff-viewer": "^3.1.1",
|
||||
"react-dom": "18.2.0",
|
||||
"react-github-btn": "^1.4.0",
|
||||
"react-icons": "^4.10.1",
|
||||
"react-json-tree": "^0.18.0",
|
||||
"react-select": "^5.7.4",
|
||||
"react-syntax-highlighter": "^15.5.0",
|
||||
"react-textarea-autosize": "^8.5.0",
|
||||
|
||||
108
pnpm-lock.yaml
generated
108
pnpm-lock.yaml
generated
@@ -17,12 +17,18 @@ dependencies:
|
||||
'@babel/standalone':
|
||||
specifier: ^7.22.9
|
||||
version: 7.22.9
|
||||
'@chakra-ui/anatomy':
|
||||
specifier: ^2.2.0
|
||||
version: 2.2.0
|
||||
'@chakra-ui/next-js':
|
||||
specifier: ^2.1.4
|
||||
version: 2.1.4(@chakra-ui/react@2.7.1)(@emotion/react@11.11.1)(next@13.4.2)(react@18.2.0)
|
||||
'@chakra-ui/react':
|
||||
specifier: ^2.7.1
|
||||
version: 2.7.1(@emotion/react@11.11.1)(@emotion/styled@11.11.0)(@types/react@18.2.6)(framer-motion@10.12.17)(react-dom@18.2.0)(react@18.2.0)
|
||||
'@chakra-ui/styled-system':
|
||||
specifier: ^2.9.1
|
||||
version: 2.9.1
|
||||
'@emotion/react':
|
||||
specifier: ^11.11.1
|
||||
version: 11.11.1(@types/react@18.2.6)(react@18.2.0)
|
||||
@@ -132,8 +138,8 @@ dependencies:
|
||||
specifier: ^2.0.1
|
||||
version: 2.0.1(next@13.4.2)
|
||||
openai:
|
||||
specifier: 4.0.0-beta.2
|
||||
version: 4.0.0-beta.2
|
||||
specifier: 4.0.0-beta.7
|
||||
version: 4.0.0-beta.7
|
||||
pluralize:
|
||||
specifier: ^8.0.0
|
||||
version: 8.0.0
|
||||
@@ -155,9 +161,15 @@ dependencies:
|
||||
react-dom:
|
||||
specifier: 18.2.0
|
||||
version: 18.2.0(react@18.2.0)
|
||||
react-github-btn:
|
||||
specifier: ^1.4.0
|
||||
version: 1.4.0(react@18.2.0)
|
||||
react-icons:
|
||||
specifier: ^4.10.1
|
||||
version: 4.10.1(react@18.2.0)
|
||||
react-json-tree:
|
||||
specifier: ^0.18.0
|
||||
version: 0.18.0(@types/react@18.2.6)(react@18.2.0)
|
||||
react-select:
|
||||
specifier: ^5.7.4
|
||||
version: 5.7.4(@types/react@18.2.6)(react-dom@18.2.0)(react@18.2.0)
|
||||
@@ -686,6 +698,10 @@ packages:
|
||||
resolution: {integrity: sha512-pKfOS/mztc4sUXHNc8ypJ1gPWSolWT770jrgVRfolVbYlki8y5Y+As996zMF6k5lewTu6j9DQequ7Cc9a69IVQ==}
|
||||
dev: false
|
||||
|
||||
/@chakra-ui/anatomy@2.2.0:
|
||||
resolution: {integrity: sha512-cD8Ms5C8+dFda0LrORMdxiFhAZwOIY1BSlCadz6/mHUIgNdQy13AHPrXiq6qWdMslqVHq10k5zH7xMPLt6kjFg==}
|
||||
dev: false
|
||||
|
||||
/@chakra-ui/avatar@2.2.11(@chakra-ui/system@2.5.8)(react@18.2.0):
|
||||
resolution: {integrity: sha512-CJFkoWvlCTDJTUBrKA/aVyG5Zz6TBEIVmmsJtqC6VcQuVDTxkWod8ruXnjb0LT2DUveL7xR5qZM9a5IXcsH3zg==}
|
||||
peerDependencies:
|
||||
@@ -2831,6 +2847,10 @@ packages:
|
||||
'@babel/types': 7.22.5
|
||||
dev: true
|
||||
|
||||
/@types/base16@1.0.2:
|
||||
resolution: {integrity: sha512-oYO/U4VD1DavwrKuCSQWdLG+5K22SLPem2OQaHmFcQuwHoVeGC+JGVRji2MUqZUAIQZHEonOeVfAX09hYiLsdg==}
|
||||
dev: false
|
||||
|
||||
/@types/body-parser@1.19.2:
|
||||
resolution: {integrity: sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==}
|
||||
dependencies:
|
||||
@@ -3013,6 +3033,7 @@ packages:
|
||||
|
||||
/@types/qs@6.9.7:
|
||||
resolution: {integrity: sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==}
|
||||
dev: true
|
||||
|
||||
/@types/range-parser@1.2.4:
|
||||
resolution: {integrity: sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==}
|
||||
@@ -3658,6 +3679,10 @@ packages:
|
||||
resolution: {integrity: sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==}
|
||||
dev: false
|
||||
|
||||
/base16@1.0.0:
|
||||
resolution: {integrity: sha512-pNdYkNPiJUnEhnfXV56+sQy8+AaPcG3POZAUnwr4EeqCUZFz4u2PePbo3e5Gj4ziYPCWGUZT9RHisvJKnwFuBQ==}
|
||||
dev: false
|
||||
|
||||
/base64-js@0.0.8:
|
||||
resolution: {integrity: sha512-3XSA2cR/h/73EzlXXdU6YNycmYI7+kicTxks4eJg2g39biHR84slg2+des+p7iHYhbRg/udIS4TD53WabcOUkw==}
|
||||
engines: {node: '>= 0.4'}
|
||||
@@ -3916,10 +3941,24 @@ packages:
|
||||
/color-name@1.1.4:
|
||||
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
|
||||
|
||||
/color-string@1.9.1:
|
||||
resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==}
|
||||
dependencies:
|
||||
color-name: 1.1.4
|
||||
simple-swizzle: 0.2.2
|
||||
dev: false
|
||||
|
||||
/color2k@2.0.2:
|
||||
resolution: {integrity: sha512-kJhwH5nAwb34tmyuqq/lgjEKzlFXn1U99NlnB6Ws4qVaERcRUYeYP1cBw6BJ4vxaWStAUEef4WMr7WjOCnBt8w==}
|
||||
dev: false
|
||||
|
||||
/color@3.2.1:
|
||||
resolution: {integrity: sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==}
|
||||
dependencies:
|
||||
color-convert: 1.9.3
|
||||
color-string: 1.9.1
|
||||
dev: false
|
||||
|
||||
/combined-stream@1.0.8:
|
||||
resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
|
||||
engines: {node: '>= 0.8'}
|
||||
@@ -5210,6 +5249,10 @@ packages:
|
||||
dependencies:
|
||||
resolve-pkg-maps: 1.0.0
|
||||
|
||||
/github-buttons@2.27.0:
|
||||
resolution: {integrity: sha512-PmfRMI2Rttg/2jDfKBeSl621sEznrsKF019SuoLdoNlO7qRUZaOyEI5Li4uW+79pVqnDtKfIEVuHTIJ5lgy64w==}
|
||||
dev: false
|
||||
|
||||
/glob-parent@5.1.2:
|
||||
resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}
|
||||
engines: {node: '>= 6'}
|
||||
@@ -5536,6 +5579,10 @@ packages:
|
||||
resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==}
|
||||
dev: false
|
||||
|
||||
/is-arrayish@0.3.2:
|
||||
resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==}
|
||||
dev: false
|
||||
|
||||
/is-bigint@1.0.4:
|
||||
resolution: {integrity: sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==}
|
||||
dependencies:
|
||||
@@ -5910,6 +5957,10 @@ packages:
|
||||
resolution: {integrity: sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==}
|
||||
dev: false
|
||||
|
||||
/lodash.curry@4.1.1:
|
||||
resolution: {integrity: sha512-/u14pXGviLaweY5JI0IUzgzF2J6Ne8INyzAZjImcryjgkZ+ebruBxy2/JaOOkTqScddcYtakjhSaeemV8lR0tA==}
|
||||
dev: false
|
||||
|
||||
/lodash.merge@4.6.2:
|
||||
resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==}
|
||||
dev: true
|
||||
@@ -6387,19 +6438,17 @@ packages:
|
||||
is-wsl: 2.2.0
|
||||
dev: true
|
||||
|
||||
/openai@4.0.0-beta.2:
|
||||
resolution: {integrity: sha512-zTuAxBFe5nSO7LngbV+/O0udtgHWfXb2lFei8/sDY4GB5cOdnrRoSOtiyUfV65ANdvlI4F75oYZX7w067cxj3w==}
|
||||
/openai@4.0.0-beta.7:
|
||||
resolution: {integrity: sha512-jHjwvpMuGkNxiQ3erwLZsOvPEhcVrMtwtfNeYmGCjhbdB+oStVw/7pIhIPkualu8rlhLwgMR7awknIaN3IQcOA==}
|
||||
dependencies:
|
||||
'@types/node': 18.16.0
|
||||
'@types/node-fetch': 2.6.4
|
||||
'@types/qs': 6.9.7
|
||||
abort-controller: 3.0.0
|
||||
agentkeepalive: 4.3.0
|
||||
digest-fetch: 1.3.0
|
||||
form-data-encoder: 1.7.2
|
||||
formdata-node: 4.4.1
|
||||
node-fetch: 2.6.12
|
||||
qs: 6.11.2
|
||||
transitivePeerDependencies:
|
||||
- encoding
|
||||
- supports-color
|
||||
@@ -6832,13 +6881,6 @@ packages:
|
||||
side-channel: 1.0.4
|
||||
dev: false
|
||||
|
||||
/qs@6.11.2:
|
||||
resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==}
|
||||
engines: {node: '>=0.6'}
|
||||
dependencies:
|
||||
side-channel: 1.0.4
|
||||
dev: false
|
||||
|
||||
/queue-microtask@1.2.3:
|
||||
resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==}
|
||||
dev: true
|
||||
@@ -6875,6 +6917,18 @@ packages:
|
||||
webpack: 5.88.2
|
||||
dev: true
|
||||
|
||||
/react-base16-styling@0.9.1:
|
||||
resolution: {integrity: sha512-1s0CY1zRBOQ5M3T61wetEpvQmsYSNtWEcdYzyZNxKa8t7oDvaOn9d21xrGezGAHFWLM7SHcktPuPTrvoqxSfKw==}
|
||||
dependencies:
|
||||
'@babel/runtime': 7.22.6
|
||||
'@types/base16': 1.0.2
|
||||
'@types/lodash': 4.14.195
|
||||
base16: 1.0.0
|
||||
color: 3.2.1
|
||||
csstype: 3.1.2
|
||||
lodash.curry: 4.1.1
|
||||
dev: false
|
||||
|
||||
/react-clientside-effect@1.2.6(react@18.2.0):
|
||||
resolution: {integrity: sha512-XGGGRQAKY+q25Lz9a/4EPqom7WRjz3z9R2k4jhVKA/puQFH/5Nt27vFZYql4m4NVNdUvX8PS3O7r/Zzm7cjUlg==}
|
||||
peerDependencies:
|
||||
@@ -6934,6 +6988,15 @@ packages:
|
||||
use-sidecar: 1.1.2(@types/react@18.2.6)(react@18.2.0)
|
||||
dev: false
|
||||
|
||||
/react-github-btn@1.4.0(react@18.2.0):
|
||||
resolution: {integrity: sha512-lV4FYClAfjWnBfv0iNlJUGhamDgIq6TayD0kPZED6VzHWdpcHmPfsYOZ/CFwLfPv4Zp+F4m8QKTj0oy2HjiGXg==}
|
||||
peerDependencies:
|
||||
react: '>=16.3.0'
|
||||
dependencies:
|
||||
github-buttons: 2.27.0
|
||||
react: 18.2.0
|
||||
dev: false
|
||||
|
||||
/react-icons@4.10.1(react@18.2.0):
|
||||
resolution: {integrity: sha512-/ngzDP/77tlCfqthiiGNZeYFACw85fUjZtLbedmJ5DTlNDIwETxhwBzdOJ21zj4iJdvc0J3y7yOsX3PpxAJzrw==}
|
||||
peerDependencies:
|
||||
@@ -6949,6 +7012,19 @@ packages:
|
||||
resolution: {integrity: sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==}
|
||||
dev: true
|
||||
|
||||
/react-json-tree@0.18.0(@types/react@18.2.6)(react@18.2.0):
|
||||
resolution: {integrity: sha512-Qe6HKSXrr++n9Y31nkRJ3XvQMATISpqigH1vEKhLwB56+nk5thTP0ITThpjxY6ZG/ubpVq/aEHIcyLP/OPHxeA==}
|
||||
peerDependencies:
|
||||
'@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0
|
||||
react: ^16.8.0 || ^17.0.0 || ^18.0.0
|
||||
dependencies:
|
||||
'@babel/runtime': 7.22.6
|
||||
'@types/lodash': 4.14.195
|
||||
'@types/react': 18.2.6
|
||||
react: 18.2.0
|
||||
react-base16-styling: 0.9.1
|
||||
dev: false
|
||||
|
||||
/react-remove-scroll-bar@2.3.4(@types/react@18.2.6)(react@18.2.0):
|
||||
resolution: {integrity: sha512-63C4YQBUt0m6ALadE9XV56hV8BgJWDmmTPY758iIJjfQKt2nYwoUrPk0LXRXcB/yIj82T1/Ixfdpdk68LwIB0A==}
|
||||
engines: {node: '>=10'}
|
||||
@@ -7360,6 +7436,12 @@ packages:
|
||||
resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==}
|
||||
dev: true
|
||||
|
||||
/simple-swizzle@0.2.2:
|
||||
resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==}
|
||||
dependencies:
|
||||
is-arrayish: 0.3.2
|
||||
dev: false
|
||||
|
||||
/slash@3.0.0:
|
||||
resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
@@ -22,8 +22,8 @@ export const ModelSearch = (props: {
|
||||
const [containerRef, containerDimensions] = useElementDimensions();
|
||||
|
||||
return (
|
||||
<VStack ref={containerRef as LegacyRef<HTMLDivElement>} w="full">
|
||||
<Text>Browse Models</Text>
|
||||
<VStack ref={containerRef as LegacyRef<HTMLDivElement>} w="full" fontFamily="inconsolata">
|
||||
<Text fontWeight="bold">Browse Models</Text>
|
||||
<Select<ProviderModel>
|
||||
styles={{ control: (provided) => ({ ...provided, width: containerDimensions?.width }) }}
|
||||
getOptionLabel={(data) => modelLabel(data.provider, data.model)}
|
||||
|
||||
@@ -23,16 +23,24 @@ export const ModelStatsCard = ({
|
||||
{label}
|
||||
</Text>
|
||||
|
||||
<VStack w="full" spacing={6} bgColor="gray.100" p={4} borderRadius={4}>
|
||||
<VStack
|
||||
w="full"
|
||||
spacing={6}
|
||||
borderWidth={1}
|
||||
borderColor="gray.300"
|
||||
p={4}
|
||||
borderRadius={8}
|
||||
fontFamily="inconsolata"
|
||||
>
|
||||
<HStack w="full" align="flex-start">
|
||||
<Text flex={1} fontSize="lg">
|
||||
<Text as="span" color="gray.600">
|
||||
{model.provider} /{" "}
|
||||
</Text>
|
||||
<VStack flex={1} fontSize="lg" alignItems="flex-start">
|
||||
<Text as="span" fontWeight="bold" color="gray.900">
|
||||
{model.name}
|
||||
</Text>
|
||||
</Text>
|
||||
<Text as="span" color="gray.600" fontSize="sm">
|
||||
Provider: {model.provider}
|
||||
</Text>
|
||||
</VStack>
|
||||
<Link
|
||||
href={model.learnMoreUrl}
|
||||
isExternal
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
import { type StackProps, VStack } from "@chakra-ui/react";
|
||||
import { CellOptions } from "./CellOptions";
|
||||
|
||||
export const CellContent = ({
|
||||
hardRefetch,
|
||||
hardRefetching,
|
||||
children,
|
||||
...props
|
||||
}: {
|
||||
hardRefetch: () => void;
|
||||
hardRefetching: boolean;
|
||||
} & StackProps) => (
|
||||
<VStack w="full" alignItems="flex-start" {...props}>
|
||||
<CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} />
|
||||
<VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto">
|
||||
{children}
|
||||
</VStack>
|
||||
</VStack>
|
||||
);
|
||||
@@ -1,37 +0,0 @@
|
||||
import { Button, HStack, Icon, Spinner, Tooltip } from "@chakra-ui/react";
|
||||
import { BsArrowClockwise } from "react-icons/bs";
|
||||
import { useExperimentAccess } from "~/utils/hooks";
|
||||
|
||||
export const CellOptions = ({
|
||||
refetchingOutput,
|
||||
refetchOutput,
|
||||
}: {
|
||||
refetchingOutput: boolean;
|
||||
refetchOutput: () => void;
|
||||
}) => {
|
||||
const { canModify } = useExperimentAccess();
|
||||
return (
|
||||
<HStack justifyContent="flex-end" w="full">
|
||||
{canModify && (
|
||||
<Tooltip label="Refetch output" aria-label="refetch output">
|
||||
<Button
|
||||
size="xs"
|
||||
w={4}
|
||||
h={4}
|
||||
py={4}
|
||||
px={4}
|
||||
minW={0}
|
||||
borderRadius={8}
|
||||
color="gray.500"
|
||||
variant="ghost"
|
||||
cursor="pointer"
|
||||
onClick={refetchOutput}
|
||||
aria-label="refetch output"
|
||||
>
|
||||
<Icon as={refetchingOutput ? Spinner : BsArrowClockwise} boxSize={4} />
|
||||
</Button>
|
||||
</Tooltip>
|
||||
)}
|
||||
</HStack>
|
||||
);
|
||||
};
|
||||
@@ -1,17 +1,17 @@
|
||||
import { api } from "~/utils/api";
|
||||
import { type PromptVariant, type Scenario } from "../types";
|
||||
import { Text, VStack } from "@chakra-ui/react";
|
||||
import { type StackProps, Text, VStack } from "@chakra-ui/react";
|
||||
import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
|
||||
import SyntaxHighlighter from "react-syntax-highlighter";
|
||||
import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
|
||||
import stringify from "json-stringify-pretty-compact";
|
||||
import { type ReactElement, useState, useEffect, Fragment } from "react";
|
||||
import { type ReactElement, useState, useEffect, Fragment, useCallback } from "react";
|
||||
import useSocket from "~/utils/useSocket";
|
||||
import { OutputStats } from "./OutputStats";
|
||||
import { RetryCountdown } from "./RetryCountdown";
|
||||
import frontendModelProviders from "~/modelProviders/frontendModelProviders";
|
||||
import { ResponseLog } from "./ResponseLog";
|
||||
import { CellContent } from "./CellContent";
|
||||
import { CellOptions } from "./TopActions";
|
||||
|
||||
const WAITING_MESSAGE_INTERVAL = 20000;
|
||||
|
||||
@@ -72,37 +72,49 @@ export default function OutputCell({
|
||||
// TODO: disconnect from socket if we're not streaming anymore
|
||||
const streamedMessage = useSocket<OutputSchema>(cell?.id);
|
||||
|
||||
const mostRecentResponse = cell?.modelResponses[cell.modelResponses.length - 1];
|
||||
|
||||
const CellWrapper = useCallback(
|
||||
({ children, ...props }: StackProps) => (
|
||||
<VStack w="full" alignItems="flex-start" {...props} px={2} py={2} h="100%">
|
||||
{cell && (
|
||||
<CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} cell={cell} />
|
||||
)}
|
||||
<VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto" flex={1}>
|
||||
{children}
|
||||
</VStack>
|
||||
{mostRecentResponse && (
|
||||
<OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
|
||||
)}
|
||||
</VStack>
|
||||
),
|
||||
[hardRefetching, hardRefetch, mostRecentResponse, scenario],
|
||||
);
|
||||
|
||||
if (!vars) return null;
|
||||
|
||||
if (!cell && !fetchingOutput)
|
||||
return (
|
||||
<CellContent hardRefetching={hardRefetching} hardRefetch={hardRefetch}>
|
||||
<CellWrapper>
|
||||
<Text color="gray.500">Error retrieving output</Text>
|
||||
</CellContent>
|
||||
</CellWrapper>
|
||||
);
|
||||
|
||||
if (cell && cell.errorMessage) {
|
||||
return (
|
||||
<CellContent hardRefetching={hardRefetching} hardRefetch={hardRefetch}>
|
||||
<CellWrapper>
|
||||
<Text color="red.500">{cell.errorMessage}</Text>
|
||||
</CellContent>
|
||||
</CellWrapper>
|
||||
);
|
||||
}
|
||||
|
||||
if (disabledReason) return <Text color="gray.500">{disabledReason}</Text>;
|
||||
|
||||
const mostRecentResponse = cell?.modelResponses[cell.modelResponses.length - 1];
|
||||
const showLogs = !streamedMessage && !mostRecentResponse?.output;
|
||||
|
||||
if (showLogs)
|
||||
return (
|
||||
<CellContent
|
||||
hardRefetching={hardRefetching}
|
||||
hardRefetch={hardRefetch}
|
||||
alignItems="flex-start"
|
||||
fontFamily="inconsolata, monospace"
|
||||
spacing={0}
|
||||
>
|
||||
<CellWrapper alignItems="flex-start" fontFamily="inconsolata, monospace" spacing={0}>
|
||||
{cell?.jobQueuedAt && <ResponseLog time={cell.jobQueuedAt} title="Job queued" />}
|
||||
{cell?.jobStartedAt && <ResponseLog time={cell.jobStartedAt} title="Job started" />}
|
||||
{cell?.modelResponses?.map((response) => {
|
||||
@@ -124,9 +136,13 @@ export default function OutputCell({
|
||||
Array.from({ length: numWaitingMessages }, (_, i) => (
|
||||
<ResponseLog
|
||||
key={`waiting-${i}`}
|
||||
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
||||
time={new Date(response.requestedAt!.getTime() + i * WAITING_MESSAGE_INTERVAL)}
|
||||
title="Waiting for response"
|
||||
time={
|
||||
new Date(
|
||||
(response.requestedAt?.getTime?.() ?? 0) +
|
||||
(i + 1) * WAITING_MESSAGE_INTERVAL,
|
||||
)
|
||||
}
|
||||
title="Waiting for response..."
|
||||
/>
|
||||
))}
|
||||
{response.receivedAt && (
|
||||
@@ -144,7 +160,7 @@ export default function OutputCell({
|
||||
{mostRecentResponse?.retryTime && (
|
||||
<RetryCountdown retryTime={mostRecentResponse.retryTime} />
|
||||
)}
|
||||
</CellContent>
|
||||
</CellWrapper>
|
||||
);
|
||||
|
||||
const normalizedOutput = mostRecentResponse?.output
|
||||
@@ -155,50 +171,27 @@ export default function OutputCell({
|
||||
|
||||
if (mostRecentResponse?.output && normalizedOutput?.type === "json") {
|
||||
return (
|
||||
<VStack
|
||||
w="100%"
|
||||
h="100%"
|
||||
fontSize="xs"
|
||||
flexWrap="wrap"
|
||||
overflowX="hidden"
|
||||
justifyContent="space-between"
|
||||
>
|
||||
<CellContent
|
||||
hardRefetching={hardRefetching}
|
||||
hardRefetch={hardRefetch}
|
||||
w="full"
|
||||
flex={1}
|
||||
spacing={0}
|
||||
<CellWrapper>
|
||||
<SyntaxHighlighter
|
||||
customStyle={{ overflowX: "unset", width: "100%", flex: 1 }}
|
||||
language="json"
|
||||
style={docco}
|
||||
lineProps={{
|
||||
style: { wordBreak: "break-all", whiteSpace: "pre-wrap" },
|
||||
}}
|
||||
wrapLines
|
||||
>
|
||||
<SyntaxHighlighter
|
||||
customStyle={{ overflowX: "unset", width: "100%", flex: 1 }}
|
||||
language="json"
|
||||
style={docco}
|
||||
lineProps={{
|
||||
style: { wordBreak: "break-all", whiteSpace: "pre-wrap" },
|
||||
}}
|
||||
wrapLines
|
||||
>
|
||||
{stringify(normalizedOutput.value, { maxLength: 40 })}
|
||||
</SyntaxHighlighter>
|
||||
</CellContent>
|
||||
<OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
|
||||
</VStack>
|
||||
{stringify(normalizedOutput.value, { maxLength: 40 })}
|
||||
</SyntaxHighlighter>
|
||||
</CellWrapper>
|
||||
);
|
||||
}
|
||||
|
||||
const contentToDisplay = (normalizedOutput?.type === "text" && normalizedOutput.value) || "";
|
||||
|
||||
return (
|
||||
<VStack w="100%" h="100%" justifyContent="space-between" whiteSpace="pre-wrap">
|
||||
<VStack w="full" alignItems="flex-start" spacing={0}>
|
||||
<CellContent hardRefetching={hardRefetching} hardRefetch={hardRefetch}>
|
||||
<Text>{contentToDisplay}</Text>
|
||||
</CellContent>
|
||||
</VStack>
|
||||
{mostRecentResponse?.output && (
|
||||
<OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
|
||||
)}
|
||||
</VStack>
|
||||
<CellWrapper>
|
||||
<Text>{contentToDisplay}</Text>
|
||||
</CellWrapper>
|
||||
);
|
||||
}
|
||||
|
||||
36
src/components/OutputsTable/OutputCell/PromptModal.tsx
Normal file
36
src/components/OutputsTable/OutputCell/PromptModal.tsx
Normal file
@@ -0,0 +1,36 @@
|
||||
import {
|
||||
Modal,
|
||||
ModalBody,
|
||||
ModalCloseButton,
|
||||
ModalContent,
|
||||
ModalHeader,
|
||||
ModalOverlay,
|
||||
type UseDisclosureReturn,
|
||||
} from "@chakra-ui/react";
|
||||
import { type RouterOutputs } from "~/utils/api";
|
||||
import { JSONTree } from "react-json-tree";
|
||||
|
||||
export default function ExpandedModal(props: {
|
||||
cell: NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>;
|
||||
disclosure: UseDisclosureReturn;
|
||||
}) {
|
||||
return (
|
||||
<Modal isOpen={props.disclosure.isOpen} onClose={props.disclosure.onClose} size="2xl">
|
||||
<ModalOverlay />
|
||||
<ModalContent>
|
||||
<ModalHeader>Prompt</ModalHeader>
|
||||
<ModalCloseButton />
|
||||
<ModalBody>
|
||||
<JSONTree
|
||||
data={props.cell.prompt}
|
||||
invertTheme={true}
|
||||
theme="chalk"
|
||||
shouldExpandNodeInitially={() => true}
|
||||
getItemString={() => ""}
|
||||
hideRoot
|
||||
/>
|
||||
</ModalBody>
|
||||
</ModalContent>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
53
src/components/OutputsTable/OutputCell/TopActions.tsx
Normal file
53
src/components/OutputsTable/OutputCell/TopActions.tsx
Normal file
@@ -0,0 +1,53 @@
|
||||
import { HStack, Icon, IconButton, Spinner, Tooltip, useDisclosure } from "@chakra-ui/react";
|
||||
import { BsArrowClockwise, BsInfoCircle } from "react-icons/bs";
|
||||
import { useExperimentAccess } from "~/utils/hooks";
|
||||
import ExpandedModal from "./PromptModal";
|
||||
import { type RouterOutputs } from "~/utils/api";
|
||||
|
||||
export const CellOptions = ({
|
||||
cell,
|
||||
refetchingOutput,
|
||||
refetchOutput,
|
||||
}: {
|
||||
cell: RouterOutputs["scenarioVariantCells"]["get"];
|
||||
refetchingOutput: boolean;
|
||||
refetchOutput: () => void;
|
||||
}) => {
|
||||
const { canModify } = useExperimentAccess();
|
||||
|
||||
const modalDisclosure = useDisclosure();
|
||||
|
||||
return (
|
||||
<HStack justifyContent="flex-end" w="full">
|
||||
{cell && (
|
||||
<>
|
||||
<Tooltip label="See Prompt">
|
||||
<IconButton
|
||||
aria-label="See Prompt"
|
||||
icon={<Icon as={BsInfoCircle} boxSize={4} />}
|
||||
onClick={modalDisclosure.onOpen}
|
||||
size="xs"
|
||||
colorScheme="gray"
|
||||
color="gray.500"
|
||||
variant="ghost"
|
||||
/>
|
||||
</Tooltip>
|
||||
<ExpandedModal cell={cell} disclosure={modalDisclosure} />
|
||||
</>
|
||||
)}
|
||||
{canModify && (
|
||||
<Tooltip label="Refetch output">
|
||||
<IconButton
|
||||
size="xs"
|
||||
color="gray.500"
|
||||
variant="ghost"
|
||||
cursor="pointer"
|
||||
onClick={refetchOutput}
|
||||
aria-label="refetch output"
|
||||
icon={<Icon as={refetchingOutput ? Spinner : BsArrowClockwise} boxSize={4} />}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
</HStack>
|
||||
);
|
||||
};
|
||||
@@ -1,9 +1,8 @@
|
||||
import { useEffect, type DragEvent } from "react";
|
||||
import { api } from "~/utils/api";
|
||||
import { isEqual } from "lodash-es";
|
||||
import { type Scenario } from "./types";
|
||||
import { useEffect, useState, type DragEvent } from "react";
|
||||
import { api } from "~/utils/api";
|
||||
import { useExperiment, useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
|
||||
import { useState } from "react";
|
||||
import { type Scenario } from "./types";
|
||||
|
||||
import {
|
||||
Box,
|
||||
@@ -12,14 +11,12 @@ import {
|
||||
Icon,
|
||||
IconButton,
|
||||
Spinner,
|
||||
Stack,
|
||||
Text,
|
||||
Tooltip,
|
||||
VStack,
|
||||
Text,
|
||||
} from "@chakra-ui/react";
|
||||
import { cellPadding } from "../constants";
|
||||
import { BsArrowsAngleExpand, BsX } from "react-icons/bs";
|
||||
import { RiDraggable } from "react-icons/ri";
|
||||
import { cellPadding } from "../constants";
|
||||
import { FloatingLabelInput } from "./FloatingLabelInput";
|
||||
import { ScenarioEditorModal } from "./ScenarioEditorModal";
|
||||
|
||||
@@ -115,60 +112,44 @@ export default function ScenarioEditor({
|
||||
onDrop={onReorder}
|
||||
backgroundColor={isDragTarget ? "gray.100" : "transparent"}
|
||||
>
|
||||
{canModify && props.canHide && (
|
||||
<Stack
|
||||
alignSelf="flex-start"
|
||||
opacity={props.hovered ? 1 : 0}
|
||||
spacing={0}
|
||||
ml={-cellPadding.x}
|
||||
>
|
||||
<Tooltip label="Hide scenario" hasArrow>
|
||||
{/* for some reason the tooltip can't position itself properly relative to the icon without the wrapping box */}
|
||||
<Button
|
||||
variant="unstyled"
|
||||
color="gray.400"
|
||||
height="unset"
|
||||
width="unset"
|
||||
minW="unset"
|
||||
onClick={onHide}
|
||||
_hover={{
|
||||
color: "gray.800",
|
||||
cursor: "pointer",
|
||||
}}
|
||||
>
|
||||
<Icon as={hidingInProgress ? Spinner : BsX} boxSize={hidingInProgress ? 4 : 6} />
|
||||
</Button>
|
||||
</Tooltip>
|
||||
<Icon
|
||||
as={RiDraggable}
|
||||
boxSize={6}
|
||||
color="gray.400"
|
||||
_hover={{ color: "gray.800", cursor: "pointer" }}
|
||||
/>
|
||||
</Stack>
|
||||
)}
|
||||
|
||||
{variableLabels.length === 0 ? (
|
||||
<Box color="gray.500">
|
||||
{vars.data ? "No scenario variables configured" : "Loading..."}
|
||||
</Box>
|
||||
) : (
|
||||
<VStack spacing={4} flex={1} py={2}>
|
||||
<HStack justifyContent="space-between" w="100%">
|
||||
<Text color="gray.500">Scenario</Text>
|
||||
<IconButton
|
||||
className="fullscreen-toggle"
|
||||
aria-label="Maximize"
|
||||
icon={<BsArrowsAngleExpand />}
|
||||
onClick={() => setScenarioEditorModalOpen(true)}
|
||||
boxSize={6}
|
||||
borderRadius={4}
|
||||
p={1.5}
|
||||
minW={0}
|
||||
colorScheme="gray"
|
||||
color="gray.500"
|
||||
variant="ghost"
|
||||
/>
|
||||
<HStack justifyContent="space-between" w="100%" align="center" spacing={0}>
|
||||
<Text flex={1}>Scenario</Text>
|
||||
<Tooltip label="Expand" hasArrow>
|
||||
<IconButton
|
||||
aria-label="Expand"
|
||||
icon={<Icon as={BsArrowsAngleExpand} boxSize={3} />}
|
||||
onClick={() => setScenarioEditorModalOpen(true)}
|
||||
size="xs"
|
||||
colorScheme="gray"
|
||||
color="gray.500"
|
||||
variant="ghost"
|
||||
/>
|
||||
</Tooltip>
|
||||
{canModify && props.canHide && (
|
||||
<Tooltip label="Delete" hasArrow>
|
||||
<IconButton
|
||||
aria-label="Delete"
|
||||
icon={
|
||||
<Icon
|
||||
as={hidingInProgress ? Spinner : BsX}
|
||||
boxSize={hidingInProgress ? 4 : 6}
|
||||
/>
|
||||
}
|
||||
onClick={onHide}
|
||||
size="xs"
|
||||
display="flex"
|
||||
colorScheme="gray"
|
||||
color="gray.500"
|
||||
variant="ghost"
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
</HStack>
|
||||
{variableLabels.map((key) => {
|
||||
const value = values[key] ?? "";
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import {
|
||||
Button,
|
||||
HStack,
|
||||
Icon,
|
||||
Modal,
|
||||
ModalBody,
|
||||
ModalCloseButton,
|
||||
@@ -14,7 +13,6 @@ import {
|
||||
VStack,
|
||||
} from "@chakra-ui/react";
|
||||
import { useEffect, useState } from "react";
|
||||
import { BsFileTextFill } from "react-icons/bs";
|
||||
import { isEqual } from "lodash-es";
|
||||
|
||||
import { api } from "~/utils/api";
|
||||
@@ -60,8 +58,6 @@ export const ScenarioEditorModal = ({
|
||||
await utils.scenarios.list.invalidate();
|
||||
}, [mutation, values]);
|
||||
|
||||
console.log("scenario", scenario);
|
||||
|
||||
const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
|
||||
const variableLabels = vars.data?.map((v) => v.label) ?? [];
|
||||
|
||||
@@ -73,12 +69,7 @@ export const ScenarioEditorModal = ({
|
||||
>
|
||||
<ModalOverlay />
|
||||
<ModalContent w={1200}>
|
||||
<ModalHeader>
|
||||
<HStack>
|
||||
<Icon as={BsFileTextFill} />
|
||||
<Text>Scenario</Text>
|
||||
</HStack>
|
||||
</ModalHeader>
|
||||
<ModalHeader />
|
||||
<ModalCloseButton />
|
||||
<ModalBody maxW="unset">
|
||||
<VStack spacing={8}>
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { Box, GridItem } from "@chakra-ui/react";
|
||||
import { GridItem } from "@chakra-ui/react";
|
||||
import React, { useState } from "react";
|
||||
import { cellPadding } from "../constants";
|
||||
import OutputCell from "./OutputCell/OutputCell";
|
||||
import ScenarioEditor from "./ScenarioEditor";
|
||||
import type { PromptVariant, Scenario } from "./types";
|
||||
@@ -39,9 +38,7 @@ const ScenarioRow = (props: {
|
||||
colStart={i + 2}
|
||||
{...borders}
|
||||
>
|
||||
<Box h="100%" w="100%" px={cellPadding.x} py={cellPadding.y}>
|
||||
<OutputCell key={variant.id} scenario={props.scenario} variant={variant} />
|
||||
</Box>
|
||||
<OutputCell key={variant.id} scenario={props.scenario} variant={variant} />
|
||||
</GridItem>
|
||||
))}
|
||||
</>
|
||||
|
||||
@@ -35,7 +35,7 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
|
||||
pb={24}
|
||||
pl={8}
|
||||
display="grid"
|
||||
gridTemplateColumns={`250px repeat(${variants.data.length}, minmax(300px, 1fr)) auto`}
|
||||
gridTemplateColumns={`250px repeat(${variants.data.length}, minmax(360px, 1fr)) auto`}
|
||||
sx={{
|
||||
"> *": {
|
||||
borderColor: "gray.300",
|
||||
|
||||
@@ -97,7 +97,7 @@ export const RefinePromptModal = ({
|
||||
<ModalCloseButton />
|
||||
<ModalBody maxW="unset">
|
||||
<VStack spacing={8}>
|
||||
<VStack spacing={4}>
|
||||
<VStack spacing={4} w="full">
|
||||
{Object.keys(refinementActions).length && (
|
||||
<>
|
||||
<SimpleGrid columns={{ base: 1, md: 2 }} spacing={8}>
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"model": {
|
||||
"description": "The model that will complete your prompt.\nAs we improve Claude, we develop new versions of it that you can query.\nThis parameter controls which version of Claude answers your request.\nRight now we are offering two model families: Claude, and Claude Instant.\nYou can use them by setting model to \"claude-2\" or \"claude-instant-1\", respectively.\nSee models for additional details.\n",
|
||||
"description": "The model that will complete your prompt.",
|
||||
"x-oaiTypeLabel": "string",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
@@ -13,116 +13,50 @@
|
||||
]
|
||||
},
|
||||
"prompt": {
|
||||
"description": "The prompt that you want Claude to complete.\n\nFor proper response generation you will need to format your prompt as follows:\n\\n\\nHuman: ${userQuestion}\\n\\nAssistant:\nSee our comments on prompts for more context.\n",
|
||||
"description": "The prompt that you want Claude to complete.\n\nFor proper response generation you will need to format your prompt as follows:\n\"\\n\\nHuman: all instructions for the assistant\\n\\nAssistant:\". The prompt string should begin with the characters \"Human:\" and end with \"Assistant:\".",
|
||||
"default": "<|endoftext|>",
|
||||
"nullable": true,
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"example": "This is a test."
|
||||
},
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"example": "This is a test."
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"example": "[1212, 318, 257, 1332, 13]"
|
||||
},
|
||||
{
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"example": "[[1212, 318, 257, 1332, 13]]"
|
||||
}
|
||||
]
|
||||
"example": "\\n\\nHuman: What is the correct translation of ${scenario.input}? I would like a long analysis followed by a short answer.\\n\\nAssistant:",
|
||||
"type": "string"
|
||||
},
|
||||
"max_tokens_to_sample": {
|
||||
"type": "integer",
|
||||
"minimum": 1,
|
||||
"default": 256,
|
||||
"example": 256,
|
||||
"nullable": true,
|
||||
"description": "The maximum number of tokens to generate before stopping.\n\nNote that our models may stop before reaching this maximum. This parameter only specifies the absolute maximum number of tokens to generate.\n"
|
||||
"description": "The maximum number of tokens to generate before stopping."
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1,
|
||||
"default": 1,
|
||||
"example": 1,
|
||||
"nullable": true,
|
||||
"description": "Amount of randomness injected into the response.\n\nDefaults to 1. Ranges from 0 to 1. Use temp closer to 0 for analytical / multiple choice, and closer to 1 for creative and generative tasks.\n"
|
||||
"description": "Amount of randomness injected into the response.\n\nDefaults to 1."
|
||||
},
|
||||
"top_p": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1,
|
||||
"default": 1,
|
||||
"example": 1,
|
||||
"nullable": true,
|
||||
"description": "Use nucleus sampling.\n\nIn nucleus sampling, we compute the cumulative distribution over all the options \nfor each subsequent token in decreasing probability order and cut it off once \nit reaches a particular probability specified by top_p. You should either alter temperature or top_p, but not both.\n"
|
||||
"description": "Use nucleus sampling.\n\nYou should either alter temperature or top_p, but not both.\n"
|
||||
},
|
||||
"top_k": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"default": 5,
|
||||
"example": 5,
|
||||
"nullable": true,
|
||||
"description": "Only sample from the top K options for each subsequent token.\n\nUsed to remove \"long tail\" low probability responses. Learn more technical details here.\n"
|
||||
"description": "Only sample from the top K options for each subsequent token."
|
||||
},
|
||||
"stream": {
|
||||
"description": "Whether to incrementally stream the response using server-sent events.\nSee this guide to SSE events for details.type: boolean\n",
|
||||
"description": "Whether to incrementally stream the response using server-sent events.",
|
||||
"type": "boolean",
|
||||
"nullable": true,
|
||||
"default": false
|
||||
},
|
||||
"stop_sequences": {
|
||||
"description": "Sequences that will cause the model to stop generating completion text.\nOur models stop on \"\\n\\nHuman:\", and may include additional built-in stop sequences in the future. By providing the stop_sequences parameter, you may include additional strings that will cause the model to stop generating.\n",
|
||||
"description": "Sequences that will cause the model to stop generating completion text.\nBy default, our models stop on \"\\n\\nHuman:\".",
|
||||
"default": null,
|
||||
"nullable": true,
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"default": "<|endoftext|>",
|
||||
"example": "\n",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"maxItems": 4,
|
||||
"items": {
|
||||
"type": "string",
|
||||
"example": "[\"\\n\"]"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_id": {
|
||||
"type": "string",
|
||||
"example": "13803d75-b4b5-4c3e-b2a2-6f21399b021b",
|
||||
"description": "An external identifier for the user who is associated with the request.\n\nThis should be a uuid, hash value, or other opaque identifier. Anthropic may use this id to help detect abuse. \nDo not include any identifying information such as name, email address, or phone number.\n"
|
||||
}
|
||||
},
|
||||
"description": "An object describing metadata about the request.\n"
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": ["model", "prompt", "max_tokens_to_sample"]
|
||||
|
||||
@@ -15,6 +15,7 @@ const frontendModelProvider: FrontendModelProvider<SupportedModel, Completion> =
|
||||
speed: "medium",
|
||||
provider: "anthropic",
|
||||
learnMoreUrl: "https://www.anthropic.com/product",
|
||||
apiDocsUrl: "https://docs.anthropic.com/claude/reference/complete_post",
|
||||
},
|
||||
"claude-instant-1.1": {
|
||||
name: "Claude Instant 1.1",
|
||||
@@ -24,6 +25,7 @@ const frontendModelProvider: FrontendModelProvider<SupportedModel, Completion> =
|
||||
speed: "fast",
|
||||
provider: "anthropic",
|
||||
learnMoreUrl: "https://www.anthropic.com/product",
|
||||
apiDocsUrl: "https://docs.anthropic.com/claude/reference/complete_post",
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
} from "openai/resources/chat";
|
||||
import { countOpenAIChatTokens } from "~/utils/countTokens";
|
||||
import { type CompletionResponse } from "../types";
|
||||
import { omit } from "lodash-es";
|
||||
import { isArray, isString, omit } from "lodash-es";
|
||||
import { openai } from "~/server/utils/openai";
|
||||
import { truthyFilter } from "~/utils/utils";
|
||||
import { APIError } from "openai";
|
||||
@@ -40,6 +40,8 @@ const mergeStreamedChunks = (
|
||||
((choice.delta.function_call.arguments as string) ?? "");
|
||||
}
|
||||
} else {
|
||||
// @ts-expect-error the types are correctly telling us that finish_reason
|
||||
// could be null, but don't want to fix it right now.
|
||||
choices.push({ ...omit(choice, "delta"), message: { role: "assistant", ...choice.delta } });
|
||||
}
|
||||
}
|
||||
@@ -64,6 +66,7 @@ export async function getCompletion(
|
||||
|
||||
try {
|
||||
if (onStream) {
|
||||
console.log("got started");
|
||||
const resp = await openai.chat.completions.create(
|
||||
{ ...input, stream: true },
|
||||
{
|
||||
@@ -71,9 +74,11 @@ export async function getCompletion(
|
||||
},
|
||||
);
|
||||
for await (const part of resp) {
|
||||
console.log("got part", part);
|
||||
finalCompletion = mergeStreamedChunks(finalCompletion, part);
|
||||
onStream(finalCompletion);
|
||||
}
|
||||
console.log("got final", finalCompletion);
|
||||
if (!finalCompletion) {
|
||||
return {
|
||||
type: "error",
|
||||
@@ -121,9 +126,17 @@ export async function getCompletion(
|
||||
};
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof APIError) {
|
||||
// The types from the sdk are wrong
|
||||
const rawMessage = error.message as string | string[];
|
||||
// If the message is not a string, stringify it
|
||||
const message = isString(rawMessage)
|
||||
? rawMessage
|
||||
: isArray(rawMessage)
|
||||
? rawMessage.map((m) => m.toString()).join("\n")
|
||||
: (rawMessage as any).toString();
|
||||
return {
|
||||
type: "error",
|
||||
message: error.message,
|
||||
message,
|
||||
autoRetry: error.status === 429 || error.status === 503,
|
||||
statusCode: error.status,
|
||||
};
|
||||
|
||||
@@ -54,7 +54,7 @@ const modelProvider: ReplicateLlama2Provider = {
|
||||
temperature: {
|
||||
type: "number",
|
||||
description:
|
||||
"Adjusts randomness of outputs, greater than 1 is random and 0 is deterministic, 0.75 is a good starting value. (minimum: 0.01; maximum: 5)",
|
||||
"Adjusts randomness of outputs, 0.1 is a good starting value. (minimum: 0.01; maximum: 5)",
|
||||
},
|
||||
top_p: {
|
||||
type: "number",
|
||||
|
||||
@@ -21,6 +21,7 @@ export type Model = {
|
||||
provider: SupportedProvider;
|
||||
description?: string;
|
||||
learnMoreUrl?: string;
|
||||
apiDocsUrl?: string;
|
||||
};
|
||||
|
||||
export type ProviderModel = { provider: z.infer<typeof ZodSupportedProvider>; model: string };
|
||||
|
||||
@@ -9,7 +9,7 @@ const inconsolataRegularFontP = fetch(
|
||||
new URL("../../../../public/fonts/Inconsolata_SemiExpanded-Medium.ttf", import.meta.url),
|
||||
).then((res) => res.arrayBuffer());
|
||||
|
||||
const OgImage = async (req: NextApiRequest, res: NextApiResponse) => {
|
||||
const OgImage = async (req: NextApiRequest, _res: NextApiResponse) => {
|
||||
// @ts-expect-error - nextUrl is not defined on NextApiRequest for some reason
|
||||
const searchParams = req.nextUrl?.searchParams as URLSearchParams;
|
||||
const experimentLabel = searchParams.get("experimentLabel");
|
||||
|
||||
@@ -27,24 +27,45 @@ import UserMenu from "~/components/nav/UserMenu";
|
||||
import { api } from "~/utils/api";
|
||||
import dayjs from "~/utils/dayjs";
|
||||
import { useHandledAsyncCallback } from "~/utils/hooks";
|
||||
import GitHubButton from "react-github-btn";
|
||||
|
||||
const TopNavbar = () => (
|
||||
<DarkMode>
|
||||
<GlobalStyle />
|
||||
<HStack px={4} py={2}>
|
||||
<HStack as={Link} href="/" _hover={{ textDecoration: "none" }} spacing={0} py={2} pr={16}>
|
||||
<Image src="/logo.svg" alt="" boxSize={6} mr={4} />
|
||||
<Heading size="md" fontFamily="inconsolata, monospace">
|
||||
OpenPipe
|
||||
</Heading>
|
||||
</HStack>
|
||||
<HStack px={4} py={2} align="center" justify="center">
|
||||
<HStack
|
||||
as={Link}
|
||||
href="/"
|
||||
_hover={{ textDecoration: "none" }}
|
||||
spacing={0}
|
||||
py={2}
|
||||
pr={16}
|
||||
flex={1}
|
||||
sx={{
|
||||
".widget": {
|
||||
display: "block",
|
||||
},
|
||||
}}
|
||||
>
|
||||
<Image src="/logo.svg" alt="" boxSize={6} mr={4} />
|
||||
<Heading size="md" fontFamily="inconsolata, monospace">
|
||||
OpenPipe
|
||||
</Heading>
|
||||
</HStack>
|
||||
</DarkMode>
|
||||
<Box pt="6px">
|
||||
<GitHubButton
|
||||
href="https://github.com/openpipe/openpipe"
|
||||
data-color-scheme="no-preference: dark; light: dark; dark: dark;"
|
||||
data-size="large"
|
||||
aria-label="Follow @openpipe on GitHub"
|
||||
>
|
||||
Github
|
||||
</GitHubButton>
|
||||
</Box>
|
||||
</HStack>
|
||||
);
|
||||
|
||||
// Shows how long until the competition starts. Refreshes every second
|
||||
function CountdownTimer(props: { date: Date } & TextProps) {
|
||||
const [now, setNow] = useState(dayjs(0));
|
||||
const [now, setNow] = useState(dayjs());
|
||||
|
||||
useInterval(() => {
|
||||
setNow(dayjs());
|
||||
@@ -52,7 +73,7 @@ function CountdownTimer(props: { date: Date } & TextProps) {
|
||||
|
||||
const { date, ...rest } = props;
|
||||
|
||||
const kickoff = dayjs(props.date);
|
||||
const kickoff = dayjs(date);
|
||||
const diff = kickoff.diff(now, "second");
|
||||
const days = Math.floor(diff / 86400);
|
||||
const hours = Math.floor((diff % 86400) / 3600);
|
||||
@@ -60,7 +81,7 @@ function CountdownTimer(props: { date: Date } & TextProps) {
|
||||
const seconds = Math.floor(diff % 60);
|
||||
|
||||
return (
|
||||
<Text {...rest}>
|
||||
<Text {...rest} suppressHydrationWarning>
|
||||
<Text as="span" fontWeight="bold">
|
||||
Kickoff in
|
||||
</Text>{" "}
|
||||
@@ -112,7 +133,11 @@ function ApplicationStatus(props: BoxProps) {
|
||||
</Text>
|
||||
) : entrant ? (
|
||||
<Text fontSize="sm">
|
||||
Application submitted successfully! We'll notify you by email before August 14th.
|
||||
✅ Application submitted successfully. We'll notify you by email before August 14th.{" "}
|
||||
<Link href="https://github.com/openpipe/openpipe" isExternal textDecor="underline">
|
||||
Star our Github
|
||||
</Link>{" "}
|
||||
for updates while you wait!
|
||||
</Text>
|
||||
) : (
|
||||
<Button onClick={onApply} colorScheme="orange">
|
||||
@@ -143,10 +168,12 @@ export default function Signup() {
|
||||
/>
|
||||
</Head>
|
||||
|
||||
<Box bgColor="gray.900" color="gray.200" minH="100vh" w="full">
|
||||
<Box color="gray.200" minH="100vh" w="full">
|
||||
<TopNavbar />
|
||||
<VStack mx="auto" py={24} maxW="2xl" align="start" fontSize="lg">
|
||||
<Heading size="lg">🏆 Prompt Engineering World Championships</Heading>
|
||||
<VStack mx="auto" py={24} maxW="2xl" px={4} align="center" fontSize="lg">
|
||||
<Heading size="lg" textAlign="center">
|
||||
🏆 Prompt Engineering World Championships
|
||||
</Heading>
|
||||
<CountdownTimer
|
||||
date={new Date("2023-08-14T00:00:00Z")}
|
||||
fontSize="2xl"
|
||||
@@ -156,7 +183,7 @@ export default function Signup() {
|
||||
|
||||
<ApplicationStatus py={8} alignSelf="center" />
|
||||
|
||||
<Text fontSize="lg">
|
||||
<Text fontSize="lg" textAlign="left">
|
||||
Think you have what it takes to be the best? Compete with the world's top prompt
|
||||
engineers and see where you rank!
|
||||
</Text>
|
||||
|
||||
@@ -3,7 +3,7 @@ import { prisma } from "~/server/db";
|
||||
import { requireNothing } from "~/utils/accessControl";
|
||||
|
||||
export const worldChampsRouter = createTRPCRouter({
|
||||
userStatus: publicProcedure.query(async ({ input, ctx }) => {
|
||||
userStatus: publicProcedure.query(async ({ ctx }) => {
|
||||
const userId = ctx.session?.user.id;
|
||||
|
||||
if (!userId) {
|
||||
|
||||
@@ -3,7 +3,7 @@ import ivm from "isolated-vm";
|
||||
import dedent from "dedent";
|
||||
import { openai } from "./openai";
|
||||
import { isObject } from "lodash-es";
|
||||
import { type CompletionCreateParams } from "openai/resources/chat/completions";
|
||||
import type { CreateChatCompletionRequestMessage } from "openai/resources/chat/completions";
|
||||
import formatPromptConstructor from "~/utils/formatPromptConstructor";
|
||||
import { type SupportedProvider, type Model } from "~/modelProviders/types";
|
||||
import modelProviders from "~/modelProviders/modelProviders";
|
||||
@@ -44,7 +44,7 @@ const requestUpdatedPromptFunction = async (
|
||||
let newContructionFn = "";
|
||||
for (let i = 0; i < NUM_RETRIES; i++) {
|
||||
try {
|
||||
const messages: CompletionCreateParams.CreateChatCompletionRequestNonStreaming.Message[] = [
|
||||
const messages: CreateChatCompletionRequestMessage[] = [
|
||||
{
|
||||
role: "system",
|
||||
content: `Your job is to update prompt constructor functions. Here is the api shape for the current model:\n---\n${JSON.stringify(
|
||||
@@ -66,9 +66,11 @@ const requestUpdatedPromptFunction = async (
|
||||
if (newModel.provider !== originalModel.provider) {
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: `The old provider was ${originalModel.provider}. The new provider is ${
|
||||
content: `As seen in the first argument to definePrompt, the old provider endpoint was "${
|
||||
originalModel.provider
|
||||
}". The new provider endpoint is "${
|
||||
newModel.provider
|
||||
}. Here is the schema for the new model:\n---\n${JSON.stringify(
|
||||
}". Here is the schema for the new model:\n---\n${JSON.stringify(
|
||||
modelProviders[newModel.provider].inputSchema,
|
||||
null,
|
||||
2,
|
||||
|
||||
@@ -1,11 +1,31 @@
|
||||
import { extendTheme } from "@chakra-ui/react";
|
||||
import "@fontsource/inconsolata";
|
||||
import { ChakraProvider } from "@chakra-ui/react";
|
||||
import { modalAnatomy } from "@chakra-ui/anatomy";
|
||||
import { createMultiStyleConfigHelpers } from "@chakra-ui/styled-system";
|
||||
|
||||
const systemFont =
|
||||
'ui-sans-serif, -apple-system, "system-ui", "Segoe UI", Helvetica, "Apple Color Emoji", Arial, sans-serif, "Segoe UI Emoji", "Segoe UI Symbol"';
|
||||
|
||||
/* eslint-disable @typescript-eslint/unbound-method */
|
||||
const { definePartsStyle, defineMultiStyleConfig } = createMultiStyleConfigHelpers(
|
||||
modalAnatomy.keys,
|
||||
);
|
||||
|
||||
const modalTheme = defineMultiStyleConfig({
|
||||
baseStyle: definePartsStyle({
|
||||
dialog: { borderRadius: "sm" },
|
||||
}),
|
||||
});
|
||||
|
||||
const theme = extendTheme({
|
||||
styles: {
|
||||
global: (props: { colorMode: "dark" | "light" }) => ({
|
||||
"html, body": {
|
||||
backgroundColor: props.colorMode === "dark" ? "gray.900" : "white",
|
||||
},
|
||||
}),
|
||||
},
|
||||
fonts: {
|
||||
heading: systemFont,
|
||||
body: systemFont,
|
||||
@@ -32,6 +52,7 @@ const theme = extendTheme({
|
||||
},
|
||||
},
|
||||
},
|
||||
Modal: modalTheme,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user