From dbbf5f2f72565190bcaadaafc2ae8a61b71a5386 Mon Sep 17 00:00:00 2001
From: Ian Webster <ianw_github@ianww.com>
Date: Fri, 16 Jun 2023 19:02:59 -0700
Subject: [PATCH] Test github action

---
 .github/workflows/prompt-eval.yml           | 61 +++++++++++++++++++++
 examples/github-action/prompt1.json         |  4 ++
 examples/github-action/prompt2.json         |  4 ++
 examples/github-action/promptfooconfig.yaml | 10 ++++
 4 files changed, 79 insertions(+)
 create mode 100644 .github/workflows/prompt-eval.yml
 create mode 100644 examples/github-action/prompt1.json
 create mode 100644 examples/github-action/prompt2.json
 create mode 100644 examples/github-action/promptfooconfig.yaml

diff --git a/.github/workflows/prompt-eval.yml b/.github/workflows/prompt-eval.yml
new file mode 100644
index 0000000..f2cf142
--- /dev/null
+++ b/.github/workflows/prompt-eval.yml
@@ -0,0 +1,61 @@
+name: Prompt Evaluation
+
+on:
+  pull_request:
+    paths:
+      - 'examples/github_action/prompt1.json'
+
+jobs:
+  evaluate:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout base ref (original)
+      uses: actions/checkout@v2
+      with:
+        ref: ${{ github.base_ref }}
+        path: base
+
+    - name: Checkout head ref (modified)
+      uses: actions/checkout@v2
+      with:
+        ref: ${{ github.head_ref }}
+        path: head
+
+    - name: Cache promptfoo data
+      id: cache
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/promptfoo
+        key: ${{ runner.os }}-promptfoo-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-promptfoo-
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v2
+      with:
+        node-version: 14
+
+    - name: Install dependencies
+      run: npm install promptfoo@0.13.0
+
+    - name: Run promptfoo evaluation
+      id: eval
+      env:
+        OPENAI_API_TOKEN: ${{ secrets.OPENAI_API_TOKEN }}
+        PROMPTFOO_CACHE_PATH: ~/.cache/promptfoo
+      run: |
+        echo "::set-output name=result::$(npx promptfoo eval --prompts base/examples/github_action/prompt1.json head/examples/github_action/prompt1.json)"
+
+    - name: Comment PR
+      uses: actions/github-script@v0.9.0
+      with:
+        github-token: ${{secrets.GITHUB_TOKEN}}
+        script: |
+          const output = `${{steps.eval.outputs.result}}`;
+          github.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: output
+          });
diff --git a/examples/github-action/prompt1.json b/examples/github-action/prompt1.json
new file mode 100644
index 0000000..3d63641
--- /dev/null
+++ b/examples/github-action/prompt1.json
@@ -0,0 +1,4 @@
+[
+  { "role": "system", "content": "You are a helpful assistant." },
+  { "role": "user", "content": "Who won the world series in {{ year }}?" }
+]
diff --git a/examples/github-action/prompt2.json b/examples/github-action/prompt2.json
new file mode 100644
index 0000000..6527687
--- /dev/null
+++ b/examples/github-action/prompt2.json
@@ -0,0 +1,4 @@
+[
+  { "role": "system", "content": "You are a sports historian." },
+  { "role": "user", "content": "This team won the world series in {{ year }}:" }
+]
diff --git a/examples/github-action/promptfooconfig.yaml b/examples/github-action/promptfooconfig.yaml
new file mode 100644
index 0000000..ca8d8d2
--- /dev/null
+++ b/examples/github-action/promptfooconfig.yaml
@@ -0,0 +1,10 @@
+providers: [openai:gpt-3.5-turbo]
+tests:
+  - vars:
+      year: 1950
+  - vars:
+      year: 1984
+  - vars:
+      year: 2020
+  - vars:
+      year: 2023