mirror of
https://github.com/growthbook/growthbook.git
synced 2021-08-07 14:23:53 +03:00
Python stats (#20)
* Use Python for stats back-end instead of jStat * Change count metrics to use a Gaussian prior instead of Gamma * New more robust method for calculating credible intervals * Expose Bayesian Risk in front-end * Use Violin plots instead of bar chart to show credible intervals * Simplify back-end build process * Add typescript definitions for jStat (just the methods used) * Better sample data and onboarding experience
This commit is contained in:
Jeremy Dorn
8
.gitignore
vendored
8
.gitignore
vendored
@@ -10,4 +10,10 @@ out
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
uploads
|
||||
uploads
|
||||
packages/back-end/globalConfig.json
|
||||
|
||||
# Python compiled files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
FROM node:14-alpine
|
||||
|
||||
# Install python for stats models
|
||||
RUN apk add --no-cache \
|
||||
python3 \
|
||||
py3-numpy \
|
||||
py3-scipy
|
||||
|
||||
WORKDIR /usr/local/src/app
|
||||
|
||||
# Copy only the required files
|
||||
COPY . /usr/local/src/app
|
||||
|
||||
RUN \
|
||||
# Install with dev dependencies
|
||||
# Install app with dev dependencies
|
||||
yarn install --frozen-lockfile --ignore-optional \
|
||||
# Build the app
|
||||
&& yarn build \
|
||||
|
||||
62
README.md
62
README.md
@@ -4,48 +4,58 @@
|
||||
<a href="https://github.com/growthbook/growthbook/actions/workflows/ci.yml"><img src="https://img.shields.io/github/workflow/status/growthbook/growthbook/CI" alt="Build Status" height="22"/></a>
|
||||
<a href="https://github.com/growthbook/growthbook/blob/main/LICENSE"><img src="https://img.shields.io/github/license/growthbook/growthbook" alt="MIT License" height="22"/></a>
|
||||
<a href="https://github.com/growthbook/growthbook/releases"><img src="https://img.shields.io/github/v/release/growthbook/growthbook?color=blue&sort=semver" alt="Release" height="22"/></a>
|
||||
<a href="https://join.slack.com/t/growthbookusers/shared_invite/zt-oiq9s1qd-dHHvw4xjpnoRV1QQrq6vUg"><img src="https://img.shields.io/badge/slack-join-E01E5A" alt="Join us on Slack" height="22"/></a>
|
||||
<a href="https://join.slack.com/t/growthbookusers/shared_invite/zt-oiq9s1qd-dHHvw4xjpnoRV1QQrq6vUg"><img src="https://img.shields.io/badge/slack-join-E01E5A?logo=slack" alt="Join us on Slack" height="22"/></a>
|
||||
</p>
|
||||
|
||||
|
||||
Get up and running in 1 minute with:
|
||||
|
||||
```sh
|
||||
git clone https://github.com/growthbook/growthbook.git && cd growthbook
|
||||
docker-compose up
|
||||
# Then visit http://localhost:3000
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Major Features
|
||||
|
||||
- Client libraries for [React](https://github.com/growthbook/growthbook-react), [Javascript](https://github.com/growthbook/growthbook-js), [PHP](https://github.com/growthbook/growthbook-php), [Ruby](https://github.com/growthbook/growthbook-ruby), and [Python](https://github.com/growthbook/growthbook-python) with more coming soon
|
||||
- [Visual Editor](https://docs.growthbook.io/app/visual) for non-technical users to create experiments _(beta)_
|
||||
- Query multiple data sources (Snowflake, Redshift, BigQuery, ClickHouse, Mixpanel, Postgres, Athena, and Google Analytics)
|
||||
- Bayesian statistics engine with support for binomial, count, duration, and revenue metrics
|
||||
- Drill down into A/B test results (e.g. by browser, country, etc.)
|
||||
- Lightweight idea board and prioritization framework
|
||||
- Document everything! (upload screenshots, add markdown comments, and more)
|
||||
- Automated email alerts when tests become significant
|
||||
- ❄️ Pull data from Snowflake, Redshift, BigQuery, ClickHouse, Mixpanel, Postgres, Athena, or Google Analytics
|
||||
- 🆎 Bayesian statistics engine with support for binomial, count, duration, and revenue metrics
|
||||
- ⬇️ Drill down into A/B test results by browser, country, or any other attribute
|
||||
- 💻 Client libraries for [React](https://github.com/growthbook/growthbook-react), [Javascript](https://github.com/growthbook/growthbook-js), [PHP](https://github.com/growthbook/growthbook-php), [Ruby](https://github.com/growthbook/growthbook-ruby), and [Python](https://github.com/growthbook/growthbook-python) with more coming soon
|
||||
- 👁️ [Visual Editor](https://docs.growthbook.io/app/visual) for non-technical users to create experiments _(beta)_
|
||||
- 📝 Document experiments with screenshots and GitHub Flavored Markdown
|
||||
- 🔔 Automated email alerts when tests become significant
|
||||
- 💡 Lightweight idea board and objective prioritization framework
|
||||
|
||||
## Requirements
|
||||
## Try Growth Book
|
||||
|
||||
- Docker (plus docker-compose for running locally)
|
||||
- MongoDB 3.2 or higher
|
||||
- A compatible data source (Snowflake, Redshift, BigQuery, ClickHouse, Mixpanel, Postgres, Athena, or Google Analytics)
|
||||
- _(optional)_ An SMTP server for emailing invites, reset password links, etc.
|
||||
- _(optional)_ Google OAuth keys (only if using Google Analytics as a data source)
|
||||
### Managed Cloud Hosting
|
||||
|
||||
We also offer a hosted cloud version that's free to get started: https://app.growthbook.io
|
||||
Create a free [Growth Book Cloud](https://app.growthbook.io) account to get started.
|
||||
|
||||
## Quick Start
|
||||
### Open Source
|
||||
|
||||
1. Clone this repo: `git clone https://github.com/growthbook/growthbook.git && cd growthbook`
|
||||
2. Start docker-compose: `docker-compose up -d`
|
||||
3. Visit http://localhost:3000
|
||||
Growth Book is built with a NextJS front-end, an ExpressJS API, and a Python stats engine. It uses MongoDB to store login credentials, cached experiment results, and meta data.
|
||||
|
||||
The front-end, back-end, and stats engine are bundled together into a single Docker image. View the image on [Docker Hub](https://hub.docker.com/r/growthbook/growthbook) for all possible environment variables and commands.
|
||||
|
||||
If you have any questions or need help, [please get in touch](mailto:hello@growthbook.io)!
|
||||
|
||||
## Documentation and Support
|
||||
|
||||
View [Docker](https://hub.docker.com/r/growthbook/growthbook) for all configuration options.
|
||||
|
||||
View the [Growth Book Docs](https://docs.growthbook.io) for info on how to setup and use the platform.
|
||||
|
||||
Join [our Slack community](https://join.slack.com/t/growthbookusers/shared_invite/zt-oiq9s1qd-dHHvw4xjpnoRV1QQrq6vUg) if you need help, want to chat, or are thinking of a new feature. We're here to help - and to make Growth Book even better.
|
||||
Join [our Slack community](https://join.slack.com/t/growthbookusers/shared_invite/zt-oiq9s1qd-dHHvw4xjpnoRV1QQrq6vUg) if you get stuck, want to chat, or are thinking of a new feature. We're here to help - and to make Growth Book even better.
|
||||
|
||||
Check out [CONTRIBUTING.md](/CONTRIBUTING.md) for instructions on setting up a local development environment.
|
||||
## Contributors
|
||||
|
||||
We ❤️ all contributions!
|
||||
|
||||
Read [CONTRIBUTING.md](/CONTRIBUTING.md) for how to setup your local development environment.
|
||||
|
||||
If you want to, you can reach out via [Slack](https://join.slack.com/t/growthbookusers/shared_invite/zt-oiq9s1qd-dHHvw4xjpnoRV1QQrq6vUg) or [email](mailto:hello@growthbook.io) and we'll set up a pair programming session to get you started.
|
||||
|
||||
## License
|
||||
|
||||
This project uses the MIT license. The core Growth Book app will always remain free, although we may add some commercial enterprise add-ons in the future.
|
||||
This project uses the MIT license. The core Growth Book app will always remain open and free, although we may add some commercial enterprise add-ons in the future.
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
{
|
||||
"mongoUri": "mongodb://127.0.0.1:33563/276ef01f-479a-4888-aa32-2c3d6800146f?"
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
module.exports = {
|
||||
globals: {
|
||||
"ts-jest": {
|
||||
tsconfig: "tsconfig.json",
|
||||
tsconfig: "./tsconfig.spec.json",
|
||||
},
|
||||
},
|
||||
moduleFileExtensions: ["ts", "js"],
|
||||
|
||||
@@ -3,9 +3,12 @@
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"copy-templates": "mkdir -p dist/templates && cp -r src/templates/* dist/templates",
|
||||
"dev": "yarn copy-templates && node-dev src/server.ts",
|
||||
"build": "rm -rf dist && tsc && rm -rf dist/types && mv dist/src/* dist/ && rm -rf dist/src && yarn copy-templates",
|
||||
"build:emails": "mkdir -p dist/templates && cp -r src/templates/* dist/templates",
|
||||
"build:python": "rm -rf dist/python && mkdir -p dist/python && cp -r src/python/* dist/python",
|
||||
"build:clean": "rm -rf dist",
|
||||
"build:typescript": "tsc",
|
||||
"dev": "node-dev src/server.ts",
|
||||
"build": "yarn build:clean && yarn build:typescript && yarn build:python && yarn build:emails",
|
||||
"start": "node dist/server.js",
|
||||
"test": "jest --forceExit --coverage --verbose --detectOpenHandles",
|
||||
"type-check": "tsc --pretty --noEmit",
|
||||
@@ -48,6 +51,7 @@
|
||||
"objects-to-csv": "^1.3.6",
|
||||
"pg": "^8.6.0",
|
||||
"pino-http": "^5.3.0",
|
||||
"python-shell": "^3.0.0",
|
||||
"snowflake-promise": "^4.5.0",
|
||||
"sql-formatter": "^4.0.2",
|
||||
"stripe": "^8.106.0",
|
||||
|
||||
@@ -17,14 +17,16 @@ import {
|
||||
processPastExperiments,
|
||||
} from "../services/experiments";
|
||||
import uniqid from "uniqid";
|
||||
import { MetricAnalysis, MetricInterface } from "../../types/metric";
|
||||
import {
|
||||
MetricAnalysis,
|
||||
MetricInterface,
|
||||
MetricStats,
|
||||
} from "../../types/metric";
|
||||
import { ExperimentModel } from "../models/ExperimentModel";
|
||||
import {
|
||||
getDataSourceById,
|
||||
getSourceIntegrationObject,
|
||||
} from "../services/datasource";
|
||||
import { binomialABTest, srm } from "../services/stats";
|
||||
import { ExperimentSnapshotInterface } from "../../types/experiment-snapshot";
|
||||
import { addTagsDiff } from "../services/tag";
|
||||
import { userHasAccess } from "../services/organizations";
|
||||
import { removeExperimentFromPresentations } from "../services/presentations";
|
||||
@@ -119,114 +121,6 @@ export async function getExperimentsFrequencyMonth(
|
||||
});
|
||||
}
|
||||
|
||||
// eslint-disable-next-line
|
||||
function getMockSnapshot(phase: number = 0): ExperimentSnapshotInterface {
|
||||
const USERS_A = Math.floor(Math.random() * 300 + 4850);
|
||||
const USERS_B = Math.floor(Math.random() * 300 + 4850);
|
||||
const CONV_A: number[] = [];
|
||||
const CONV_B: number[] = [];
|
||||
for (let i = 0; i < 4; i++) {
|
||||
CONV_A.push(
|
||||
Math.floor(((0.25 * Math.random()) / (Math.pow(i, 2) + 1)) * USERS_A)
|
||||
);
|
||||
}
|
||||
for (let i = 0; i < 4; i++) {
|
||||
CONV_B.push(
|
||||
Math.floor(((0.33 * Math.random()) / (Math.pow(i, 2) + 1)) * USERS_B)
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
id: "something",
|
||||
experiment: "Growth Book",
|
||||
dateCreated: new Date(),
|
||||
phase,
|
||||
manual: false,
|
||||
results: [
|
||||
{
|
||||
name: "All",
|
||||
srm: srm([USERS_A, USERS_B], [0.5, 0.5]),
|
||||
variations: [
|
||||
{
|
||||
users: USERS_A,
|
||||
metrics: {
|
||||
"modal opens": {
|
||||
value: CONV_A[0],
|
||||
cr: CONV_A[0] / USERS_A,
|
||||
users: USERS_A,
|
||||
},
|
||||
"member registrations": {
|
||||
value: CONV_A[1],
|
||||
cr: CONV_A[1] / USERS_A,
|
||||
users: USERS_A,
|
||||
},
|
||||
purchases: {
|
||||
value: CONV_A[2],
|
||||
cr: CONV_A[2] / USERS_A,
|
||||
users: USERS_A,
|
||||
},
|
||||
refunds: {
|
||||
value: CONV_A[3],
|
||||
cr: CONV_A[3] / USERS_A,
|
||||
users: USERS_A,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
users: USERS_B,
|
||||
metrics: {
|
||||
"modal opens": {
|
||||
value: CONV_B[0],
|
||||
cr: CONV_B[0] / USERS_B,
|
||||
users: USERS_B,
|
||||
...binomialABTest(
|
||||
CONV_A[0],
|
||||
USERS_A - CONV_A[0],
|
||||
CONV_B[0],
|
||||
USERS_B - CONV_B[0]
|
||||
),
|
||||
},
|
||||
"member registrations": {
|
||||
value: CONV_B[1],
|
||||
cr: CONV_B[1] / USERS_B,
|
||||
users: USERS_B,
|
||||
...binomialABTest(
|
||||
CONV_A[1],
|
||||
USERS_A - CONV_A[1],
|
||||
CONV_B[1],
|
||||
USERS_B - CONV_B[1]
|
||||
),
|
||||
},
|
||||
purchases: {
|
||||
value: CONV_B[2],
|
||||
cr: CONV_B[2] / USERS_B,
|
||||
users: USERS_B,
|
||||
...binomialABTest(
|
||||
CONV_A[2],
|
||||
USERS_A - CONV_A[2],
|
||||
CONV_B[2],
|
||||
USERS_B - CONV_B[2]
|
||||
),
|
||||
},
|
||||
refunds: {
|
||||
value: CONV_B[3],
|
||||
cr: CONV_B[3] / USERS_B,
|
||||
users: USERS_B,
|
||||
...binomialABTest(
|
||||
CONV_A[3],
|
||||
USERS_A - CONV_A[3],
|
||||
CONV_B[3],
|
||||
USERS_B - CONV_B[3]
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export async function getExperiment(req: AuthRequest, res: Response) {
|
||||
const { id }: { id: string } = req.params;
|
||||
|
||||
@@ -260,9 +154,6 @@ async function _getSnapshot(
|
||||
phase: string,
|
||||
dimension?: string
|
||||
) {
|
||||
// Change this to use mock data
|
||||
const USE_MOCK_DATA = false;
|
||||
|
||||
const experiment = await getExperimentById(id);
|
||||
|
||||
if (!experiment) {
|
||||
@@ -273,9 +164,7 @@ async function _getSnapshot(
|
||||
throw new Error("You do not have access to view this experiment");
|
||||
}
|
||||
|
||||
return USE_MOCK_DATA
|
||||
? getMockSnapshot()
|
||||
: await getLatestSnapshot(experiment.id, parseInt(phase), dimension);
|
||||
return await getLatestSnapshot(experiment.id, parseInt(phase), dimension);
|
||||
}
|
||||
|
||||
export async function getSnapshotWithDimension(
|
||||
@@ -1309,12 +1198,14 @@ export async function putMetric(
|
||||
}
|
||||
|
||||
export async function previewManualSnapshot(
|
||||
req: AuthRequest<{ [key: string]: number[] }>,
|
||||
req: AuthRequest<{
|
||||
users: number[];
|
||||
metrics: { [key: string]: MetricStats[] };
|
||||
}>,
|
||||
res: Response
|
||||
) {
|
||||
const { id, phase }: { id: string; phase: string } = req.params;
|
||||
|
||||
const { users, ...metrics } = req.body;
|
||||
const exp = await getExperimentById(id);
|
||||
|
||||
if (!exp) {
|
||||
@@ -1335,7 +1226,12 @@ export async function previewManualSnapshot(
|
||||
}
|
||||
|
||||
try {
|
||||
const data = await getManualSnapshotData(exp, phaseIndex, users, metrics);
|
||||
const data = await getManualSnapshotData(
|
||||
exp,
|
||||
phaseIndex,
|
||||
req.body.users,
|
||||
req.body.metrics
|
||||
);
|
||||
res.status(200).json({
|
||||
status: 200,
|
||||
snapshot: data,
|
||||
@@ -1352,7 +1248,8 @@ export async function postSnapshot(
|
||||
req: AuthRequest<{
|
||||
phase: number;
|
||||
dimension?: string;
|
||||
data?: { [key: string]: number[] };
|
||||
users?: number[];
|
||||
metrics?: { [key: string]: MetricStats[] };
|
||||
}>,
|
||||
res: Response
|
||||
) {
|
||||
@@ -1389,12 +1286,13 @@ export async function postSnapshot(
|
||||
|
||||
// Manual snapshot
|
||||
if (!exp.datasource) {
|
||||
const {
|
||||
data: { users, ...metrics },
|
||||
} = req.body;
|
||||
|
||||
try {
|
||||
const snapshot = await createManualSnapshot(exp, phase, users, metrics);
|
||||
const snapshot = await createManualSnapshot(
|
||||
exp,
|
||||
phase,
|
||||
req.body.users,
|
||||
req.body.metrics
|
||||
);
|
||||
res.status(200).json({
|
||||
status: 200,
|
||||
snapshot,
|
||||
|
||||
@@ -60,6 +60,7 @@ import { MetricModel } from "../models/MetricModel";
|
||||
import { MetricInterface } from "../../types/metric";
|
||||
import { format } from "sql-formatter";
|
||||
import { PostgresConnectionParams } from "../../types/integrations/postgres";
|
||||
import uniqid from "uniqid";
|
||||
|
||||
export async function getUser(req: AuthRequest, res: Response) {
|
||||
// Ensure user exists in database
|
||||
@@ -100,14 +101,14 @@ export async function postSampleData(req: AuthRequest, res: Response) {
|
||||
|
||||
const existingMetric = await MetricModel.findOne({
|
||||
organization: orgId,
|
||||
id: "met_sample",
|
||||
id: /^met_sample/,
|
||||
});
|
||||
if (existingMetric) {
|
||||
throw new Error("Sample data already exists");
|
||||
}
|
||||
|
||||
const metric: Partial<MetricInterface> = {
|
||||
id: "met_sample",
|
||||
const metric1: Partial<MetricInterface> = {
|
||||
id: uniqid("met_sample_"),
|
||||
dateCreated: new Date(),
|
||||
dateUpdated: new Date(),
|
||||
name: "Sample Conversions",
|
||||
@@ -116,13 +117,25 @@ export async function postSampleData(req: AuthRequest, res: Response) {
|
||||
organization: orgId,
|
||||
userIdType: "anonymous",
|
||||
};
|
||||
await MetricModel.create(metric);
|
||||
await MetricModel.create(metric1);
|
||||
|
||||
const metric2: Partial<MetricInterface> = {
|
||||
id: uniqid("met_sample_"),
|
||||
dateCreated: new Date(),
|
||||
dateUpdated: new Date(),
|
||||
name: "Sample Revenue per User",
|
||||
description: `Part of the Growth Book sample data set. Feel free to delete when finished exploring.`,
|
||||
type: "revenue",
|
||||
organization: orgId,
|
||||
userIdType: "anonymous",
|
||||
};
|
||||
await MetricModel.create(metric2);
|
||||
|
||||
const lastWeek = new Date();
|
||||
lastWeek.setDate(lastWeek.getDate() - 7);
|
||||
|
||||
const experiment: ExperimentInterface = {
|
||||
id: "exp_sample",
|
||||
id: uniqid("exp_sample_"),
|
||||
organization: orgId,
|
||||
archived: false,
|
||||
name: "Sample Experiment",
|
||||
@@ -159,7 +172,7 @@ export async function postSampleData(req: AuthRequest, res: Response) {
|
||||
dateCreated: new Date(),
|
||||
dateUpdated: new Date(),
|
||||
implementation: "code",
|
||||
metrics: [metric.id],
|
||||
metrics: [metric1.id, metric2.id],
|
||||
owner: req.userId,
|
||||
trackingKey: "sample-experiment",
|
||||
userIdType: "anonymous",
|
||||
@@ -167,6 +180,11 @@ export async function postSampleData(req: AuthRequest, res: Response) {
|
||||
conversionWindowDays: 3,
|
||||
results: "won",
|
||||
winner: 1,
|
||||
analysis: `Calling this test a winner given the significant increase in conversions! 💵 🍾
|
||||
|
||||
Revenue did not reach 95% significance, but the risk is so low it doesn't seem worth it to keep waiting.
|
||||
|
||||
**Ready to get some wins yourself?** [Finish setting up your account](/getstarted)`,
|
||||
phases: [
|
||||
{
|
||||
dateStarted: lastWeek,
|
||||
@@ -182,13 +200,35 @@ export async function postSampleData(req: AuthRequest, res: Response) {
|
||||
await ExperimentModel.create(experiment);
|
||||
|
||||
await createManualSnapshot(experiment, 0, [15500, 15400], {
|
||||
[metric.id]: [950, 1025],
|
||||
[metric1.id]: [
|
||||
{
|
||||
count: 950,
|
||||
mean: 1,
|
||||
stddev: 1,
|
||||
},
|
||||
{
|
||||
count: 1025,
|
||||
mean: 1,
|
||||
stddev: 1,
|
||||
},
|
||||
],
|
||||
[metric2.id]: [
|
||||
{
|
||||
count: 950,
|
||||
mean: 26.54,
|
||||
stddev: 16.75,
|
||||
},
|
||||
{
|
||||
count: 1025,
|
||||
mean: 25.13,
|
||||
stddev: 16.87,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
res.status(200).json({
|
||||
status: 200,
|
||||
experiment: experiment.id,
|
||||
metric: metric.id,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -10,13 +10,7 @@ import {
|
||||
getSourceIntegrationObject,
|
||||
getDataSourceById,
|
||||
} from "../services/datasource";
|
||||
import {
|
||||
countABTest,
|
||||
binomialABTest,
|
||||
ABTestStats,
|
||||
bootstrapABTest,
|
||||
getValueCR,
|
||||
} from "../services/stats";
|
||||
import { ABTestStats, getValueCR, abtest } from "../services/stats";
|
||||
import { getMetricsByDatasource } from "../services/experiments";
|
||||
import {
|
||||
QueryMap,
|
||||
@@ -31,7 +25,6 @@ import {
|
||||
MetricValueResult,
|
||||
UsersResult,
|
||||
UsersQueryParams,
|
||||
MetricStats,
|
||||
} from "../types/Integration";
|
||||
import { SegmentModel, SegmentDocument } from "../models/SegmentModel";
|
||||
import { SegmentInterface } from "../../types/segment";
|
||||
@@ -39,6 +32,7 @@ import {
|
||||
SegmentComparisonInterface,
|
||||
SegmentComparisonResults,
|
||||
} from "../../types/segment-comparison";
|
||||
import { MetricStats } from "../../types/metric";
|
||||
|
||||
export async function getAllSegments(req: AuthRequest, res: Response) {
|
||||
const segments = await SegmentModel.find({
|
||||
@@ -178,82 +172,56 @@ async function processResults(
|
||||
// Stats for each metric
|
||||
const metrics = await getMetricsByDatasource(doc.datasource);
|
||||
const selectedMetrics = metrics.filter((m) => doc.metrics.includes(m.id));
|
||||
selectedMetrics.forEach((m) => {
|
||||
const segment1Result: MetricValueResult = data.get(`${m.id}_segment1`)
|
||||
?.result;
|
||||
const segment2Result: MetricValueResult = data.get(`${m.id}_segment2`)
|
||||
?.result;
|
||||
await Promise.all(
|
||||
selectedMetrics.map(async (m) => {
|
||||
const segment1Result: MetricValueResult = data.get(`${m.id}_segment1`)
|
||||
?.result;
|
||||
const segment2Result: MetricValueResult = data.get(`${m.id}_segment2`)
|
||||
?.result;
|
||||
|
||||
// TODO: support calculating total from dates
|
||||
// TODO: support calculating total from dates
|
||||
|
||||
const v1Stats: MetricStats = {
|
||||
count: segment1Result?.count || 0,
|
||||
mean: segment1Result?.mean || 0,
|
||||
stddev: segment1Result?.stddev || 0,
|
||||
};
|
||||
const v2Stats: MetricStats = {
|
||||
count: segment2Result?.count || 0,
|
||||
mean: segment2Result?.mean || 0,
|
||||
stddev: segment2Result?.stddev || 0,
|
||||
};
|
||||
|
||||
const v1 = v1Stats.mean * v1Stats.count;
|
||||
const v2 = v2Stats.mean * v2Stats.count;
|
||||
|
||||
let stats: ABTestStats;
|
||||
if (!v1 || !v2 || !results.users.segment1 || !results.users.segment2) {
|
||||
stats = {
|
||||
buckets: [],
|
||||
chanceToWin: 0,
|
||||
ci: [0, 0],
|
||||
expected: 0,
|
||||
const v1Stats: MetricStats = {
|
||||
count: segment1Result?.count || 0,
|
||||
mean: segment1Result?.mean || 0,
|
||||
stddev: segment1Result?.stddev || 0,
|
||||
};
|
||||
const v2Stats: MetricStats = {
|
||||
count: segment2Result?.count || 0,
|
||||
mean: segment2Result?.mean || 0,
|
||||
stddev: segment2Result?.stddev || 0,
|
||||
};
|
||||
} else if (m.type === "duration") {
|
||||
stats = bootstrapABTest(
|
||||
v1Stats,
|
||||
results.users.segment1,
|
||||
v2Stats,
|
||||
results.users.segment2,
|
||||
m.ignoreNulls
|
||||
);
|
||||
} else if (m.type === "revenue") {
|
||||
stats = bootstrapABTest(
|
||||
v1Stats,
|
||||
results.users.segment1,
|
||||
v2Stats,
|
||||
results.users.segment2,
|
||||
m.ignoreNulls
|
||||
);
|
||||
} else if (m.type === "count") {
|
||||
stats = countABTest(
|
||||
v1,
|
||||
results.users.segment1,
|
||||
v2,
|
||||
results.users.segment2
|
||||
);
|
||||
} else if (m.type === "binomial") {
|
||||
stats = binomialABTest(
|
||||
v1,
|
||||
results.users.segment1 - v1,
|
||||
v2,
|
||||
results.users.segment2 - v2
|
||||
);
|
||||
} else {
|
||||
throw new Error("Not support for metrics of type " + m.type);
|
||||
}
|
||||
|
||||
if (m.inverse) {
|
||||
stats.chanceToWin = 1 - stats.chanceToWin;
|
||||
}
|
||||
const v1 = v1Stats.mean * v1Stats.count;
|
||||
const v2 = v2Stats.mean * v2Stats.count;
|
||||
|
||||
results.metrics[m.id] = {
|
||||
segment1: getValueCR(m, v1, v1Stats.count, results.users.segment1),
|
||||
segment2: {
|
||||
...getValueCR(m, v2, v2Stats.count, results.users.segment2),
|
||||
...stats,
|
||||
},
|
||||
};
|
||||
});
|
||||
let stats: ABTestStats;
|
||||
if (!v1 || !v2 || !results.users.segment1 || !results.users.segment2) {
|
||||
stats = {
|
||||
buckets: [],
|
||||
chanceToWin: 0,
|
||||
ci: [0, 0],
|
||||
expected: 0,
|
||||
};
|
||||
} else {
|
||||
stats = await abtest(
|
||||
m,
|
||||
results.users.segment1,
|
||||
v1Stats,
|
||||
results.users.segment2,
|
||||
v2Stats
|
||||
);
|
||||
}
|
||||
|
||||
results.metrics[m.id] = {
|
||||
segment1: getValueCR(m, v1, v1Stats.count, results.users.segment1),
|
||||
segment2: {
|
||||
...getValueCR(m, v2, v2Stats.count, results.users.segment2),
|
||||
...stats,
|
||||
},
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -28,7 +28,18 @@ const experimentSnapshotSchema = new mongoose.Schema({
|
||||
cr: Number,
|
||||
users: Number,
|
||||
ci: [Number],
|
||||
uplift: {
|
||||
dist: String,
|
||||
mean: Number,
|
||||
stddev: Number,
|
||||
},
|
||||
stats: {
|
||||
mean: Number,
|
||||
count: Number,
|
||||
stddev: Number,
|
||||
},
|
||||
expected: Number,
|
||||
risk: [Number],
|
||||
buckets: [
|
||||
{
|
||||
_id: false,
|
||||
|
||||
@@ -41,6 +41,7 @@ const segmentComparisonSchema = new mongoose.Schema({
|
||||
cr: Number,
|
||||
users: Number,
|
||||
ci: [Number],
|
||||
risk: [Number],
|
||||
expected: Number,
|
||||
buckets: [
|
||||
{
|
||||
|
||||
141
packages/back-end/src/python/bayesian/main.py
Normal file
141
packages/back-end/src/python/bayesian/main.py
Normal file
@@ -0,0 +1,141 @@
|
||||
# Medium article inspiration:
|
||||
# https://towardsdatascience.com/how-to-do-bayesian-a-b-testing-fast-41ee00d55be8
|
||||
# Original code:
|
||||
# https://github.com/itamarfaran/public-sandbox/tree/master/bayesian_blog
|
||||
|
||||
|
||||
import numpy as np
|
||||
from scipy.stats import norm, beta
|
||||
from scipy.special import digamma, polygamma, roots_hermitenorm
|
||||
from orthogonal import roots_sh_jacobi
|
||||
import sys
|
||||
import json
|
||||
|
||||
|
||||
def log_beta_mean(a, b): return digamma(a) - digamma(a + b)
|
||||
def var_beta_mean(a, b): return polygamma(1, a) - polygamma(1, a + b)
|
||||
|
||||
|
||||
def beta_gq(n, a, b):
|
||||
x, w, m = roots_sh_jacobi(n, a + b - 1, a, True)
|
||||
w /= m
|
||||
return x, w
|
||||
|
||||
|
||||
def norm_gq(n, loc, scale):
|
||||
x, w, m = roots_hermitenorm(n, True)
|
||||
x = scale * x + loc
|
||||
w /= m
|
||||
return x, w
|
||||
|
||||
|
||||
def binomial_ab_test(x_a, n_a, x_b, n_b):
|
||||
# Uninformative prior
|
||||
alpha_0, beta_0 = 1, 1
|
||||
|
||||
# Updating prior with data
|
||||
alpha_a = alpha_0 + x_a
|
||||
beta_a = beta_0 + n_a - x_a
|
||||
|
||||
alpha_b = alpha_0 + x_b
|
||||
beta_b = beta_0 + n_b - x_b
|
||||
|
||||
# Chance to win
|
||||
d1_beta = norm(
|
||||
loc=beta.mean(alpha_b, beta_b) - beta.mean(alpha_a, beta_a),
|
||||
scale=np.sqrt(beta.var(alpha_b, beta_b) + beta.var(alpha_a, beta_a))
|
||||
)
|
||||
|
||||
# Credible Interval
|
||||
ci_mean = log_beta_mean(alpha_b, beta_b) - log_beta_mean(alpha_a, beta_a)
|
||||
ci_std = np.sqrt(var_beta_mean(alpha_b, beta_b) +
|
||||
var_beta_mean(alpha_a, beta_a))
|
||||
d2_beta = norm(
|
||||
loc=ci_mean,
|
||||
scale=ci_std
|
||||
)
|
||||
|
||||
# Risk
|
||||
nodes_a, weights_a = beta_gq(24, alpha_a, beta_a)
|
||||
nodes_b, weights_b = beta_gq(24, alpha_b, beta_b)
|
||||
gq = sum(nodes_a * beta.cdf(nodes_a, alpha_b, beta_b) * weights_a) + \
|
||||
sum(nodes_b * beta.cdf(nodes_b, alpha_a, beta_a) * weights_b)
|
||||
risk_beta = gq - beta.mean((alpha_a, alpha_b), (beta_a, beta_b))
|
||||
|
||||
return {
|
||||
"chance_to_win": d1_beta.sf(0),
|
||||
"expected": (np.exp(d2_beta.ppf(0.5)) - 1),
|
||||
"ci": (np.exp(d2_beta.ppf((.025, .975))) - 1).tolist(),
|
||||
"uplift": {
|
||||
"dist": "lognormal",
|
||||
"mean": ci_mean,
|
||||
"stddev": ci_std,
|
||||
},
|
||||
"risk": risk_beta.tolist()
|
||||
}
|
||||
|
||||
|
||||
def gaussian_ab_test(n_a, m_a, s_a, n_b, m_b, s_b):
|
||||
# Uninformative prior
|
||||
mu0, s0, n0 = 0, 1, 0
|
||||
|
||||
# Update the prior
|
||||
inv_vars = n0 / np.power(s0, 2), n_a / np.power(s_a, 2)
|
||||
mu_a = np.average((mu0, m_a), weights=inv_vars)
|
||||
sd_a = 1 / np.sqrt(np.sum(inv_vars))
|
||||
|
||||
inv_vars = n0 / np.power(s0, 2), n_b / np.power(s_b, 2)
|
||||
mu_b = np.average((mu0, m_b), weights=inv_vars)
|
||||
sd_b = 1 / np.sqrt(np.sum(inv_vars))
|
||||
|
||||
# Chance to win
|
||||
d1_norm = norm(loc=mu_b - mu_a, scale=np.sqrt(sd_a ** 2 + sd_b ** 2))
|
||||
|
||||
# Credible interval
|
||||
ci_mean = np.log(mu_b) - np.log(mu_a)
|
||||
ci_std = np.sqrt((sd_a / mu_a)**2 + (sd_b / mu_b)**2)
|
||||
d2_norm = norm(loc=ci_mean, scale=ci_std)
|
||||
|
||||
# Risk
|
||||
nodes_a, weights_a = norm_gq(24, mu_a, sd_a)
|
||||
nodes_b, weights_b = norm_gq(24, mu_b, sd_b)
|
||||
|
||||
gq = sum(nodes_a * norm.cdf(nodes_a, mu_b, sd_b) * weights_a) + \
|
||||
sum(nodes_b * norm.cdf(nodes_b, mu_a, sd_a) * weights_b)
|
||||
risk_norm = gq - norm.mean((mu_a, mu_b))
|
||||
|
||||
return {
|
||||
"chance_to_win": d1_norm.sf(0),
|
||||
"expected": (np.exp(d2_norm.ppf(0.5)) - 1),
|
||||
"ci": (np.exp(d2_norm.ppf((.025, .975))) - 1).tolist(),
|
||||
"uplift": {
|
||||
"dist": "lognormal",
|
||||
"mean": ci_mean,
|
||||
"stddev": ci_std,
|
||||
},
|
||||
"risk": risk_norm.tolist()
|
||||
}
|
||||
|
||||
|
||||
# python main.py binomial \
|
||||
# '{"users":[1283,1321],"count":[254,289],"mean":[52.3,14.1],"stddev":[14.1,13.7]}'
|
||||
# python main.py normal \
|
||||
# '{"users":[1283,1321],"count":[254,289],"mean":[52.3,14.1],"stddev":[14.1,13.7]}'
|
||||
if __name__ == '__main__':
|
||||
metric = sys.argv[1]
|
||||
data = json.loads(sys.argv[2])
|
||||
|
||||
x_a, x_b = data["count"]
|
||||
n_a, n_b = data["users"]
|
||||
m_a, m_b = data["mean"]
|
||||
s_a, s_b = data["stddev"]
|
||||
|
||||
if metric == 'binomial':
|
||||
print(json.dumps(binomial_ab_test(
|
||||
x_a, n_a, x_b, n_b
|
||||
)))
|
||||
|
||||
else:
|
||||
print(json.dumps(gaussian_ab_test(
|
||||
n_a, m_a, s_a, n_b, m_b, s_b
|
||||
)))
|
||||
93
packages/back-end/src/python/bayesian/orthogonal.py
Normal file
93
packages/back-end/src/python/bayesian/orthogonal.py
Normal file
@@ -0,0 +1,93 @@
|
||||
import numpy as np
|
||||
from scipy import linalg
|
||||
from scipy.special import betaln, eval_jacobi
|
||||
|
||||
|
||||
"""
|
||||
see scipy.special.orthogonal.
|
||||
copied source code and implemented log trick to avoid OverflowError
|
||||
"""
|
||||
|
||||
|
||||
def _gen_roots_and_weights(n, log_mu0, an_func, bn_func, f, df, mu):
|
||||
"""
|
||||
see _gen_roots_and_weights in scipy.special.orthogonal
|
||||
"""
|
||||
k = np.arange(n, dtype='d')
|
||||
c = np.zeros((2, n))
|
||||
c[0, 1:] = bn_func(k[1:])
|
||||
c[1, :] = an_func(k)
|
||||
x = linalg.eigvals_banded(c, overwrite_a_band=True)
|
||||
|
||||
# improve roots by one application of Newton's method
|
||||
y = f(n, x)
|
||||
dy = df(n, x)
|
||||
x -= y/dy
|
||||
|
||||
fm = f(n-1, x)
|
||||
fm /= np.abs(fm).max()
|
||||
dy /= np.abs(dy).max()
|
||||
w = 1.0 / (fm * dy)
|
||||
|
||||
log_w = np.log(w) + log_mu0 - np.log(w.sum())
|
||||
|
||||
if mu:
|
||||
return x, log_w, log_mu0
|
||||
else:
|
||||
return x, log_w
|
||||
|
||||
|
||||
def roots_jacobi(n, alpha, beta):
|
||||
"""
|
||||
Gauss-Jacobi quadrature.
|
||||
see scipy.special.root_jacobi
|
||||
"""
|
||||
|
||||
def an_func(k):
|
||||
if a + b == 0.0:
|
||||
return np.where(k == 0, (b - a) / (2 + a + b), 0.0)
|
||||
return np.where(
|
||||
k == 0,
|
||||
(b - a) / (2 + a + b),
|
||||
(b * b - a * a) / ((2.0 * k + a + b) * (2.0 * k + a + b + 2))
|
||||
)
|
||||
|
||||
def bn_func(k):
|
||||
return 2.0 / (2.0*k+a+b)*np.sqrt((k+a)*(k+b) / (2*k+a+b+1)) \
|
||||
* np.where(k == 1, 1.0, np.sqrt(k*(k+a+b) / (2.0*k+a+b-1)))
|
||||
|
||||
def f(n, x):
|
||||
return eval_jacobi(n, a, b, x)
|
||||
|
||||
def df(n, x):
|
||||
return 0.5 * (n + a + b + 1) * eval_jacobi(n-1, a+1, b+1, x)
|
||||
|
||||
m = int(n)
|
||||
if n < 1 or n != m:
|
||||
raise ValueError("n must be a positive integer.")
|
||||
if alpha <= -1 or beta <= -1:
|
||||
raise ValueError("alpha and beta must be greater than -1.")
|
||||
|
||||
log_mu0 = (alpha+beta+1)*np.log(2.0) + betaln(alpha+1, beta+1)
|
||||
a = alpha
|
||||
b = beta
|
||||
|
||||
return _gen_roots_and_weights(m, log_mu0, an_func, bn_func, f, df, True)
|
||||
|
||||
|
||||
def roots_sh_jacobi(n, p1, q1, mu=False):
|
||||
"""
|
||||
Gauss-Jacobi (shifted) quadrature.
|
||||
see scipy.special.roots_sh_jacobi
|
||||
used the log trick to integrate over large values of a,b
|
||||
"""
|
||||
if (p1-q1) <= -1 or q1 <= 0:
|
||||
raise ValueError(
|
||||
"(p - q) must be greater than -1, and q must be greater than 0.")
|
||||
x, log_w, log_m = roots_jacobi(n, p1 - q1, q1 - 1)
|
||||
x = (x + 1) / 2
|
||||
w = np.exp(log_w - log_m)
|
||||
if mu:
|
||||
return x, w, 1
|
||||
else:
|
||||
return x, w
|
||||
@@ -5,21 +5,14 @@ import {
|
||||
} from "../../types/experiment-snapshot";
|
||||
import { MetricModel } from "../models/MetricModel";
|
||||
import uniqid from "uniqid";
|
||||
import {
|
||||
binomialABTest,
|
||||
srm,
|
||||
ABTestStats,
|
||||
countABTest,
|
||||
bootstrapABTest,
|
||||
getValueCR,
|
||||
} from "./stats";
|
||||
import { srm, ABTestStats, abtest, getValueCR } from "./stats";
|
||||
import { getSourceIntegrationObject } from "./datasource";
|
||||
import { addTags } from "./tag";
|
||||
import { WatchModel } from "../models/WatchModel";
|
||||
import { QueryMap } from "./queries";
|
||||
import { PastExperimentResult } from "../types/Integration";
|
||||
import { ExperimentSnapshotModel } from "../models/ExperimentSnapshotModel";
|
||||
import { MetricInterface } from "../../types/metric";
|
||||
import { MetricInterface, MetricStats } from "../../types/metric";
|
||||
import { ExperimentInterface } from "../../types/experiment";
|
||||
import { DimensionInterface } from "../../types/dimension";
|
||||
import { DataSourceInterface } from "../../types/datasource";
|
||||
@@ -198,7 +191,9 @@ export async function getManualSnapshotData(
|
||||
experiment: ExperimentInterface,
|
||||
phaseIndex: number,
|
||||
users: number[],
|
||||
metrics: { [key: string]: number[] }
|
||||
metrics: {
|
||||
[key: string]: MetricStats[];
|
||||
}
|
||||
) {
|
||||
// Default variation values, override from SQL results if available
|
||||
const variations: SnapshotVariation[] = experiment.variations.map((v, i) => ({
|
||||
@@ -214,64 +209,45 @@ export async function getManualSnapshotData(
|
||||
metricMap.set(m.id, m);
|
||||
});
|
||||
|
||||
Object.keys(metrics).forEach((m) => {
|
||||
const metric = metricMap.get(m);
|
||||
experiment.variations.forEach((v, i) => {
|
||||
// Baseline
|
||||
if (!i) {
|
||||
variations[i].metrics[m] = getValueCR(
|
||||
metric,
|
||||
metrics[m][i],
|
||||
users[i],
|
||||
users[i]
|
||||
);
|
||||
}
|
||||
// Variation
|
||||
else {
|
||||
const type = metric.type;
|
||||
let stats: ABTestStats;
|
||||
|
||||
if (type === "binomial") {
|
||||
stats = binomialABTest(
|
||||
metrics[m][0],
|
||||
users[0] - metrics[m][0],
|
||||
metrics[m][i],
|
||||
users[i] - metrics[m][i]
|
||||
await Promise.all(
|
||||
Object.keys(metrics).map((m) => {
|
||||
const metric = metricMap.get(m);
|
||||
return Promise.all(
|
||||
experiment.variations.map(async (v, i) => {
|
||||
const valueCR = getValueCR(
|
||||
metric,
|
||||
metrics[m][i].mean * metrics[m][i].count,
|
||||
metrics[m][i].count,
|
||||
users[i]
|
||||
);
|
||||
} else if (type === "count") {
|
||||
stats = countABTest(metrics[m][0], users[0], metrics[m][i], users[i]);
|
||||
} else if (type === "duration") {
|
||||
stats = bootstrapABTest(
|
||||
{
|
||||
mean: metrics[m][0] / users[0],
|
||||
count: users[0],
|
||||
stddev: metrics[m][0] / users[0],
|
||||
},
|
||||
users[0],
|
||||
{
|
||||
mean: metrics[m][i] / users[i],
|
||||
count: users[i],
|
||||
stddev: metrics[m][i] / users[i],
|
||||
},
|
||||
users[i],
|
||||
metric?.ignoreNulls || false
|
||||
);
|
||||
} else {
|
||||
throw new Error("Metric type not supported: " + type);
|
||||
}
|
||||
// TODO: support other metric types
|
||||
|
||||
if (metric.inverse) {
|
||||
stats.chanceToWin = 1 - stats.chanceToWin;
|
||||
}
|
||||
// Baseline
|
||||
if (!i) {
|
||||
variations[i].metrics[m] = {
|
||||
...valueCR,
|
||||
stats: metrics[m][i],
|
||||
};
|
||||
}
|
||||
// Variation
|
||||
else {
|
||||
const result = await abtest(
|
||||
metric,
|
||||
users[0],
|
||||
metrics[m][0],
|
||||
users[i],
|
||||
metrics[m][i]
|
||||
);
|
||||
|
||||
variations[i].metrics[m] = {
|
||||
...getValueCR(metric, metrics[m][i], users[i], users[i]),
|
||||
...stats,
|
||||
};
|
||||
}
|
||||
});
|
||||
});
|
||||
variations[i].metrics[m] = {
|
||||
...valueCR,
|
||||
...result,
|
||||
stats: metrics[m][i],
|
||||
};
|
||||
}
|
||||
})
|
||||
);
|
||||
})
|
||||
);
|
||||
|
||||
// Check to see if the observed number of samples per variation matches what we expect
|
||||
// This returns a p-value and a small value indicates the results are untrustworthy
|
||||
@@ -290,7 +266,9 @@ export async function createManualSnapshot(
|
||||
experiment: ExperimentInterface,
|
||||
phaseIndex: number,
|
||||
users: number[],
|
||||
metrics: { [key: string]: number[] }
|
||||
metrics: {
|
||||
[key: string]: MetricStats[];
|
||||
}
|
||||
) {
|
||||
const { srm, variations } = await getManualSnapshotData(
|
||||
experiment,
|
||||
@@ -373,127 +351,105 @@ export async function createSnapshot(
|
||||
variations: SnapshotVariation[];
|
||||
}[] = [];
|
||||
|
||||
rows.forEach((d) => {
|
||||
// Default variation values, override from SQL results if available
|
||||
const variations: SnapshotVariation[] = experiment.variations.map(() => ({
|
||||
users: 0,
|
||||
metrics: {},
|
||||
}));
|
||||
await Promise.all(
|
||||
rows.map(async (d) => {
|
||||
// Default variation values, override from SQL results if available
|
||||
const variations: SnapshotVariation[] = experiment.variations.map(() => ({
|
||||
users: 0,
|
||||
metrics: {},
|
||||
}));
|
||||
|
||||
const metricData = new Map<
|
||||
string,
|
||||
{ count: number; mean: number; stddev: number }[]
|
||||
>();
|
||||
d.variations.forEach((row) => {
|
||||
const variation = row.variation;
|
||||
if (!variations[variation]) {
|
||||
return;
|
||||
}
|
||||
variations[variation].users = row.users || 0;
|
||||
|
||||
row.metrics.forEach((m) => {
|
||||
const doc = metricData.get(m.metric) || [];
|
||||
doc[variation] = {
|
||||
count: m.count,
|
||||
mean: m.mean,
|
||||
stddev: m.stddev,
|
||||
};
|
||||
metricData.set(m.metric, doc);
|
||||
});
|
||||
});
|
||||
|
||||
metricData.forEach((v, k) => {
|
||||
const baselineSuccess = v[0]?.count * v[0]?.mean || 0;
|
||||
|
||||
v.forEach((data, i) => {
|
||||
const success = data.count * data.mean;
|
||||
|
||||
const metric = metricMap.get(k);
|
||||
const type = metric?.type || "binomial";
|
||||
const ignoreNulls = metric?.ignoreNulls || false;
|
||||
|
||||
const value = success;
|
||||
|
||||
// Don't do stats for the baseline or when breaking down by dimension
|
||||
// We aren't doing a correction for multiple tests, so the numbers would be misleading for the break down
|
||||
// Can enable this later when we have a more robust stats engine
|
||||
if (!i || dimension) {
|
||||
variations[i].metrics[k] = getValueCR(
|
||||
metric,
|
||||
value,
|
||||
data.count,
|
||||
variations[i].users
|
||||
);
|
||||
const metricData = new Map<
|
||||
string,
|
||||
{ count: number; mean: number; stddev: number }[]
|
||||
>();
|
||||
d.variations.forEach((row) => {
|
||||
const variation = row.variation;
|
||||
if (!variations[variation]) {
|
||||
return;
|
||||
}
|
||||
variations[variation].users = row.users || 0;
|
||||
|
||||
let stats: ABTestStats;
|
||||
// Short cut if either the baseline or variation has no data
|
||||
if (!baselineSuccess || !success) {
|
||||
stats = {
|
||||
buckets: [],
|
||||
chanceToWin: 0,
|
||||
ci: [0, 0],
|
||||
expected: 0,
|
||||
row.metrics.forEach((m) => {
|
||||
const doc = metricData.get(m.metric) || [];
|
||||
doc[variation] = {
|
||||
count: m.count,
|
||||
mean: m.mean,
|
||||
stddev: m.stddev,
|
||||
};
|
||||
} else if (type === "binomial") {
|
||||
stats = binomialABTest(
|
||||
baselineSuccess,
|
||||
variations[0].users - baselineSuccess,
|
||||
success,
|
||||
variations[i].users - success
|
||||
);
|
||||
} else if (type === "count") {
|
||||
stats = countABTest(
|
||||
baselineSuccess,
|
||||
variations[0].users,
|
||||
success,
|
||||
variations[i].users
|
||||
);
|
||||
} else if (type === "duration") {
|
||||
stats = bootstrapABTest(
|
||||
v[0],
|
||||
variations[0].users,
|
||||
data,
|
||||
variations[i].users,
|
||||
ignoreNulls
|
||||
);
|
||||
} else if (type === "revenue") {
|
||||
stats = bootstrapABTest(
|
||||
v[0],
|
||||
variations[0].users,
|
||||
data,
|
||||
variations[i].users,
|
||||
ignoreNulls
|
||||
);
|
||||
} else {
|
||||
throw new Error("Metric type not supported: " + type);
|
||||
}
|
||||
|
||||
if (metric.inverse) {
|
||||
stats.chanceToWin = 1 - stats.chanceToWin;
|
||||
}
|
||||
|
||||
variations[i].metrics[k] = {
|
||||
...getValueCR(metric, value, data.count, variations[i].users),
|
||||
...stats,
|
||||
};
|
||||
metricData.set(m.metric, doc);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Check to see if the observed number of samples per variation matches what we expect
|
||||
// This returns a p-value and a small value indicates the results are untrustworthy
|
||||
const sampleRatioMismatch = srm(
|
||||
variations.map((v) => v.users),
|
||||
phase.variationWeights
|
||||
);
|
||||
const metricPromises: Promise<void[]>[] = [];
|
||||
metricData.forEach((v, k) => {
|
||||
const baselineSuccess = v[0]?.count * v[0]?.mean || 0;
|
||||
|
||||
results.push({
|
||||
name: d.dimension,
|
||||
srm: sampleRatioMismatch,
|
||||
variations,
|
||||
});
|
||||
});
|
||||
metricPromises.push(
|
||||
Promise.all(
|
||||
v.map(async (data, i) => {
|
||||
const success = data.count * data.mean;
|
||||
|
||||
const metric = metricMap.get(k);
|
||||
const value = success;
|
||||
|
||||
// Don't do stats for the baseline or when breaking down by dimension
|
||||
// We aren't doing a correction for multiple tests, so the numbers would be misleading for the break down
|
||||
// Can enable this later when we have a more robust stats engine
|
||||
if (!i || dimension) {
|
||||
variations[i].metrics[k] = {
|
||||
...getValueCR(metric, value, data.count, variations[i].users),
|
||||
stats: data,
|
||||
};
|
||||
return;
|
||||
}
|
||||
|
||||
let result: ABTestStats;
|
||||
// Short cut if either the baseline or variation has no data
|
||||
if (!baselineSuccess || !success) {
|
||||
result = {
|
||||
buckets: [],
|
||||
chanceToWin: 0,
|
||||
ci: [0, 0],
|
||||
risk: [0, 0],
|
||||
expected: 0,
|
||||
};
|
||||
} else {
|
||||
result = await abtest(
|
||||
metric,
|
||||
variations[0].users,
|
||||
v[0],
|
||||
variations[i].users,
|
||||
data
|
||||
);
|
||||
}
|
||||
|
||||
variations[i].metrics[k] = {
|
||||
...getValueCR(metric, value, data.count, variations[i].users),
|
||||
...result,
|
||||
stats: data,
|
||||
};
|
||||
})
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
await Promise.all(metricPromises);
|
||||
|
||||
// Check to see if the observed number of samples per variation matches what we expect
|
||||
// This returns a p-value and a small value indicates the results are untrustworthy
|
||||
const sampleRatioMismatch = srm(
|
||||
variations.map((v) => v.users),
|
||||
phase.variationWeights
|
||||
);
|
||||
|
||||
results.push({
|
||||
name: d.dimension,
|
||||
srm: sampleRatioMismatch,
|
||||
variations,
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
const data: ExperimentSnapshotInterface = {
|
||||
id: uniqid("snp_"),
|
||||
|
||||
@@ -1,217 +1,93 @@
|
||||
// eslint-disable-next-line
|
||||
/// <reference path="../types/jstat.d.ts" />
|
||||
import { jStat } from "jstat";
|
||||
import { MetricInterface } from "../../types/metric";
|
||||
import { MetricStats } from "../types/Integration";
|
||||
import { MetricInterface, MetricStats } from "../../types/metric";
|
||||
import { PythonShell } from "python-shell";
|
||||
import path from "path";
|
||||
import { promisify } from "util";
|
||||
|
||||
export interface ABTestStats {
|
||||
expected: number;
|
||||
chanceToWin: number;
|
||||
uplift?: {
|
||||
dist: string;
|
||||
mean?: number;
|
||||
stddev?: number;
|
||||
};
|
||||
ci: [number, number];
|
||||
risk?: [number, number];
|
||||
buckets: {
|
||||
x: number;
|
||||
y: number;
|
||||
}[];
|
||||
}
|
||||
|
||||
// From: https://www.evanmiller.org/bayesian-ab-testing.html
|
||||
function binomialChanceToWin(
|
||||
aSuccess: number,
|
||||
aFailure: number,
|
||||
bSuccess: number,
|
||||
bFailure: number
|
||||
) {
|
||||
let total = 0;
|
||||
for (let i = 0; i < bSuccess; i++) {
|
||||
total += Math.exp(
|
||||
jStat.betaln(aSuccess + i + 1, bFailure + aFailure + 2) -
|
||||
Math.log(bFailure + i + 1) -
|
||||
jStat.betaln(1 + i, bFailure + 1) -
|
||||
jStat.betaln(aSuccess + 1, aFailure + 1)
|
||||
);
|
||||
export async function abtest(
|
||||
metric: MetricInterface,
|
||||
aUsers: number,
|
||||
aStats: MetricStats,
|
||||
bUsers: number,
|
||||
bStats: MetricStats
|
||||
): Promise<ABTestStats> {
|
||||
if (metric.ignoreNulls) {
|
||||
aUsers = aStats.count;
|
||||
bUsers = bStats.count;
|
||||
} else {
|
||||
aStats = {
|
||||
...aStats,
|
||||
mean: (aStats.mean * aStats.count) / aUsers,
|
||||
stddev: (aStats.stddev * Math.sqrt(aStats.count)) / Math.sqrt(aUsers),
|
||||
};
|
||||
bStats = {
|
||||
...bStats,
|
||||
mean: (bStats.mean * bStats.count) / bUsers,
|
||||
stddev: (bStats.stddev * Math.sqrt(bStats.count)) / Math.sqrt(bUsers),
|
||||
};
|
||||
}
|
||||
return total;
|
||||
}
|
||||
function countChanceToWin(
|
||||
aCount: number,
|
||||
aVisits: number,
|
||||
bCount: number,
|
||||
bVisits: number
|
||||
) {
|
||||
let total = 0;
|
||||
for (let k = 0; k < bCount; k++) {
|
||||
total += Math.exp(
|
||||
k * Math.log(bVisits) +
|
||||
aCount * Math.log(aVisits) -
|
||||
(k + aCount) * Math.log(bVisits + aVisits) -
|
||||
Math.log(k + aCount) -
|
||||
jStat.betaln(k + 1, aCount)
|
||||
);
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
function abTest(
|
||||
sampleA: () => number,
|
||||
sampleB: () => number,
|
||||
chanceToWin: number | null,
|
||||
expected: number
|
||||
): ABTestStats {
|
||||
const NUM_SAMPLES = 1e5;
|
||||
const options = {
|
||||
args: [
|
||||
metric.type,
|
||||
JSON.stringify({
|
||||
users: [aUsers, bUsers],
|
||||
count: [aStats.count, bStats.count],
|
||||
mean: [aStats.mean, bStats.mean],
|
||||
stddev: [aStats.stddev, bStats.stddev],
|
||||
}),
|
||||
],
|
||||
};
|
||||
|
||||
// Simulate the distributions a bunch of times to get a list of percent changes
|
||||
const change: number[] = Array(NUM_SAMPLES);
|
||||
let wins = 0;
|
||||
for (let i = 0; i < NUM_SAMPLES; i++) {
|
||||
const a = sampleA();
|
||||
const b = sampleB();
|
||||
change[i] = (b - a) / a;
|
||||
if (change[i] > 0) wins++;
|
||||
}
|
||||
change.sort((a, b) => {
|
||||
return a - b;
|
||||
});
|
||||
const script = path.join(__dirname, "..", "python", "bayesian", "main.py");
|
||||
|
||||
const simulatedCI: [number, number] = [
|
||||
change[Math.floor(change.length * 0.025)],
|
||||
change[Math.floor(change.length * 0.975)],
|
||||
];
|
||||
|
||||
let ci: [number, number];
|
||||
|
||||
// Using the simulation for chanceToWin and the CI
|
||||
if (chanceToWin === null) {
|
||||
chanceToWin = wins / NUM_SAMPLES;
|
||||
ci = simulatedCI;
|
||||
}
|
||||
// If chanceToWin was calculated using a Bayesian formula
|
||||
else {
|
||||
// If it's close to significance, estimate the CI from the chanceToWin, otherwise, use the simulation data for the CI.
|
||||
// This is a hacky fix for when the simulation CI crosses zero even though chanceToWin is significant.
|
||||
// The bug happens because the CI and chanceToWin are calculated using different methods and don't always 100% agree.
|
||||
// A better fix is to calculate a Bayesian credible interval instead of using a frequentist Confidence Interval
|
||||
if (
|
||||
(chanceToWin > 0.7 && chanceToWin < 0.99) ||
|
||||
(chanceToWin < 0.3 && chanceToWin > 0.01)
|
||||
) {
|
||||
ci = getCIFromChanceToWin(chanceToWin, expected);
|
||||
} else {
|
||||
ci = simulatedCI;
|
||||
}
|
||||
const result = await promisify(PythonShell.run)(script, options);
|
||||
let parsed: {
|
||||
chance_to_win: number;
|
||||
expected: number;
|
||||
ci: [number, number];
|
||||
risk: [number, number];
|
||||
uplift: {
|
||||
dist: string;
|
||||
mean?: number;
|
||||
stddev?: number;
|
||||
};
|
||||
};
|
||||
try {
|
||||
parsed = JSON.parse(result[0]);
|
||||
} catch (e) {
|
||||
console.error("Failed to run stats model", options.args, result);
|
||||
throw e;
|
||||
}
|
||||
|
||||
return {
|
||||
ci,
|
||||
expected,
|
||||
expected: parsed.expected,
|
||||
chanceToWin: metric.inverse
|
||||
? 1 - parsed.chance_to_win
|
||||
: parsed.chance_to_win,
|
||||
ci: parsed.ci,
|
||||
risk: parsed.risk,
|
||||
uplift: parsed.uplift,
|
||||
buckets: [],
|
||||
chanceToWin,
|
||||
};
|
||||
}
|
||||
|
||||
function getCIFromChanceToWin(
|
||||
chanceToWin: number,
|
||||
percentImprovement: number
|
||||
): [number, number] {
|
||||
const alpha = 0.05;
|
||||
|
||||
const a = jStat.normal.inv(1 - chanceToWin, 0, 1);
|
||||
const b = jStat.normal.inv(chanceToWin > 0.5 ? alpha : 1 - alpha, 0, 1);
|
||||
|
||||
const d = Math.abs((percentImprovement * b) / a);
|
||||
console.log({
|
||||
chanceToWin,
|
||||
percentImprovement,
|
||||
a,
|
||||
b,
|
||||
d,
|
||||
});
|
||||
return [percentImprovement - d, percentImprovement + d];
|
||||
}
|
||||
|
||||
function getExpectedValue(
|
||||
a: number,
|
||||
nA: number,
|
||||
b: number,
|
||||
nB: number
|
||||
): number {
|
||||
const pA = a / nA;
|
||||
const pB = b / nB;
|
||||
return (pB - pA) / pA;
|
||||
}
|
||||
|
||||
export function binomialABTest(
|
||||
aSuccess: number,
|
||||
aFailure: number,
|
||||
bSuccess: number,
|
||||
bFailure: number
|
||||
) {
|
||||
return abTest(
|
||||
() => jStat.beta.sample(aSuccess + 1, aFailure + 1),
|
||||
() => jStat.beta.sample(bSuccess + 1, bFailure + 1),
|
||||
binomialChanceToWin(aSuccess, aFailure, bSuccess, bFailure),
|
||||
getExpectedValue(
|
||||
aSuccess,
|
||||
aSuccess + aFailure,
|
||||
bSuccess,
|
||||
bSuccess + bFailure
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export function countABTest(
|
||||
aCount: number,
|
||||
aVisits: number,
|
||||
bCount: number,
|
||||
bVisits: number
|
||||
) {
|
||||
return abTest(
|
||||
() => jStat.gamma.sample(aCount, 1 / aVisits),
|
||||
() => jStat.gamma.sample(bCount, 1 / bVisits),
|
||||
countChanceToWin(aCount, aVisits, bCount, bVisits),
|
||||
getExpectedValue(aCount, aVisits, bCount, bVisits)
|
||||
);
|
||||
}
|
||||
|
||||
export function bootstrapABTest(
|
||||
aStats: MetricStats,
|
||||
aVisits: number,
|
||||
bStats: MetricStats,
|
||||
bVisits: number,
|
||||
ignoreNulls: boolean
|
||||
) {
|
||||
const getSampleFunction = (stats: MetricStats, visits: number) => {
|
||||
// Standard error (using the Central Limit Theorem)
|
||||
const se = stats.stddev / Math.sqrt(stats.count);
|
||||
|
||||
if (ignoreNulls) {
|
||||
return () => jStat.normal.sample(stats.mean, se);
|
||||
}
|
||||
|
||||
// Standard deviation of the conversion rate
|
||||
const crStddev = Math.sqrt(stats.count * (1 - stats.count / visits));
|
||||
|
||||
return () =>
|
||||
(jStat.normal.sample(stats.count, crStddev) / visits) *
|
||||
jStat.normal.sample(stats.mean, se);
|
||||
};
|
||||
|
||||
let expected: number;
|
||||
if (ignoreNulls) {
|
||||
expected = (bStats.mean - aStats.mean) / aStats.mean;
|
||||
} else {
|
||||
const aTotalMean = (aStats.mean * aStats.count) / aVisits;
|
||||
const bTotalMean = (bStats.mean * bStats.count) / bVisits;
|
||||
expected = (bTotalMean - aTotalMean) / aTotalMean;
|
||||
}
|
||||
|
||||
return abTest(
|
||||
getSampleFunction(aStats, aVisits),
|
||||
getSampleFunction(bStats, bVisits),
|
||||
null,
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
export function getValueCR(
|
||||
metric: MetricInterface,
|
||||
value: number,
|
||||
|
||||
@@ -4,15 +4,9 @@ import {
|
||||
} from "../../types/datasource";
|
||||
import { DimensionInterface } from "../../types/dimension";
|
||||
import { ExperimentInterface, ExperimentPhase } from "../../types/experiment";
|
||||
import { MetricInterface } from "../../types/metric";
|
||||
import { MetricInterface, MetricStats } from "../../types/metric";
|
||||
import { SegmentInterface } from "../../types/segment";
|
||||
|
||||
export interface MetricStats {
|
||||
count: number;
|
||||
stddev: number;
|
||||
mean: number;
|
||||
}
|
||||
|
||||
export type VariationMetricResult = MetricStats & {
|
||||
metric: string;
|
||||
};
|
||||
|
||||
1
packages/back-end/src/types/jstat.d.ts
vendored
1
packages/back-end/src/types/jstat.d.ts
vendored
@@ -1 +0,0 @@
|
||||
declare module "jstat";
|
||||
@@ -7,7 +7,8 @@
|
||||
"noImplicitAny": true,
|
||||
"moduleResolution": "node",
|
||||
"sourceMap": true,
|
||||
"outDir": "dist"
|
||||
"outDir": "dist",
|
||||
"typeRoots": ["node_modules/@types", "./typings"]
|
||||
},
|
||||
"include": ["src/**/*", "types/AuditModel.ts", "types/**/*"]
|
||||
"include": ["src/**/*", "types/**/*.d.ts", "typings/jstat/index.d.ts"]
|
||||
}
|
||||
|
||||
13
packages/back-end/tsconfig.spec.json
Normal file
13
packages/back-end/tsconfig.spec.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "commonjs",
|
||||
"esModuleInterop": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"target": "es6",
|
||||
"noImplicitAny": true,
|
||||
"moduleResolution": "node",
|
||||
"sourceMap": true,
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src/**/*", "types/**/*", "typings/jstat/index.d.ts"]
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { QueryLanguage } from "./datasource";
|
||||
import { MetricStats } from "./metric";
|
||||
|
||||
export interface SnapshotMetric {
|
||||
value: number;
|
||||
@@ -6,6 +7,13 @@ export interface SnapshotMetric {
|
||||
users: number;
|
||||
ci?: [number, number];
|
||||
expected?: number;
|
||||
risk?: [number, number];
|
||||
stats?: MetricStats;
|
||||
uplift?: {
|
||||
dist: string;
|
||||
mean?: number;
|
||||
stddev?: number;
|
||||
};
|
||||
buckets?: {
|
||||
x: number;
|
||||
y: number;
|
||||
@@ -3,6 +3,12 @@ import { Queries } from "./query";
|
||||
export type Operator = "=" | "!=" | "~" | "!~" | ">" | "<" | "<=" | ">=";
|
||||
export type MetricType = "binomial" | "count" | "duration" | "revenue";
|
||||
|
||||
export interface MetricStats {
|
||||
count: number;
|
||||
stddev: number;
|
||||
mean: number;
|
||||
}
|
||||
|
||||
export interface MetricAnalysis {
|
||||
createdAt: Date;
|
||||
users: number;
|
||||
11
packages/back-end/typings/jstat/index.d.ts
vendored
Normal file
11
packages/back-end/typings/jstat/index.d.ts
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
declare module "jstat" {
|
||||
namespace jStat {
|
||||
namespace normal {
|
||||
export function inv(n: number, mean: number, stddev: number): number;
|
||||
export function pdf(x: number, mean: number, stddev: number): number;
|
||||
}
|
||||
namespace chisquare {
|
||||
export function cdf(x: number, l: number): number;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -52,18 +52,24 @@ The Client Libraries never communicate with the Growth Book servers. That means
|
||||
This separation has huge performance and reliability benefits (if Growth Book goes down, it has no effect on your app), but it can be a bit unintuitive when you press the "Stop" button in the UI and people continue to be put into the experiment.
|
||||
|
||||
To get the best of both worlds, you can integrate with the [API](/api-docs). In essense, you would periodically fetch and cache the latest experiment settings in something like Redis. Then your app would query the cache to determine which experiments are running. You get the benefits of separation and the convenience of using the Growth Book UI to manage experiments.
|
||||
|
||||
## Results
|
||||
|
||||
|
||||
|
||||
Each row of this table is a different metric.
|
||||
|
||||
**Chance to Beat Control** is a bayesian statistic telling you how likely it is that the variation is better. Typically,
|
||||
you want to wait until this reaches 95% at which point the row will turn green. Likewise, if this number drops below 5%, the row will turn red indicating the variation is significantly worse than the control.
|
||||
**Chance to Beat Control** tells you the probability that the variation is better. The closer this gets to 100%, the more likely the variation is better. The closer it gets to 0% the more likely the variation is worse.
|
||||
Numbers in the middle are inconclusive and require more data. We recommend waiting until it reaches 95% or 5% before deciding on a winner as a general rule of thumb.
|
||||
|
||||
**Percent Change** shows how much better/worse the variation is compared to the control. To generate these numbers, we use statistical bootstrapping and simulate your experiment 10,000 times. What you see is the 95% confidence interval from these simulations.
|
||||
**Risk** tells you how much you will lose if you choose the variation as the winner, but it ends up actually being worse (no matter how unlikely that is).
|
||||
|
||||
If you have pre-defined dimensions for your users, you can use the **Dimension** dropdown to drill down into your results.
|
||||
**Percent Change** shows how much better/worse the variation is compared to the control. It is a probability density graph and the thicker the area, the more likely the true percent change will be there.
|
||||
As you collect more data, the tails of the graphs will shorten, indicating more certainty around the estimates.
|
||||
|
||||
### Dimensions
|
||||
|
||||
If you have defined dimensions for your users, you can use the **Dimension** dropdown to drill down into your results.
|
||||
This is very useful for debugging (e.g. if Safari is down, but the other browser are fine, you may have an implementation bug).
|
||||
|
||||
Be careful, the more metrics and dimensions you look at, the more likely you are to see a false positive. If you find something that looks
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 64 KiB After Width: | Height: | Size: 66 KiB |
@@ -5,9 +5,15 @@ import { scaleLinear } from "@visx/scale";
|
||||
import ParentSize from "@visx/responsive/lib/components/ParentSize";
|
||||
import { Line } from "@visx/shape";
|
||||
import { FaArrowUp, FaArrowDown } from "react-icons/fa";
|
||||
import { ViolinPlot } from "@visx/stats";
|
||||
import { jStat } from "jstat";
|
||||
|
||||
export interface Props {
|
||||
id: string;
|
||||
ci?: [number, number] | [];
|
||||
barType?: "pill" | "violin";
|
||||
barFillType?: "gradient" | "significant";
|
||||
uplift?: { dist: string; mean?: number; stddev?: number };
|
||||
domain: [number, number];
|
||||
//width: string | number;
|
||||
height: number;
|
||||
@@ -27,7 +33,11 @@ export interface Props {
|
||||
}
|
||||
|
||||
const AlignedGraph: FC<Props> = ({
|
||||
id,
|
||||
ci,
|
||||
barType = "pill",
|
||||
barFillType = "gradient",
|
||||
uplift,
|
||||
domain,
|
||||
expected,
|
||||
significant = false,
|
||||
@@ -43,8 +53,11 @@ const AlignedGraph: FC<Props> = ({
|
||||
barColor = "#aaaaaaaa",
|
||||
sigBarColorPos = "#0D8C8Ccc",
|
||||
sigBarColorNeg = "#D94032cc",
|
||||
expectedColor = "#fb8500",
|
||||
}) => {
|
||||
if (barType == "violin" && !uplift) {
|
||||
barType = "pill";
|
||||
}
|
||||
|
||||
const barThickness = 16;
|
||||
|
||||
const tickLabelColor = axisColor;
|
||||
@@ -77,6 +90,35 @@ const AlignedGraph: FC<Props> = ({
|
||||
// todo: make ticks programic based roughtly on the width
|
||||
// todo: make the significant threashold centralized, and adjustable.
|
||||
|
||||
const gradient: { color: string; percent: number }[] = [];
|
||||
const gradientId = "gr_" + id;
|
||||
if (ci && barFillType === "gradient") {
|
||||
if (ci[0] < 0) {
|
||||
gradient.push({ color: sigBarColorNeg, percent: 0 });
|
||||
if (ci[1] > 0) {
|
||||
const w = ci[1] - ci[0];
|
||||
const wNeg = (100 * (-1 * ci[0])) / w;
|
||||
gradient.push({ color: sigBarColorNeg, percent: wNeg });
|
||||
gradient.push({ color: sigBarColorPos, percent: wNeg + 0.001 });
|
||||
gradient.push({ color: sigBarColorPos, percent: 100 });
|
||||
} else {
|
||||
gradient.push({ color: sigBarColorNeg, percent: 100 });
|
||||
}
|
||||
} else {
|
||||
gradient.push({ color: sigBarColorPos, percent: 0 });
|
||||
gradient.push({ color: sigBarColorPos, percent: 100 });
|
||||
}
|
||||
}
|
||||
|
||||
const barFill =
|
||||
barFillType === "gradient"
|
||||
? `url(#${gradientId})`
|
||||
: significant
|
||||
? expected > 0
|
||||
? sigBarColorPos
|
||||
: sigBarColorNeg
|
||||
: barColor;
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="d-flex aligned-graph align-items-center aligned-graph-row">
|
||||
@@ -94,6 +136,25 @@ const AlignedGraph: FC<Props> = ({
|
||||
});
|
||||
return (
|
||||
<svg width={graphWidth} height={height}>
|
||||
{gradient.length > 0 && (
|
||||
<defs>
|
||||
<linearGradient
|
||||
id={gradientId}
|
||||
x1="0%"
|
||||
y1="0%"
|
||||
x2="100%"
|
||||
y2="0%"
|
||||
>
|
||||
{gradient.map((g) => (
|
||||
<stop
|
||||
key={g.percent}
|
||||
offset={g.percent + "%"}
|
||||
stopColor={g.color}
|
||||
/>
|
||||
))}
|
||||
</linearGradient>
|
||||
</defs>
|
||||
)}
|
||||
{!showAxis && (
|
||||
<>
|
||||
<GridColumns
|
||||
@@ -133,24 +194,68 @@ const AlignedGraph: FC<Props> = ({
|
||||
)}
|
||||
{!axisOnly && (
|
||||
<>
|
||||
<rect
|
||||
x={xScale(ci[0])}
|
||||
y={barHeight}
|
||||
width={xScale(ci[1]) - xScale(ci[0])}
|
||||
height={barThickness}
|
||||
fill={
|
||||
significant
|
||||
? expected > 0
|
||||
? sigBarColorPos
|
||||
: sigBarColorNeg
|
||||
: barColor
|
||||
}
|
||||
rx={8}
|
||||
/>
|
||||
{barType === "violin" && (
|
||||
<ViolinPlot
|
||||
top={barHeight}
|
||||
width={barThickness}
|
||||
left={xScale(ci[0])}
|
||||
data={[
|
||||
0.025,
|
||||
0.05,
|
||||
0.1,
|
||||
0.2,
|
||||
0.3,
|
||||
0.4,
|
||||
0.5,
|
||||
0.6,
|
||||
0.7,
|
||||
0.8,
|
||||
0.9,
|
||||
0.95,
|
||||
0.975,
|
||||
].map((n) => {
|
||||
let x = jStat.normal.inv(
|
||||
n,
|
||||
uplift.mean,
|
||||
uplift.stddev
|
||||
);
|
||||
const y = jStat.normal.pdf(
|
||||
x,
|
||||
uplift.mean,
|
||||
uplift.stddev
|
||||
);
|
||||
|
||||
if (uplift.dist === "lognormal") {
|
||||
x = Math.exp(x) - 1;
|
||||
}
|
||||
|
||||
return {
|
||||
x,
|
||||
y,
|
||||
};
|
||||
})}
|
||||
valueScale={xScale}
|
||||
count={(d) => d.y}
|
||||
value={(d) => d.x}
|
||||
horizontal={true}
|
||||
fill={barFill}
|
||||
fillOpacity={0.8}
|
||||
/>
|
||||
)}
|
||||
{barType === "pill" && (
|
||||
<rect
|
||||
x={xScale(ci[0])}
|
||||
y={barHeight}
|
||||
width={xScale(ci[1]) - xScale(ci[0])}
|
||||
height={barThickness}
|
||||
fill={barFill}
|
||||
rx={8}
|
||||
/>
|
||||
)}
|
||||
<Line
|
||||
fill="#000000"
|
||||
strokeWidth={3}
|
||||
stroke={expectedColor}
|
||||
stroke={"#666"}
|
||||
from={{ x: xScale(expected), y: barHeight }}
|
||||
to={{
|
||||
x: xScale(expected),
|
||||
@@ -167,64 +272,6 @@ const AlignedGraph: FC<Props> = ({
|
||||
</div>
|
||||
{!axisOnly && (
|
||||
<>
|
||||
<div className="experiment-tooltip">
|
||||
<div className="tooltip-results d-flex justify-content-center">
|
||||
{inverse ? (
|
||||
<>
|
||||
<div className="d-flex justify-content-center">
|
||||
<div className="px-1 result-text">Best case:</div>
|
||||
<div
|
||||
className={`px-1 tooltip-ci ci-worst ${
|
||||
ci[0] < 0 ? "ci-pos" : "ci-neg"
|
||||
}`}
|
||||
>
|
||||
{ci[0] > 0 && "+"}
|
||||
{parseFloat((ci[0] * 100).toFixed(2))}%
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="d-flex justify-content-center">
|
||||
<div className="px-1 result-text">Worst case:</div>
|
||||
<div
|
||||
className={`px-1 tooltip-ci ci-best ${
|
||||
ci[1] < 0 ? "ci-pos" : "ci-neg"
|
||||
}`}
|
||||
>
|
||||
{ci[1] > 0 && "+"}
|
||||
{parseFloat((ci[1] * 100).toFixed(2))}%
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
// regular, non inverse case
|
||||
<>
|
||||
<div className="d-flex justify-content-center">
|
||||
<div className="px-1 result-text">Worst case:</div>
|
||||
<div
|
||||
className={`px-1 tooltip-ci ci-worst ${
|
||||
ci[0] < 0 ? "ci-neg" : "ci-pos"
|
||||
}`}
|
||||
>
|
||||
{ci[0] > 0 && "+"}
|
||||
{parseFloat((ci[0] * 100).toFixed(2))}%
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="d-flex justify-content-center">
|
||||
<div className="px-1 result-text">Best case:</div>
|
||||
<div
|
||||
className={`px-1 tooltip-ci ci-best ${
|
||||
ci[1] < 0 ? "ci-neg" : "ci-pos"
|
||||
}`}
|
||||
>
|
||||
{ci[1] > 0 && "+"}
|
||||
{parseFloat((ci[1] * 100).toFixed(2))}%
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="expectedwrap text-right">
|
||||
<span className="expectedArrows">
|
||||
{expected > 0 ? <FaArrowUp /> : <FaArrowDown />}
|
||||
@@ -232,9 +279,6 @@ const AlignedGraph: FC<Props> = ({
|
||||
<span className="expected bold">
|
||||
{parseFloat((expected * 100).toFixed(1)) + "%"}{" "}
|
||||
</span>
|
||||
<span className="errorrange">
|
||||
± {parseFloat(((ci[1] - expected) * 100).toFixed(1))}%
|
||||
</span>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
@@ -10,6 +10,7 @@ import { formatDistance } from "date-fns";
|
||||
import { MdSwapCalls } from "react-icons/md";
|
||||
import Tooltip from "../Tooltip";
|
||||
import useConfidenceLevels from "../../hooks/useConfidenceLevels";
|
||||
import { FaQuestionCircle } from "react-icons/fa";
|
||||
|
||||
const numberFormatter = new Intl.NumberFormat();
|
||||
const percentFormatter = new Intl.NumberFormat(undefined, {
|
||||
@@ -17,12 +18,23 @@ const percentFormatter = new Intl.NumberFormat(undefined, {
|
||||
maximumFractionDigits: 2,
|
||||
});
|
||||
|
||||
function hasEnoughData(value1: number, value2: number): boolean {
|
||||
return Math.max(value1, value2) >= 150 && Math.min(value1, value2) >= 25;
|
||||
}
|
||||
|
||||
const CompactResults: FC<{
|
||||
snapshot: ExperimentSnapshotInterface;
|
||||
experiment: ExperimentInterfaceStringDates;
|
||||
}> = ({ snapshot, experiment }) => {
|
||||
barFillType?: "gradient" | "significant";
|
||||
barType?: "pill" | "violin";
|
||||
}> = ({
|
||||
snapshot,
|
||||
experiment,
|
||||
barFillType = "gradient",
|
||||
barType = "violin",
|
||||
}) => {
|
||||
const { getMetricById } = useDefinitions();
|
||||
const { ciUpper, ciLower, ciUpperDisplay } = useConfidenceLevels();
|
||||
const { ciUpper, ciLower } = useConfidenceLevels();
|
||||
|
||||
const results = snapshot.results[0];
|
||||
const variations = results?.variations || [];
|
||||
@@ -35,8 +47,7 @@ const CompactResults: FC<{
|
||||
const stats = { ...variations[i].metrics[m] };
|
||||
if (
|
||||
i > 0 &&
|
||||
stats.value >= 150 &&
|
||||
variations[0].metrics[m]?.value >= 150
|
||||
hasEnoughData(stats.value, variations[0].metrics[m]?.value || 0)
|
||||
) {
|
||||
const ci = stats.ci || [];
|
||||
if (!lowerBound || ci[0] < lowerBound) lowerBound = ci[0];
|
||||
@@ -49,6 +60,13 @@ const CompactResults: FC<{
|
||||
domain[0] = lowerBound;
|
||||
domain[1] = upperBound;
|
||||
|
||||
const hasRisk =
|
||||
Object.values(variations[1]?.metrics || {}).filter(
|
||||
(x) => x.risk?.length > 0
|
||||
).length > 0;
|
||||
|
||||
const showControlRisk: boolean = hasRisk && false;
|
||||
|
||||
return (
|
||||
<div className="mb-4 pb-4 experiment-compact-holder">
|
||||
<SRMWarning srm={results.srm} />
|
||||
@@ -60,7 +78,11 @@ const CompactResults: FC<{
|
||||
Metric
|
||||
</th>
|
||||
{experiment.variations.map((v, i) => (
|
||||
<th colSpan={i ? 3 : 1} className="value" key={i}>
|
||||
<th
|
||||
colSpan={i ? (hasRisk ? 4 : 3) : showControlRisk ? 2 : 1}
|
||||
className="value"
|
||||
key={i}
|
||||
>
|
||||
{v.name}
|
||||
</th>
|
||||
))}
|
||||
@@ -71,14 +93,37 @@ const CompactResults: FC<{
|
||||
<th className={clsx("value", `variation${i} text-center`)}>
|
||||
Value
|
||||
</th>
|
||||
{showControlRisk && i == 0 && (
|
||||
<th className={`variation${i} text-center`}>
|
||||
Risk
|
||||
<Tooltip text="How much you will lose if you choose the Control and you are wrong">
|
||||
<FaQuestionCircle />
|
||||
</Tooltip>
|
||||
</th>
|
||||
)}
|
||||
{i > 0 && (
|
||||
<th className={`variation${i} text-center`}>
|
||||
Chance to Beat Control
|
||||
</th>
|
||||
)}
|
||||
{hasRisk && i > 0 && (
|
||||
<>
|
||||
<th className={`variation${i} text-center`}>
|
||||
Risk
|
||||
<Tooltip text="How much you will lose if you choose this Variation and you are wrong">
|
||||
<FaQuestionCircle />
|
||||
</Tooltip>
|
||||
</th>
|
||||
</>
|
||||
)}
|
||||
{i > 0 && (
|
||||
<th className={`variation${i} text-center`}>
|
||||
Percent Change ({ciUpperDisplay} CI)
|
||||
Percent Change{" "}
|
||||
{barType === "violin" && hasRisk && (
|
||||
<Tooltip text="The true value is more likely to be in the thicker parts of the graph">
|
||||
<FaQuestionCircle />
|
||||
</Tooltip>
|
||||
)}
|
||||
</th>
|
||||
)}
|
||||
</React.Fragment>
|
||||
@@ -93,12 +138,15 @@ const CompactResults: FC<{
|
||||
<td className="value">
|
||||
{numberFormatter.format(variations[i]?.users || 0)}
|
||||
</td>
|
||||
{i === 0 && showControlRisk && <td className="empty-td"></td>}
|
||||
{i > 0 && (
|
||||
<>
|
||||
<td className="empty-td"></td>
|
||||
{hasRisk && <td className="empty-td"></td>}
|
||||
<td className="p-0">
|
||||
<div>
|
||||
<AlignedGraph
|
||||
id={experiment.id + "_" + i + "_axis"}
|
||||
domain={domain}
|
||||
significant={true}
|
||||
showAxis={true}
|
||||
@@ -115,17 +163,17 @@ const CompactResults: FC<{
|
||||
</tr>
|
||||
{experiment.metrics?.map((m) => {
|
||||
const metric = getMetricById(m);
|
||||
if (!variations[0]?.metrics?.[m]) {
|
||||
if (!metric || !variations[0]?.metrics?.[m]) {
|
||||
return (
|
||||
<tr
|
||||
key={m + "nodata"}
|
||||
className={`metricrow nodata ${
|
||||
metric.inverse ? "inverse" : ""
|
||||
metric?.inverse ? "inverse" : ""
|
||||
}`}
|
||||
>
|
||||
<th className="metricname">
|
||||
{metric.name}{" "}
|
||||
{metric.inverse ? (
|
||||
{metric?.name}{" "}
|
||||
{metric?.inverse ? (
|
||||
<Tooltip
|
||||
text="metric is inverse, lower is better"
|
||||
className="inverse-indicator"
|
||||
@@ -144,7 +192,7 @@ const CompactResults: FC<{
|
||||
{stats.value ? (
|
||||
<>
|
||||
<div className="result-number">
|
||||
{formatConversionRate(metric.type, stats.cr)}
|
||||
{formatConversionRate(metric?.type, stats.cr)}
|
||||
</div>
|
||||
<div>
|
||||
<small className="text-muted">
|
||||
@@ -161,13 +209,20 @@ const CompactResults: FC<{
|
||||
<em>no data</em>
|
||||
)}
|
||||
</td>
|
||||
{i > 0 && <td colSpan={2} className="variation"></td>}
|
||||
{showControlRisk && <td className="empty-td"></td>}
|
||||
{i > 0 && (
|
||||
<td
|
||||
colSpan={hasRisk ? 3 : 2}
|
||||
className="variation"
|
||||
></td>
|
||||
)}
|
||||
</React.Fragment>
|
||||
);
|
||||
})}
|
||||
</tr>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<tr
|
||||
key={m}
|
||||
@@ -192,9 +247,10 @@ const CompactResults: FC<{
|
||||
const expected = stats.expected;
|
||||
|
||||
if (
|
||||
Math.max(stats.value, variations[0].metrics[m]?.value) <
|
||||
150 ||
|
||||
Math.min(stats.value, variations[0].metrics[m]?.value) < 25
|
||||
!hasEnoughData(
|
||||
stats.value,
|
||||
variations[0].metrics[m]?.value || 0
|
||||
)
|
||||
) {
|
||||
const percentComplete = Math.min(
|
||||
Math.max(stats.value, variations[0].metrics[m]?.value) /
|
||||
@@ -240,9 +296,15 @@ const CompactResults: FC<{
|
||||
</small>
|
||||
</div>
|
||||
</td>
|
||||
{i === 0 && showControlRisk && (
|
||||
<td className="empty-td"></td>
|
||||
)}
|
||||
{i > 0 && (
|
||||
<>
|
||||
<td className="variation text-center text-muted">
|
||||
<td
|
||||
className="variation text-center text-muted"
|
||||
colSpan={hasRisk ? 2 : 1}
|
||||
>
|
||||
<div>
|
||||
<div className="badge badge-pill badge-warning">
|
||||
not enough data
|
||||
@@ -265,12 +327,13 @@ const CompactResults: FC<{
|
||||
</td>
|
||||
<td className="variation compact-graph pb-0 align-middle">
|
||||
<AlignedGraph
|
||||
id={experiment.id + "_" + i + "_" + m}
|
||||
domain={domain}
|
||||
axisOnly={true}
|
||||
ci={[0, 0]}
|
||||
significant={false}
|
||||
showAxis={false}
|
||||
height={70}
|
||||
height={62}
|
||||
inverse={!!metric.inverse}
|
||||
/>
|
||||
</td>
|
||||
@@ -282,10 +345,14 @@ const CompactResults: FC<{
|
||||
return (
|
||||
<>
|
||||
<td
|
||||
className={clsx("value", {
|
||||
className={clsx("value align-middle", {
|
||||
variation: i > 0,
|
||||
won: stats.chanceToWin > ciUpper,
|
||||
lost: stats.chanceToWin < ciLower,
|
||||
won:
|
||||
barFillType === "significant" &&
|
||||
stats.chanceToWin > ciUpper,
|
||||
lost:
|
||||
barFillType === "significant" &&
|
||||
stats.chanceToWin < ciLower,
|
||||
})}
|
||||
>
|
||||
<div className="result-number">
|
||||
@@ -302,6 +369,50 @@ const CompactResults: FC<{
|
||||
</small>
|
||||
</div>
|
||||
</td>
|
||||
{i === 0 && showControlRisk && (
|
||||
<td className={clsx("align-middle")}>
|
||||
<div className="result-number">
|
||||
{percentFormatter.format(
|
||||
Math.max(
|
||||
...variations.map((v) => {
|
||||
const data = v.metrics[m];
|
||||
if (!data || !data.risk || !data.risk.length)
|
||||
return 0;
|
||||
return metric.inverse
|
||||
? data.risk[1]
|
||||
: data.risk[0];
|
||||
})
|
||||
) / stats.cr
|
||||
)}
|
||||
</div>
|
||||
{metric.type !== "binomial" && (
|
||||
<div>
|
||||
<small className="text-muted">
|
||||
<em>
|
||||
{formatConversionRate(
|
||||
metric.type,
|
||||
Math.max(
|
||||
...variations.map((v) => {
|
||||
const data = v.metrics[m];
|
||||
if (
|
||||
!data ||
|
||||
!data.risk ||
|
||||
!data.risk.length
|
||||
)
|
||||
return 0;
|
||||
return metric.inverse
|
||||
? data.risk[1]
|
||||
: data.risk[0];
|
||||
})
|
||||
)
|
||||
)}
|
||||
/ user
|
||||
</em>
|
||||
</small>
|
||||
</div>
|
||||
)}
|
||||
</td>
|
||||
)}
|
||||
{i > 0 && (
|
||||
<td
|
||||
className={clsx(
|
||||
@@ -315,27 +426,76 @@ const CompactResults: FC<{
|
||||
{percentFormatter.format(stats.chanceToWin)}
|
||||
</td>
|
||||
)}
|
||||
{hasRisk && i > 0 && (
|
||||
<td
|
||||
className={clsx("align-middle", {
|
||||
won:
|
||||
barFillType === "significant" &&
|
||||
stats.chanceToWin > ciUpper,
|
||||
lost:
|
||||
barFillType === "significant" &&
|
||||
stats.chanceToWin < ciLower,
|
||||
})}
|
||||
>
|
||||
{(!metric.inverse && stats.risk[1] < stats.risk[0]) ||
|
||||
(metric.inverse && stats.risk[0] < stats.risk[1]) ? (
|
||||
<>
|
||||
<div className="result-number">
|
||||
{percentFormatter.format(
|
||||
(metric.inverse
|
||||
? stats.risk[0]
|
||||
: stats.risk[1]) / stats.cr
|
||||
)}
|
||||
</div>
|
||||
|
||||
{metric.type !== "binomial" && (
|
||||
<div>
|
||||
<small className="text-muted">
|
||||
<em>
|
||||
{formatConversionRate(
|
||||
metric.type,
|
||||
metric.inverse
|
||||
? stats.risk[0]
|
||||
: stats.risk[1]
|
||||
)}
|
||||
/ user
|
||||
</em>
|
||||
</small>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
""
|
||||
)}
|
||||
</td>
|
||||
)}
|
||||
{i > 0 && (
|
||||
<td
|
||||
className={clsx(
|
||||
"variation compact-graph pb-0 align-middle",
|
||||
{
|
||||
won: stats.chanceToWin > ciUpper,
|
||||
lost: stats.chanceToWin < ciLower,
|
||||
}
|
||||
)}
|
||||
className={clsx("compact-graph pb-0 align-middle", {
|
||||
variation: barFillType === "significant",
|
||||
won:
|
||||
barFillType === "significant" &&
|
||||
stats.chanceToWin > ciUpper,
|
||||
lost:
|
||||
barFillType === "significant" &&
|
||||
stats.chanceToWin < ciLower,
|
||||
})}
|
||||
>
|
||||
<div>
|
||||
<AlignedGraph
|
||||
ci={ci}
|
||||
uplift={stats.uplift}
|
||||
id={experiment.id + "_" + i + "_" + m}
|
||||
domain={domain}
|
||||
expected={expected}
|
||||
barType={barType}
|
||||
barFillType={barFillType}
|
||||
significant={
|
||||
stats.chanceToWin > ciUpper ||
|
||||
stats.chanceToWin < ciLower
|
||||
}
|
||||
showAxis={false}
|
||||
height={70}
|
||||
height={62}
|
||||
inverse={!!metric.inverse}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -5,10 +5,14 @@ import {
|
||||
SnapshotVariation,
|
||||
} from "back-end/types/experiment-snapshot";
|
||||
import { ExperimentInterfaceStringDates } from "back-end/types/experiment";
|
||||
import { MetricInterface } from "back-end/types/metric";
|
||||
import { MetricInterface, MetricStats } from "back-end/types/metric";
|
||||
import useForm from "../../hooks/useForm";
|
||||
import { useAuth } from "../../services/auth";
|
||||
import { useDefinitions } from "../../services/DefinitionsContext";
|
||||
import {
|
||||
formatConversionRate,
|
||||
getMetricConversionTitle,
|
||||
} from "../../services/metrics";
|
||||
|
||||
type SnapshotPreview = {
|
||||
srm: number;
|
||||
@@ -22,12 +26,10 @@ const ManualSnapshotForm: FC<{
|
||||
lastSnapshot?: ExperimentSnapshotInterface;
|
||||
phase: number;
|
||||
}> = ({ experiment, close, success, lastSnapshot, phase }) => {
|
||||
const { metrics } = useDefinitions();
|
||||
const { metrics, getMetricById } = useDefinitions();
|
||||
const { apiCall } = useAuth();
|
||||
|
||||
const filteredMetrics: Partial<MetricInterface>[] = [
|
||||
{ id: "users", name: "Users" },
|
||||
];
|
||||
const filteredMetrics: Partial<MetricInterface>[] = [];
|
||||
|
||||
if (metrics) {
|
||||
experiment.metrics.forEach((mid) => {
|
||||
@@ -41,18 +43,29 @@ const ManualSnapshotForm: FC<{
|
||||
filteredMetrics.map((m) => m.id).join("-");
|
||||
|
||||
const initialValue: {
|
||||
[key: string]: number[];
|
||||
} = {};
|
||||
users: number[];
|
||||
metrics: {
|
||||
[key: string]: MetricStats[];
|
||||
};
|
||||
} = { users: Array(experiment.variations.length).fill(0), metrics: {} };
|
||||
if (lastSnapshot?.results?.[0]) {
|
||||
initialValue.users = lastSnapshot.results[0].variations.map((v) => v.users);
|
||||
}
|
||||
filteredMetrics.forEach(({ id }) => {
|
||||
initialValue[id] = Array(experiment.variations.length).fill(0);
|
||||
initialValue.metrics[id] = Array(experiment.variations.length).fill({
|
||||
count: 0,
|
||||
mean: 0,
|
||||
stddev: 0,
|
||||
});
|
||||
if (lastSnapshot?.results?.[0]) {
|
||||
for (let i = 0; i < experiment.variations.length; i++) {
|
||||
const variation = lastSnapshot.results[0].variations[i];
|
||||
if (id === "users" && variation) {
|
||||
initialValue[id][i] = variation.users;
|
||||
}
|
||||
if (variation?.metrics[id]) {
|
||||
initialValue[id][i] = variation.metrics[id].value;
|
||||
initialValue.metrics[id][i] = variation.metrics[id].stats || {
|
||||
count: variation.metrics[id].value,
|
||||
mean: variation.metrics[id].value,
|
||||
stddev: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -66,6 +79,41 @@ const ManualSnapshotForm: FC<{
|
||||
});
|
||||
const [preview, setPreview] = useState<SnapshotPreview>(null);
|
||||
|
||||
function getStats() {
|
||||
const ret: { [key: string]: MetricStats[] } = {};
|
||||
Object.keys(values.metrics).forEach((key) => {
|
||||
const m = getMetricById(key);
|
||||
ret[key] = values.metrics[key].map((v, i) => {
|
||||
if (m.type === "binomial") {
|
||||
return {
|
||||
count: v.count,
|
||||
mean: 1,
|
||||
stddev: 1,
|
||||
};
|
||||
} else if (m.type === "count") {
|
||||
return {
|
||||
count: values.users[i],
|
||||
mean: v.mean,
|
||||
stddev: Math.sqrt(v.mean),
|
||||
};
|
||||
} else if (m.type === "revenue") {
|
||||
return {
|
||||
count: v.count,
|
||||
mean: v.mean,
|
||||
stddev: v.stddev,
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
count: values.users[i],
|
||||
mean: v.mean,
|
||||
stddev: v.stddev,
|
||||
};
|
||||
}
|
||||
});
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
|
||||
// Get preview stats when the value changes
|
||||
useEffect(() => {
|
||||
if (!hash) return;
|
||||
@@ -75,14 +123,18 @@ const ManualSnapshotForm: FC<{
|
||||
setPreview(null);
|
||||
return;
|
||||
}
|
||||
const metricsToTest: { [key: string]: number[] } = {};
|
||||
Object.keys(values).forEach((key) => {
|
||||
const metricsToTest: { [key: string]: MetricStats[] } = {};
|
||||
const stats = getStats();
|
||||
Object.keys(stats).forEach((key) => {
|
||||
// Only preview metrics which have all variations filled out
|
||||
if (values[key].filter((n) => n <= 0).length > 0) {
|
||||
if (
|
||||
stats[key].filter((n) => Math.min(n.count, n.mean, n.stddev) <= 0)
|
||||
.length > 0
|
||||
) {
|
||||
setPreview(null);
|
||||
return;
|
||||
}
|
||||
metricsToTest[key] = values[key];
|
||||
metricsToTest[key] = stats[key];
|
||||
});
|
||||
|
||||
// Make sure there's at least 1 metric fully entered
|
||||
@@ -94,7 +146,10 @@ const ManualSnapshotForm: FC<{
|
||||
`/experiment/${experiment.id}/snapshot/${phase}/preview`,
|
||||
{
|
||||
method: "POST",
|
||||
body: JSON.stringify(metricsToTest),
|
||||
body: JSON.stringify({
|
||||
users: values.users,
|
||||
metrics: metricsToTest,
|
||||
}),
|
||||
}
|
||||
);
|
||||
if (cancel) return;
|
||||
@@ -118,7 +173,8 @@ const ManualSnapshotForm: FC<{
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
phase,
|
||||
data: values,
|
||||
users: values.users,
|
||||
metrics: getStats(),
|
||||
}),
|
||||
}
|
||||
);
|
||||
@@ -137,89 +193,129 @@ const ManualSnapshotForm: FC<{
|
||||
>
|
||||
<p>Manually enter the latest data for the experiment below.</p>
|
||||
<div style={{ overflowY: "auto", overflowX: "hidden" }}>
|
||||
<div className="mb-3">
|
||||
<h4>Users</h4>
|
||||
<div className="row">
|
||||
{experiment.variations.map((v, i) => (
|
||||
<div className="col-auto" key={i}>
|
||||
<div className="input-group">
|
||||
<div className="input-group-prepend">
|
||||
<div className="input-group-text">{v.name}</div>
|
||||
</div>
|
||||
<input type="number" required {...inputProps.users[i]} />
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
{preview && preview.srm < 0.001 && (
|
||||
<div className="col-12">
|
||||
<div className="my-2 alert alert-danger">
|
||||
Sample Ratio Mismatch (SRM) detected. Please double check the
|
||||
number of users. If they are correct, there is likely a bug in
|
||||
the test implementation.
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
{filteredMetrics.map((m) => (
|
||||
<div className="mb-3" key={m.id}>
|
||||
<h4>{m.name}</h4>
|
||||
{m.id === "users" ? (
|
||||
<div className="row">
|
||||
<table className="table table-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th></th>
|
||||
{m.type === "binomial" ? (
|
||||
<th>Conversions</th>
|
||||
) : m.type === "count" ? (
|
||||
<th>Average Count per User</th>
|
||||
) : (
|
||||
<>
|
||||
{m.type === "revenue" && <th>Conversions</th>}
|
||||
<th>
|
||||
Average (per{" "}
|
||||
{m.type === "revenue" ? "conversion" : "user"})
|
||||
</th>
|
||||
<th>Standard Deviation</th>
|
||||
</>
|
||||
)}
|
||||
{m.type === "binomial" && (
|
||||
<th>{getMetricConversionTitle(m.type)}</th>
|
||||
)}
|
||||
<th>Chance to Beat Baseline</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{experiment.variations.map((v, i) => (
|
||||
<div className="col-auto" key={i}>
|
||||
<div className="input-group">
|
||||
<div className="input-group-prepend">
|
||||
<div className="input-group-text">{v.name}</div>
|
||||
</div>
|
||||
<input
|
||||
type="number"
|
||||
required
|
||||
{...(inputProps[m.id] ? inputProps[m.id][i] : {})}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
{preview && preview.srm < 0.001 && (
|
||||
<div className="col-12">
|
||||
<div className="my-2 alert alert-danger">
|
||||
Sample Ratio Mismatch (SRM) detected. Please double check
|
||||
the number of users. If they are correct, there is likely
|
||||
a bug in the test implementation.
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<table className="table table-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Raw Value</th>
|
||||
<th>Conversion Rate</th>
|
||||
<th>Chance to Beat Baseline</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{experiment.variations.map((v, i) => (
|
||||
<tr key={i}>
|
||||
<tr key={i}>
|
||||
<td>{v.name}</td>
|
||||
{m.type === "binomial" ? (
|
||||
<td>
|
||||
<div className="input-group">
|
||||
<div className="input-group-prepend">
|
||||
<div className="input-group-text">{v.name}</div>
|
||||
</div>
|
||||
<input
|
||||
type="number"
|
||||
required
|
||||
{...inputProps.metrics[m.id][i].count}
|
||||
/>
|
||||
</td>
|
||||
) : m.type === "count" ? (
|
||||
<td>
|
||||
<input
|
||||
type="number"
|
||||
required
|
||||
{...inputProps.metrics[m.id][i].mean}
|
||||
/>
|
||||
</td>
|
||||
) : (
|
||||
<>
|
||||
{m.type === "revenue" && (
|
||||
<td>
|
||||
<input
|
||||
type="number"
|
||||
required
|
||||
{...inputProps.metrics[m.id][i].count}
|
||||
/>
|
||||
</td>
|
||||
)}
|
||||
<td>
|
||||
<input
|
||||
type="number"
|
||||
required
|
||||
{...(inputProps[m.id] ? inputProps[m.id][i] : {})}
|
||||
{...inputProps.metrics[m.id][i].mean}
|
||||
/>
|
||||
</div>
|
||||
</td>
|
||||
</td>
|
||||
<td>
|
||||
<input
|
||||
type="number"
|
||||
required
|
||||
{...inputProps.metrics[m.id][i].stddev}
|
||||
/>
|
||||
</td>
|
||||
</>
|
||||
)}
|
||||
{m.type === "binomial" && (
|
||||
<td>
|
||||
{values.users[i] > 0 &&
|
||||
values[m.id][i] > 0 &&
|
||||
(m.type === "binomial"
|
||||
? parseFloat(
|
||||
(
|
||||
(100 * values[m.id][i]) /
|
||||
values.users[i]
|
||||
).toFixed(2)
|
||||
) + "%"
|
||||
: parseFloat(
|
||||
(values[m.id][i] / values.users[i]).toFixed(2)
|
||||
))}
|
||||
values.metrics[m.id][i].count > 0 &&
|
||||
formatConversionRate(
|
||||
m.type,
|
||||
values.metrics[m.id][i].count / values.users[i]
|
||||
)}
|
||||
</td>
|
||||
<td>
|
||||
{i > 0 &&
|
||||
preview &&
|
||||
preview.variations[i].metrics[m.id] &&
|
||||
parseFloat(
|
||||
(
|
||||
preview.variations[i].metrics[m.id].chanceToWin *
|
||||
100
|
||||
).toFixed(2)
|
||||
) + "%"}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
)}
|
||||
<td>
|
||||
{i > 0 &&
|
||||
preview &&
|
||||
preview.variations[i].metrics[m.id] &&
|
||||
parseFloat(
|
||||
(
|
||||
preview.variations[i].metrics[m.id].chanceToWin *
|
||||
100
|
||||
).toFixed(2)
|
||||
) + "%"}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
@@ -38,13 +38,15 @@ const GetStarted = ({
|
||||
const [experimentsOpen, setExperimentsOpen] = useState(false);
|
||||
const router = useRouter();
|
||||
|
||||
const hasSampleExperiment =
|
||||
experiments.filter((m) => m.id === "exp_sample").length > 0;
|
||||
const hasSampleExperiment = experiments.filter((m) =>
|
||||
m.id.match(/^exp_sample/)
|
||||
)[0];
|
||||
|
||||
const hasDataSource = datasources.length > 0;
|
||||
const hasMetrics = metrics.filter((m) => m.id !== "met_sample").length > 0;
|
||||
const hasMetrics =
|
||||
metrics.filter((m) => !m.id.match(/^met_sample/)).length > 0;
|
||||
const hasExperiments =
|
||||
experiments.filter((m) => m.id !== "exp_sample").length > 0;
|
||||
experiments.filter((m) => !m.id.match(/^exp_sample/)).length > 0;
|
||||
const currentStep = hasExperiments
|
||||
? 4
|
||||
: hasMetrics
|
||||
@@ -111,7 +113,7 @@ const GetStarted = ({
|
||||
</Tooltip>
|
||||
</div>
|
||||
{hasSampleExperiment ? (
|
||||
<Link href="/experiment/exp_sample">
|
||||
<Link href={`/experiment/${hasSampleExperiment.id}`}>
|
||||
<a className="btn btn-sm btn-success ml-3">
|
||||
View Sample Experiment <FaChevronRight />
|
||||
</a>
|
||||
@@ -121,16 +123,15 @@ const GetStarted = ({
|
||||
color="primary"
|
||||
className="btn-sm ml-3"
|
||||
onClick={async () => {
|
||||
await apiCall<{
|
||||
const res = await apiCall<{
|
||||
experiment: string;
|
||||
metric: string;
|
||||
}>(`/organization/sample-data`, {
|
||||
method: "POST",
|
||||
});
|
||||
await mutateDefinitions();
|
||||
await mutate();
|
||||
track("Add Sample Data");
|
||||
await router.push("/experiment/exp_sample");
|
||||
await router.push("/experiment/" + res.experiment);
|
||||
}}
|
||||
>
|
||||
<FaDatabase /> Import Sample Data
|
||||
|
||||
@@ -95,28 +95,24 @@ const MetricForm: FC<MetricFormProps> = ({
|
||||
display: "Binomial",
|
||||
description: "Percent of users who do something",
|
||||
sub: "click, view, download, bounce, etc.",
|
||||
tooltip: "Uses Bayesian statistics for analysis",
|
||||
},
|
||||
{
|
||||
key: "count",
|
||||
display: "Count",
|
||||
description: "Number of actions per user",
|
||||
sub: "clicks, views, downloads, etc.",
|
||||
tooltip: "Uses Bayesian statistics for analysis",
|
||||
},
|
||||
{
|
||||
key: "duration",
|
||||
display: "Duration",
|
||||
description: "How long something takes",
|
||||
sub: "time on site, loading speed, etc.",
|
||||
tooltip: "Uses bootstrapping for analysis",
|
||||
},
|
||||
{
|
||||
key: "revenue",
|
||||
display: "Revenue",
|
||||
description: "How much money a user pays (in USD)",
|
||||
sub: "revenue per visitor, average order value, etc.",
|
||||
tooltip: "Uses bootstrapping for analysis",
|
||||
},
|
||||
];
|
||||
|
||||
@@ -153,7 +149,10 @@ const MetricForm: FC<MetricFormProps> = ({
|
||||
!!currentDataSource && !["google_analytics"].includes(datasourceType);
|
||||
|
||||
const conditionsSupported = !["google_analytics"].includes(datasourceType);
|
||||
const capSupported = !["google_analytics"].includes(datasourceType);
|
||||
const capSupported =
|
||||
datasourceType && !["google_analytics"].includes(datasourceType);
|
||||
|
||||
const ignoreNullsSupported = !["google_analytics"].includes(datasourceType);
|
||||
|
||||
const supportsSQL =
|
||||
datasourceSettingsSupport && !["mixpanel"].includes(datasourceType);
|
||||
@@ -628,7 +627,7 @@ GROUP BY
|
||||
</small>
|
||||
</div>
|
||||
)}
|
||||
{capSupported && ["duration", "revenue"].includes(value.type) && (
|
||||
{ignoreNullsSupported && ["duration", "revenue"].includes(value.type) && (
|
||||
<div className="form-group">
|
||||
Converted Users Only
|
||||
<select className="form-control" {...inputs.ignoreNulls}>
|
||||
|
||||
1
packages/front-end/next-env.d.ts
vendored
1
packages/front-end/next-env.d.ts
vendored
@@ -1,2 +1,3 @@
|
||||
/// <reference types="next" />
|
||||
/// <reference types="next/types/global" />
|
||||
/// <reference types="back-end/typings/jstat" />
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
"date-fns": "^2.15.0",
|
||||
"dirty-json": "^0.9.2",
|
||||
"json-stringify-pretty-compact": "^3.0.0",
|
||||
"jstat": "^1.9.3",
|
||||
"lodash": "^4.17.15",
|
||||
"markdown-it": "^11.0.0",
|
||||
"markdown-it-sanitizer": "^0.4.3",
|
||||
|
||||
@@ -418,6 +418,12 @@ const ExperimentPage = (): ReactElement => {
|
||||
</div>
|
||||
<Tabs>
|
||||
<Tab display="Info" anchor="info">
|
||||
{experiment.id.match(/^exp_sample_/) && (
|
||||
<div className="alert alert-info">
|
||||
Click the "Results" tab above to see how the sample
|
||||
experiment performed.
|
||||
</div>
|
||||
)}
|
||||
<div className="row mb-3">
|
||||
<div className="col-md-9">
|
||||
{canEdit && !experiment.archived && (
|
||||
|
||||
@@ -31,9 +31,10 @@ export default function Home(): React.ReactElement {
|
||||
}
|
||||
|
||||
const hasDataSource = datasources.length > 0;
|
||||
const hasMetrics = metrics.filter((m) => m.id !== "met_sample").length > 0;
|
||||
const hasMetrics =
|
||||
metrics.filter((m) => !m.id.match(/^met_sample/)).length > 0;
|
||||
const hasExperiments =
|
||||
data?.experiments?.filter((e) => e.id !== "exp_sample")?.length > 0;
|
||||
data?.experiments?.filter((e) => !e.id.match(/^exp_sample/))?.length > 0;
|
||||
const isNew = !(hasMetrics && hasExperiments && hasDataSource);
|
||||
|
||||
return (
|
||||
|
||||
@@ -274,7 +274,7 @@ pre {
|
||||
position: sticky;
|
||||
left: 0;
|
||||
width: auto;
|
||||
z-index: 999;
|
||||
z-index: 890;
|
||||
border-right: 2px solid #ccc;
|
||||
}
|
||||
|
||||
|
||||
@@ -9217,6 +9217,11 @@ purgecss@^3.1.3:
|
||||
postcss "^8.2.1"
|
||||
postcss-selector-parser "^6.0.2"
|
||||
|
||||
python-shell@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/python-shell/-/python-shell-3.0.0.tgz#4eb04b6e7e8878e715b9ccd782b15194555dd074"
|
||||
integrity sha512-vlIkpJBwkhtG8d2rBbPEweg+3UXdkoduRZ0jLbIX3efYutBjTdmdmMrEQCQy9tkabH36yUjOhwTPFkH3BvoYZQ==
|
||||
|
||||
qs@6.7.0:
|
||||
version "6.7.0"
|
||||
resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc"
|
||||
|
||||
Reference in New Issue
Block a user