mirror of
https://github.com/humanlayer/12-factor-agents.git
synced 2025-08-20 18:59:53 +03:00
wip
This commit is contained in:
dexhorthy
114
workshops/2025-07-16/hack/debug_baml_output.md
Normal file
114
workshops/2025-07-16/hack/debug_baml_output.md
Normal file
@@ -0,0 +1,114 @@
|
||||
# BAML Output Capture in Notebooks - Debug Report
|
||||
|
||||
## Summary
|
||||
|
||||
The current implementation successfully captures BAML output in notebooks. Based on my investigation, the BAML logs are being captured correctly using the helper functions in `walkthroughgen_py.py`.
|
||||
|
||||
## Key Findings
|
||||
|
||||
### 1. BAML Logs Output to stderr
|
||||
- BAML sends all logs (prompts, responses, reasoning) to stderr by default
|
||||
- The log level is controlled by the `BAML_LOG` environment variable
|
||||
- Options: error, warn, info, debug, trace
|
||||
|
||||
### 2. Current Capture Methods
|
||||
|
||||
The workshop notebooks use two primary methods:
|
||||
|
||||
#### Method A: IPython capture_output (Recommended)
|
||||
```python
|
||||
from IPython.utils.capture import capture_output
|
||||
|
||||
def run_with_baml_logs(func, *args, **kwargs):
|
||||
"""Run a function and capture BAML logs in the notebook output."""
|
||||
with capture_output() as captured:
|
||||
result = func(*args, **kwargs)
|
||||
|
||||
# Display result
|
||||
if result is not None:
|
||||
print("=== Result ===")
|
||||
print(result)
|
||||
|
||||
# Display BAML logs from stderr
|
||||
if captured.stderr:
|
||||
print("\n=== BAML Logs ===")
|
||||
# Format logs for readability
|
||||
log_lines = captured.stderr.strip().split('\n')
|
||||
for line in log_lines:
|
||||
if 'reasoning' in line.lower():
|
||||
print(f"🤔 {line}")
|
||||
else:
|
||||
print(f" {line}")
|
||||
|
||||
return result
|
||||
```
|
||||
|
||||
#### Method B: stderr Redirection (Real-time)
|
||||
```python
|
||||
@contextlib.contextmanager
|
||||
def redirect_stderr_to_stdout():
|
||||
"""Context manager to redirect stderr to stdout."""
|
||||
old_stderr = sys.stderr
|
||||
sys.stderr = sys.stdout
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
sys.stderr = old_stderr
|
||||
|
||||
def run_with_baml_logs_redirect(func, *args, **kwargs):
|
||||
"""Run with stderr redirected to stdout for immediate display."""
|
||||
with redirect_stderr_to_stdout():
|
||||
result = func(*args, **kwargs)
|
||||
return result
|
||||
```
|
||||
|
||||
### 3. Test Results
|
||||
|
||||
From running `test_notebook_colab_sim.sh`:
|
||||
- ✅ BAML logs are successfully captured and displayed
|
||||
- ✅ Python BAML client is generated correctly
|
||||
- ✅ All notebook cells execute without errors
|
||||
- ✅ The logging helpers work in both local and Colab environments
|
||||
|
||||
### 4. Usage Pattern
|
||||
|
||||
The notebooks selectively enable logging for specific calls:
|
||||
|
||||
```python
|
||||
# Normal execution (no logs)
|
||||
main("Hello world")
|
||||
|
||||
# With log capture (when you want to see prompts/reasoning)
|
||||
run_with_baml_logs(main, "Hello world")
|
||||
```
|
||||
|
||||
### 5. Configuration in walkthrough_python.yaml
|
||||
|
||||
The YAML config uses `show_logs: true` to enable logging:
|
||||
|
||||
```yaml
|
||||
steps:
|
||||
- run_main:
|
||||
args: "Hello"
|
||||
show_logs: true # This triggers use of run_with_baml_logs()
|
||||
```
|
||||
|
||||
## Recommendations
|
||||
|
||||
1. **The current implementation is working correctly** - BAML logs are being captured
|
||||
2. **Use `run_with_baml_logs()` when you need to see prompts/reasoning** in notebooks
|
||||
3. **Set `BAML_LOG=info` for optimal verbosity** (shows prompts without too much noise)
|
||||
4. **For Colab testing, always validate with the sim script** before uploading
|
||||
|
||||
## Common Issues
|
||||
|
||||
1. **baml-cli generate failures**: Ensure baml_src directory exists and has valid BAML files
|
||||
2. **Missing logs**: Check that `BAML_LOG` environment variable is set
|
||||
3. **Import errors**: Use the `get_baml_client()` pattern to handle Colab's import cache
|
||||
|
||||
## Testing Workflow
|
||||
|
||||
1. Generate notebook: `uv run python hack/walkthroughgen_py.py hack/walkthrough_python.yaml -o hack/test.ipynb`
|
||||
2. Test locally: `cd hack && ./test_notebook_colab_sim.sh test.ipynb`
|
||||
3. Check preserved test directory in `./tmp/test_TIMESTAMP/` for debugging
|
||||
4. Upload to Colab for final validation
|
||||
215
workshops/2025-07-16/hack/test_baml_logging_demo.ipynb
Normal file
215
workshops/2025-07-16/hack/test_baml_logging_demo.ipynb
Normal file
@@ -0,0 +1,215 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# BAML Logging Demo - Testing Log Capture in Notebooks\n",
|
||||
"\n",
|
||||
"This notebook demonstrates how BAML output is captured in Jupyter notebooks."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install baml-py==0.202.0 pydantic"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import subprocess\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"from IPython.utils.capture import capture_output\n",
|
||||
"\n",
|
||||
"# Set up environment\n",
|
||||
"if 'OPENAI_API_KEY' not in os.environ:\n",
|
||||
" print(\"Warning: OPENAI_API_KEY not set\")\n",
|
||||
"\n",
|
||||
"# Set BAML logging\n",
|
||||
"os.environ['BAML_LOG'] = 'info'\n",
|
||||
"\n",
|
||||
"def baml_generate():\n",
|
||||
" try:\n",
|
||||
" result = subprocess.run(\n",
|
||||
" [\"baml-cli\", \"generate\"],\n",
|
||||
" check=True,\n",
|
||||
" capture_output=True,\n",
|
||||
" text=True\n",
|
||||
" )\n",
|
||||
" if result.stdout:\n",
|
||||
" print(\"[baml-cli generate]\\n\", result.stdout)\n",
|
||||
" if result.stderr:\n",
|
||||
" print(\"[baml-cli generate stderr]\\n\", result.stderr)\n",
|
||||
" except subprocess.CalledProcessError as e:\n",
|
||||
" print(f\"baml-cli generate failed: {e}\")\n",
|
||||
" raise\n",
|
||||
"\n",
|
||||
"def get_baml_client():\n",
|
||||
" baml_generate()\n",
|
||||
" import sys\n",
|
||||
" modules_to_delete = [key for key in sys.modules.keys() if key.startswith('baml_client')]\n",
|
||||
" for module in modules_to_delete:\n",
|
||||
" del sys.modules[module]\n",
|
||||
" import baml_client\n",
|
||||
" return baml_client.sync_client.b"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Initialize BAML\n",
|
||||
"!baml-cli init"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create a simple BAML file\n",
|
||||
"baml_content = '''class DoneForNow {\n",
|
||||
" intent \"done_for_now\"\n",
|
||||
" message string\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"function DetermineNextStep(thread string) -> DoneForNow {\n",
|
||||
" client OpenAI/gpt-4o-mini\n",
|
||||
" prompt #\"\n",
|
||||
" Given the conversation thread, determine the next step.\n",
|
||||
" \n",
|
||||
" Thread:\n",
|
||||
" {{ thread }}\n",
|
||||
" \n",
|
||||
" Respond with a message.\n",
|
||||
" \"#\n",
|
||||
"}\n",
|
||||
"'''\n",
|
||||
"\n",
|
||||
"with open('baml_src/agent.baml', 'w') as f:\n",
|
||||
" f.write(baml_content)\n",
|
||||
" \n",
|
||||
"print(\"Created agent.baml\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Helper function to capture BAML logs\n",
|
||||
"def run_with_baml_logs(func, *args, **kwargs):\n",
|
||||
" \"\"\"Run a function and capture BAML logs in the notebook output.\"\"\"\n",
|
||||
" print(f\"Running with BAML_LOG={os.environ.get('BAML_LOG')}...\")\n",
|
||||
" \n",
|
||||
" # Capture both stdout and stderr\n",
|
||||
" with capture_output() as captured:\n",
|
||||
" result = func(*args, **kwargs)\n",
|
||||
" \n",
|
||||
" # Display the result first\n",
|
||||
" if result is not None:\n",
|
||||
" print(\"=== Result ===\")\n",
|
||||
" print(result)\n",
|
||||
" \n",
|
||||
" # Display captured stdout if any\n",
|
||||
" if captured.stdout:\n",
|
||||
" print(\"\\n=== Stdout ===\")\n",
|
||||
" print(captured.stdout)\n",
|
||||
" \n",
|
||||
" # Display BAML logs from stderr\n",
|
||||
" if captured.stderr:\n",
|
||||
" print(\"\\n=== BAML Logs (from stderr) ===\")\n",
|
||||
" print(captured.stderr)\n",
|
||||
" \n",
|
||||
" return result"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Test function that uses BAML\n",
|
||||
"def test_baml_call():\n",
|
||||
" b = get_baml_client()\n",
|
||||
" thread = '[{\"type\": \"user_input\", \"data\": \"Hello, how are you?\"}]'\n",
|
||||
" result = b.DetermineNextStep(thread)\n",
|
||||
" return result"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Run without log capture\n",
|
||||
"print(\"=== Running WITHOUT log capture ===\")\n",
|
||||
"result1 = test_baml_call()\n",
|
||||
"print(f\"Result: {result1}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Run WITH log capture\n",
|
||||
"print(\"=== Running WITH log capture ===\")\n",
|
||||
"result2 = run_with_baml_logs(test_baml_call)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Test with different log levels\n",
|
||||
"print(\"\\n=== Testing with DEBUG log level ===\")\n",
|
||||
"os.environ['BAML_LOG'] = 'debug'\n",
|
||||
"result3 = run_with_baml_logs(test_baml_call)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Summary\n",
|
||||
"\n",
|
||||
"This notebook demonstrates:\n",
|
||||
"1. BAML logs are written to stderr by default\n",
|
||||
"2. Using `capture_output()` from IPython can capture these logs\n",
|
||||
"3. The `run_with_baml_logs()` helper function makes it easy to see BAML logs in notebooks\n",
|
||||
"4. Different log levels (info, debug) show different amounts of detail"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python",
|
||||
"version": "3.11.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
281
workshops/2025-07-16/hack/test_baml_output.ipynb
Normal file
281
workshops/2025-07-16/hack/test_baml_output.ipynb
Normal file
@@ -0,0 +1,281 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Minimal BAML Output Test\n",
|
||||
"\n",
|
||||
"This notebook tests different methods of capturing BAML output."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install baml-py==0.202.0 pydantic"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import subprocess\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"def baml_generate():\n",
|
||||
" result = subprocess.run(\n",
|
||||
" [\"baml-cli\", \"generate\"],\n",
|
||||
" check=True,\n",
|
||||
" capture_output=True,\n",
|
||||
" text=True\n",
|
||||
" )\n",
|
||||
" if result.stdout:\n",
|
||||
" print(\"[baml-cli generate]\\n\", result.stdout)\n",
|
||||
" if result.stderr:\n",
|
||||
" print(\"[baml-cli generate]\\n\", result.stderr)\n",
|
||||
"\n",
|
||||
"def get_baml_client():\n",
|
||||
" # Set API key\n",
|
||||
" if 'OPENAI_API_KEY' not in os.environ:\n",
|
||||
" print(\"Warning: OPENAI_API_KEY not set\")\n",
|
||||
" \n",
|
||||
" baml_generate()\n",
|
||||
" \n",
|
||||
" # Force delete all baml_client modules from sys.modules\n",
|
||||
" modules_to_delete = [key for key in sys.modules.keys() if key.startswith('baml_client')]\n",
|
||||
" for module in modules_to_delete:\n",
|
||||
" del sys.modules[module]\n",
|
||||
" \n",
|
||||
" # Now import fresh\n",
|
||||
" import baml_client\n",
|
||||
" return baml_client.sync_client.b"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!baml-cli init"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create a simple BAML function\n",
|
||||
"baml_content = '''function DemoFunction {\n",
|
||||
" input: string\n",
|
||||
" output: string\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"impl<llm, DemoFunction> DemoFunctionImpl {\n",
|
||||
" prompt #\"\n",
|
||||
" Say hello to {{input}}\n",
|
||||
" \"#\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"client<llm> MyClient {\n",
|
||||
" provider openai\n",
|
||||
" options {\n",
|
||||
" model \"gpt-4o-mini\"\n",
|
||||
" }\n",
|
||||
"}\n",
|
||||
"'''\n",
|
||||
"\n",
|
||||
"with open('baml_src/demo.baml', 'w') as f:\n",
|
||||
" f.write(baml_content)\n",
|
||||
"\n",
|
||||
"print(\"Created demo.baml\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Method 1: Direct Execution (Default)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Direct execution - BAML logs go to stderr\n",
|
||||
"os.environ['BAML_LOG'] = 'info'\n",
|
||||
"b = get_baml_client()\n",
|
||||
"result = b.DemoFunction(\"World\")\n",
|
||||
"print(f\"Result: {result}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Method 2: Capture with IPython"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from IPython.utils.capture import capture_output\n",
|
||||
"\n",
|
||||
"os.environ['BAML_LOG'] = 'info'\n",
|
||||
"print(\"Capturing output with IPython...\")\n",
|
||||
"\n",
|
||||
"with capture_output() as captured:\n",
|
||||
" b = get_baml_client()\n",
|
||||
" result = b.DemoFunction(\"IPython\")\n",
|
||||
"\n",
|
||||
"print(f\"Result: {result}\")\n",
|
||||
"print(\"\\n=== Captured stdout ===\")\n",
|
||||
"print(captured.stdout)\n",
|
||||
"print(\"\\n=== Captured stderr (BAML logs) ===\")\n",
|
||||
"print(captured.stderr)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Method 3: Redirect stderr to stdout"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import contextlib\n",
|
||||
"\n",
|
||||
"@contextlib.contextmanager\n",
|
||||
"def redirect_stderr_to_stdout():\n",
|
||||
" old_stderr = sys.stderr\n",
|
||||
" sys.stderr = sys.stdout\n",
|
||||
" try:\n",
|
||||
" yield\n",
|
||||
" finally:\n",
|
||||
" sys.stderr = old_stderr\n",
|
||||
"\n",
|
||||
"os.environ['BAML_LOG'] = 'info'\n",
|
||||
"print(\"Redirecting stderr to stdout...\")\n",
|
||||
"\n",
|
||||
"with redirect_stderr_to_stdout():\n",
|
||||
" b = get_baml_client()\n",
|
||||
" result = b.DemoFunction(\"Redirect\")\n",
|
||||
"\n",
|
||||
"print(f\"\\nResult: {result}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Method 4: Cell Magic %%capture"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture captured_output\n",
|
||||
"os.environ['BAML_LOG'] = 'info'\n",
|
||||
"b = get_baml_client()\n",
|
||||
"result = b.DemoFunction(\"Cell Magic\")\n",
|
||||
"print(f\"Result: {result}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Display captured output\n",
|
||||
"print(\"=== Captured stdout ===\")\n",
|
||||
"print(captured_output.stdout)\n",
|
||||
"print(\"\\n=== Captured stderr (BAML logs) ===\")\n",
|
||||
"print(captured_output.stderr)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Method 5: Subprocess with combined output"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create a Python script that runs BAML\n",
|
||||
"script_content = '''import os\n",
|
||||
"os.environ['BAML_LOG'] = 'info'\n",
|
||||
"os.environ['OPENAI_API_KEY'] = os.environ.get('OPENAI_API_KEY', '')\n",
|
||||
"\n",
|
||||
"import baml_client\n",
|
||||
"b = baml_client.sync_client.b\n",
|
||||
"result = b.DemoFunction(\"Subprocess\")\n",
|
||||
"print(f\"Result: {result}\")\n",
|
||||
"'''\n",
|
||||
"\n",
|
||||
"with open('test_baml_script.py', 'w') as f:\n",
|
||||
" f.write(script_content)\n",
|
||||
"\n",
|
||||
"# Run as subprocess with combined output\n",
|
||||
"result = subprocess.run(\n",
|
||||
" [sys.executable, 'test_baml_script.py'],\n",
|
||||
" capture_output=True,\n",
|
||||
" text=True,\n",
|
||||
" stderr=subprocess.STDOUT # Combine stderr into stdout\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"=== Combined output ===\")\n",
|
||||
"print(result.stdout)\n",
|
||||
"\n",
|
||||
"# Clean up\n",
|
||||
"os.remove('test_baml_script.py')"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
Reference in New Issue
Block a user