more sims

This commit is contained in:
dexhorthy
2025-07-16 15:47:50 -07:00
parent 80c1254313
commit 97af81c08b
8 changed files with 1624 additions and 702 deletions

View File

@@ -54,7 +54,6 @@
## Testing Commands
• Generate notebook: `uv run python hack/walkthroughgen_py.py hack/walkthrough_python.yaml -o hack/test.ipynb`
• Test locally: `uv run python hack/test_notebook.py hack/test.ipynb`
• Full Colab sim: `cd hack && ./test_notebook_colab_sim.sh`
• Run BAML tests: `baml-cli test` (from directory with baml_src)
@@ -64,6 +63,5 @@
`walkthrough/*.baml` - BAML files fetched from GitHub during notebook execution
`hack/walkthroughgen_py.py` - Main conversion tool
`hack/walkthrough_python.yaml` - Notebook definition with all chapters
`hack/test_notebook.py` - Local testing script (skips pip/baml-cli init)
`hack/test_notebook_colab_sim.sh` - Full Colab environment simulation
`hack/workshop_final.ipynb` - Final generated notebook ready for workshop

View File

@@ -1,279 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Testing BAML Logging in Jupyter\n",
"\n",
"This notebook demonstrates how to capture BAML logs in Jupyter notebook output."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# First, set up the environment\n",
"import os\n",
"import sys\n",
"import subprocess\n",
"import logging\n",
"\n",
"# Set BAML_LOG environment variable\n",
"os.environ['BAML_LOG'] = 'info'\n",
"print(f\"BAML_LOG is set to: {os.environ['BAML_LOG']}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Method 1: Configure Python logging to capture subprocess output\n",
"import logging\n",
"\n",
"# Configure logging for Jupyter - force reconfiguration\n",
"logging.basicConfig(\n",
" level=logging.INFO,\n",
" format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',\n",
" force=True, # This forces reconfiguration in Jupyter\n",
" handlers=[\n",
" logging.StreamHandler(sys.stdout) # Output to stdout instead of stderr\n",
" ]\n",
")\n",
"\n",
"# Get logger for BAML\n",
"logger = logging.getLogger('baml')\n",
"logger.setLevel(logging.DEBUG)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Method 2: Monkey-patch print to capture BAML output\n",
"import builtins\n",
"from datetime import datetime\n",
"\n",
"# Store original print function\n",
"_original_print = builtins.print\n",
"\n",
"def patched_print(*args, **kwargs):\n",
" \"\"\"Patched print function that adds timestamp and formatting.\"\"\"\n",
" # Check if this is a BAML log message\n",
" message = ' '.join(str(arg) for arg in args)\n",
" if '[baml' in message.lower() or 'baml' in message.lower():\n",
" timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n",
" _original_print(f\"[{timestamp}] BAML LOG: {message}\", **kwargs)\n",
" else:\n",
" _original_print(*args, **kwargs)\n",
"\n",
"# Apply the patch\n",
"builtins.print = patched_print"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Method 3: Use IPython's capture magic\n",
"from IPython.utils.capture import capture_output\n",
"\n",
"def run_with_capture(func, *args, **kwargs):\n",
" \"\"\"Run a function and capture all its output.\"\"\"\n",
" with capture_output() as captured:\n",
" result = func(*args, **kwargs)\n",
" \n",
" # Display captured output\n",
" if captured.stdout:\n",
" print(\"=== Standard Output ===\")\n",
" print(captured.stdout)\n",
" \n",
" if captured.stderr:\n",
" print(\"\\n=== Error/Log Output ===\")\n",
" print(captured.stderr)\n",
" \n",
" # Also display using IPython's display\n",
" from IPython.display import display, HTML\n",
" if captured.stderr:\n",
" display(HTML(f\"<pre style='background-color: #f0f0f0; padding: 10px;'>{captured.stderr}</pre>\"))\n",
" \n",
" return result"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Method 4: Direct stderr/stdout redirection\n",
"import io\n",
"import contextlib\n",
"\n",
"@contextlib.contextmanager\n",
"def capture_all_output():\n",
" \"\"\"Capture both stdout and stderr.\"\"\"\n",
" old_stdout = sys.stdout\n",
" old_stderr = sys.stderr\n",
" \n",
" stdout_buffer = io.StringIO()\n",
" stderr_buffer = io.StringIO()\n",
" \n",
" # Create a custom writer that writes to both the buffer and the notebook\n",
" class TeeWriter:\n",
" def __init__(self, buffer, original):\n",
" self.buffer = buffer\n",
" self.original = original\n",
" \n",
" def write(self, data):\n",
" self.buffer.write(data)\n",
" self.original.write(data)\n",
" self.original.flush() # Ensure immediate display\n",
" \n",
" def flush(self):\n",
" self.buffer.flush()\n",
" self.original.flush()\n",
" \n",
" try:\n",
" sys.stdout = TeeWriter(stdout_buffer, old_stdout)\n",
" sys.stderr = TeeWriter(stderr_buffer, old_stderr)\n",
" yield stdout_buffer, stderr_buffer\n",
" finally:\n",
" sys.stdout = old_stdout\n",
" sys.stderr = old_stderr"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Test function that would use BAML\n",
"def test_baml_function():\n",
" print(\"Starting BAML test...\")\n",
" # This would be where your main() function runs\n",
" # For now, let's simulate some BAML-like output\n",
" print(\"[BAML] Initializing client...\")\n",
" print(\"[BAML] Running DetermineNextStep...\")\n",
" print(\"[BAML] Reasoning: User wants to multiply 3 and 4\")\n",
" print(\"[BAML] Result: MultiplyTool(a=3, b=4)\")\n",
" return \"Test completed\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Test different methods\n",
"print(\"Testing BAML logging capture methods...\\n\")\n",
"\n",
"# Method 1: Using IPython capture\n",
"print(\"Method 1: IPython capture\")\n",
"result = run_with_capture(test_baml_function)\n",
"print(f\"Result: {result}\\n\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Method 2: Using context manager\n",
"print(\"Method 2: Direct output capture\")\n",
"with capture_all_output() as (stdout, stderr):\n",
" result = test_baml_function()\n",
"print(f\"Result: {result}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## For actual BAML usage\n",
"\n",
"When using actual BAML functions, the logs should appear if:\n",
"\n",
"1. `BAML_LOG` environment variable is set (we set it to 'info')\n",
"2. You're using one of the capture methods above\n",
"\n",
"Example with real BAML:\n",
"\n",
"```python\n",
"# Assuming you have your main() function from the workshop\n",
"os.environ['BAML_LOG'] = 'info'\n",
"\n",
"# Use IPython capture\n",
"with capture_output() as captured:\n",
" main(\"can you multiply 3 and 4\")\n",
"\n",
"# Display the logs\n",
"print(\"=== Output ===\")\n",
"print(captured.stdout)\n",
"print(\"\\n=== Logs ===\")\n",
"print(captured.stderr)\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Method 5: Enable verbose subprocess output\n",
"# This is useful if BAML is running as a subprocess\n",
"\n",
"def run_baml_with_logging():\n",
" # Ensure BAML_LOG is set\n",
" env = os.environ.copy()\n",
" env['BAML_LOG'] = 'info'\n",
" \n",
" # If BAML runs as subprocess, capture its output\n",
" result = subprocess.run(\n",
" ['python', '-c', 'print(\"BAML subprocess test\")'],\n",
" env=env,\n",
" capture_output=True,\n",
" text=True\n",
" )\n",
" \n",
" print(\"Subprocess stdout:\", result.stdout)\n",
" print(\"Subprocess stderr:\", result.stderr)\n",
"\n",
"run_baml_with_logging()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.0"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

View File

@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Test script to verify BAML logging capture."""
import os
import sys
from contextlib import redirect_stderr
from io import StringIO
# Set BAML log level
os.environ['BAML_LOG'] = 'info'
print("Testing BAML logging capture methods...")
print("=" * 60)
# Method 1: Using sys.stderr redirection
print("\nMethod 1: Direct stderr redirection")
old_stderr = sys.stderr
sys.stderr = sys.stdout
# Simulate BAML logging to stderr
print("This would be a BAML log message", file=old_stderr)
print("BAML logs should appear here if redirected properly")
# Restore stderr
sys.stderr = old_stderr
# Method 2: Using context manager
print("\nMethod 2: Context manager with StringIO")
stderr_capture = StringIO()
with redirect_stderr(stderr_capture):
# Simulate BAML logging
print("This is a BAML log to stderr", file=sys.stderr)
print("Another BAML log message", file=sys.stderr)
# Get captured content
captured = stderr_capture.getvalue()
if captured:
print("Captured stderr content:")
print(captured)
else:
print("No stderr content captured")
print("\n" + "=" * 60)
print("Test complete!")

File diff suppressed because it is too large Load Diff

View File

@@ -1,262 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "dbb667e8",
"metadata": {},
"source": [
"# Test ls commands"
]
},
{
"cell_type": "markdown",
"id": "99e50124",
"metadata": {},
"source": [
"Quick test of ls baml_src commands"
]
},
{
"cell_type": "markdown",
"id": "997a97f4",
"metadata": {},
"source": [
"## Test ls Commands"
]
},
{
"cell_type": "markdown",
"id": "0ddc0061",
"metadata": {},
"source": [
"Testing ls after baml_setup and fetch_file"
]
},
{
"cell_type": "markdown",
"id": "55ae0fa8",
"metadata": {},
"source": [
"Setting up BAML"
]
},
{
"cell_type": "markdown",
"id": "58e52249",
"metadata": {},
"source": [
"### BAML Setup\n",
"\n",
"Don't worry too much about this setup code - it will make sense later! For now, just know that:\n",
"- BAML is a tool for working with language models\n",
"- We need some special setup code to make it work nicely in Google Colab\n",
"- The `get_baml_client()` function will be used to interact with AI models"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "39599131",
"metadata": {},
"outputs": [],
"source": [
"!pip install baml-py==0.202.0 pydantic"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0c3f24df",
"metadata": {},
"outputs": [],
"source": [
"import subprocess\n",
"import os\n",
"\n",
"# Try to import Google Colab userdata, but don't fail if not in Colab\n",
"try:\n",
" from google.colab import userdata\n",
" IN_COLAB = True\n",
"except ImportError:\n",
" IN_COLAB = False\n",
"\n",
"def baml_generate():\n",
" try:\n",
" result = subprocess.run(\n",
" [\"baml-cli\", \"generate\"],\n",
" check=True,\n",
" capture_output=True,\n",
" text=True\n",
" )\n",
" if result.stdout:\n",
" print(\"[baml-cli generate]\\n\", result.stdout)\n",
" if result.stderr:\n",
" print(\"[baml-cli generate]\\n\", result.stderr)\n",
" except subprocess.CalledProcessError as e:\n",
" msg = (\n",
" f\"`baml-cli generate` failed with exit code {e.returncode}\\n\"\n",
" f\"--- STDOUT ---\\n{e.stdout}\\n\"\n",
" f\"--- STDERR ---\\n{e.stderr}\"\n",
" )\n",
" raise RuntimeError(msg) from None\n",
"\n",
"def get_baml_client():\n",
" \"\"\"\n",
" a bunch of fun jank to work around the google colab import cache\n",
" \"\"\"\n",
" # Set API key from Colab secrets or environment\n",
" if IN_COLAB:\n",
" os.environ['OPENAI_API_KEY'] = userdata.get('OPENAI_API_KEY')\n",
" elif 'OPENAI_API_KEY' not in os.environ:\n",
" print(\"Warning: OPENAI_API_KEY not set. Please set it in your environment.\")\n",
" \n",
" baml_generate()\n",
" \n",
" # Force delete all baml_client modules from sys.modules\n",
" import sys\n",
" modules_to_delete = [key for key in sys.modules.keys() if key.startswith('baml_client')]\n",
" for module in modules_to_delete:\n",
" del sys.modules[module]\n",
" \n",
" # Now import fresh\n",
" import baml_client\n",
" return baml_client.sync_client.b\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f08dd286",
"metadata": {},
"outputs": [],
"source": [
"!baml-cli init"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7e3d15a6",
"metadata": {},
"outputs": [],
"source": [
"# Helper function to capture BAML logs in notebook output\n",
"import os\n",
"from IPython.utils.capture import capture_output\n",
"\n",
"def run_with_baml_logs(func, *args, **kwargs):\n",
" \"\"\"Run a function and capture BAML logs in the notebook output.\"\"\"\n",
" # Capture both stdout and stderr\n",
" with capture_output() as captured:\n",
" result = func(*args, **kwargs)\n",
" \n",
" # Display the captured output\n",
" if captured.stdout:\n",
" print(captured.stdout)\n",
" if captured.stderr:\n",
" # BAML logs go to stderr - format them nicely\n",
" print(\"\\n=== BAML Logs ===\")\n",
" print(captured.stderr)\n",
" print(\"=================\\n\")\n",
" \n",
" return result\n",
"\n",
"# Set BAML log level (options: error, warn, info, debug, trace)\n",
"os.environ['BAML_LOG'] = 'info'\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b91a6ba2",
"metadata": {},
"outputs": [],
"source": [
"!ls baml_src"
]
},
{
"cell_type": "markdown",
"id": "ea85a42e",
"metadata": {},
"source": [
"After setup, we should see the default BAML files"
]
},
{
"cell_type": "markdown",
"id": "563ad350",
"metadata": {},
"source": [
"Now fetching agent.baml"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "565663a4",
"metadata": {},
"outputs": [],
"source": [
"!curl -fsSL -o baml_src/agent.baml https://raw.githubusercontent.com/humanlayer/12-factor-agents/refs/heads/main/workshops/2025-07-16/./walkthrough/01-agent.baml && cat baml_src/agent.baml"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cb9a2762",
"metadata": {},
"outputs": [],
"source": [
"!ls baml_src"
]
},
{
"cell_type": "markdown",
"id": "047a28ee",
"metadata": {},
"source": [
"Now we should see agent.baml added"
]
},
{
"cell_type": "markdown",
"id": "0687d703",
"metadata": {},
"source": [
"Fetching calculator tools"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5c9b83aa",
"metadata": {},
"outputs": [],
"source": [
"!curl -fsSL -o baml_src/tool_calculator.baml https://raw.githubusercontent.com/humanlayer/12-factor-agents/refs/heads/main/workshops/2025-07-16/./walkthrough/02-tool_calculator.baml && cat baml_src/tool_calculator.baml"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8c96c5aa",
"metadata": {},
"outputs": [],
"source": [
"!ls baml_src"
]
},
{
"cell_type": "markdown",
"id": "5ec1e510",
"metadata": {},
"source": [
"Now we should see both agent.baml and tool_calculator.baml"
]
}
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}

View File

@@ -0,0 +1,81 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Minimal BAML Logging Test"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import sys\n",
"from IPython.utils.capture import capture_output\n",
"\n",
"# Set BAML log level\n",
"os.environ['BAML_LOG'] = 'info'\n",
"\n",
"# Test direct stderr write\n",
"print(\"Testing stderr capture...\")\n",
"with capture_output() as captured:\n",
" print(\"This goes to stdout\")\n",
" print(\"This goes to stderr\", file=sys.stderr)\n",
"\n",
"print(\"Captured stdout:\", captured.stdout)\n",
"print(\"Captured stderr:\", captured.stderr)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Test with real BAML if available\n",
"try:\n",
" # Simple function that might generate BAML logs\n",
" def test_function():\n",
" print(\"Function output\")\n",
" # Simulate BAML log to stderr\n",
" print(\"[BAML INFO] Test log message\", file=sys.stderr)\n",
" return \"Result\"\n",
" \n",
" # Capture with helper\n",
" def run_with_capture(func, *args, **kwargs):\n",
" with capture_output() as captured:\n",
" result = func(*args, **kwargs)\n",
" \n",
" if result:\n",
" print(f\"Result: {result}\")\n",
" if captured.stdout:\n",
" print(f\"\\nStdout:\\n{captured.stdout}\")\n",
" if captured.stderr:\n",
" print(f\"\\nStderr (logs):\\n{captured.stderr}\")\n",
" \n",
" return result\n",
" \n",
" run_with_capture(test_function)\n",
"except Exception as e:\n",
" print(f\"Error: {e}\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python",
"version": "3.9.0"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

View File

@@ -86,27 +86,81 @@ def get_baml_client():
# Fourth cell: Add BAML logging helper
logging_helper = '''# Helper function to capture BAML logs in notebook output
import os
import sys
from IPython.utils.capture import capture_output
import contextlib
def run_with_baml_logs(func, *args, **kwargs):
"""Run a function and capture BAML logs in the notebook output."""
# Ensure BAML_LOG is set
if 'BAML_LOG' not in os.environ:
os.environ['BAML_LOG'] = 'info'
print(f"Running with BAML_LOG={os.environ.get('BAML_LOG')}...")
# Capture both stdout and stderr
with capture_output() as captured:
result = func(*args, **kwargs)
# Display the captured output
# Display the result first
if result is not None:
print("=== Result ===")
print(result)
# Display captured stdout if any
if captured.stdout:
print("\\n=== Output ===")
print(captured.stdout)
# Display BAML logs from stderr
if captured.stderr:
# BAML logs go to stderr - format them nicely
print("\\n=== BAML Logs ===")
print(captured.stderr)
print("=================\\n")
# Format the logs for better readability
log_lines = captured.stderr.strip().split('\\n')
for line in log_lines:
if 'reasoning' in line.lower() or '<reasoning>' in line:
print(f"🤔 {line}")
elif 'error' in line.lower():
print(f"{line}")
elif 'warn' in line.lower():
print(f"⚠️ {line}")
else:
print(f" {line}")
return result
# Alternative: Force stderr to stdout redirection
@contextlib.contextmanager
def redirect_stderr_to_stdout():
"""Context manager to redirect stderr to stdout."""
old_stderr = sys.stderr
sys.stderr = sys.stdout
try:
yield
finally:
sys.stderr = old_stderr
def run_with_baml_logs_redirect(func, *args, **kwargs):
"""Run a function with stderr redirected to stdout for immediate display."""
if 'BAML_LOG' not in os.environ:
os.environ['BAML_LOG'] = 'info'
print(f"Running with BAML_LOG={os.environ.get('BAML_LOG')} (stderr→stdout)...")
with redirect_stderr_to_stdout():
result = func(*args, **kwargs)
if result is not None:
print("\\n=== Result ===")
print(result)
return result
# Set BAML log level (options: error, warn, info, debug, trace)
os.environ['BAML_LOG'] = 'info'
print("BAML logging helpers loaded!")
print("- Use run_with_baml_logs() to capture and display logs after execution")
print("- Use run_with_baml_logs_redirect() to see logs in real-time as they're generated")
'''
nb.cells.append(new_code_cell(logging_helper))