Monitoring & Logging
Overview
Application observability through logging, metrics collection, monitoring dashboards, and alerting systems.
Structured Logging
Pino Logger (Node.js)
import pino from 'pino';
// Base logger configuration
const logger = pino({
level: process.env.LOG_LEVEL || 'info',
formatters: {
level: (label) => ({ level: label }),
bindings: () => ({}), // Remove pid and hostname
},
timestamp: pino.stdTimeFunctions.isoTime,
redact: {
paths: ['password', 'token', 'authorization', '*.password', '*.token'],
censor: '[REDACTED]',
},
});
// Child logger with context
function createRequestLogger(req: Request) {
return logger.child({
requestId: req.headers['x-request-id'] || crypto.randomUUID(),
method: req.method,
path: req.path,
userAgent: req.headers['user-agent'],
userId: req.user?.id,
});
}
// Express middleware
app.use((req, res, next) => {
req.log = createRequestLogger(req);
const startTime = Date.now();
res.on('finish', () => {
const duration = Date.now() - startTime;
req.log.info({
statusCode: res.statusCode,
duration,
contentLength: res.get('content-length'),
}, 'request completed');
});
next();
});
// Usage in handlers
app.get('/api/users/:id', async (req, res) => {
req.log.info({ userId: req.params.id }, 'fetching user');
try {
const user = await getUser(req.params.id);
req.log.debug({ user: user.id }, 'user found');
res.json(user);
} catch (error) {
req.log.error({ error }, 'failed to fetch user');
res.status(500).json({ error: 'Internal error' });
}
});Log Levels
// Log level guidelines
logger.trace('Detailed debugging info'); // 10 - Very verbose
logger.debug('Debugging information'); // 20 - Debug mode only
logger.info('Normal operation events'); // 30 - Default level
logger.warn('Warning conditions'); // 40 - Potential issues
logger.error('Error conditions'); // 50 - Errors that need attention
logger.fatal('System-critical errors'); // 60 - System failure
// Contextual logging
logger.info({ orderId, userId, amount }, 'order placed');
logger.error({ error: err.message, stack: err.stack }, 'payment failed');
logger.warn({ retryCount, maxRetries }, 'retry attempt');Log Aggregation Format
{
"timestamp": "2024-01-15T10:30:00.000Z",
"level": "info",
"message": "request completed",
"service": "api",
"version": "1.2.3",
"environment": "production",
"requestId": "abc-123",
"traceId": "xyz-789",
"method": "GET",
"path": "/api/users/123",
"statusCode": 200,
"duration": 45,
"userId": "user-456"
}Metrics Collection
Prometheus Metrics
import { Counter, Histogram, Gauge, Registry, collectDefaultMetrics } from 'prom-client';
const register = new Registry();
// Collect default Node.js metrics
collectDefaultMetrics({ register });
// HTTP request metrics
const httpRequestsTotal = new Counter({
name: 'http_requests_total',
help: 'Total number of HTTP requests',
labelNames: ['method', 'path', 'status'],
registers: [register],
});
const httpRequestDuration = new Histogram({
name: 'http_request_duration_seconds',
help: 'Duration of HTTP requests in seconds',
labelNames: ['method', 'path'],
buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
registers: [register],
});
// Business metrics
const ordersTotal = new Counter({
name: 'orders_total',
help: 'Total number of orders',
labelNames: ['status', 'payment_method'],
registers: [register],
});
const activeUsers = new Gauge({
name: 'active_users',
help: 'Number of currently active users',
registers: [register],
});
const orderAmount = new Histogram({
name: 'order_amount_dollars',
help: 'Distribution of order amounts',
buckets: [10, 50, 100, 250, 500, 1000, 5000],
registers: [register],
});
// Middleware to collect metrics
app.use((req, res, next) => {
const end = httpRequestDuration.startTimer({
method: req.method,
path: req.route?.path || req.path,
});
res.on('finish', () => {
end();
httpRequestsTotal
.labels(req.method, req.route?.path || req.path, res.statusCode.toString())
.inc();
});
next();
});
// Metrics endpoint
app.get('/metrics', async (req, res) => {
res.set('Content-Type', register.contentType);
res.send(await register.metrics());
});
// Business metric usage
async function createOrder(order: Order) {
// ... create order
ordersTotal.labels(order.status, order.paymentMethod).inc();
orderAmount.observe(order.total);
}Custom Metrics Patterns
// Rate limiting metrics
const rateLimitHits = new Counter({
name: 'rate_limit_hits_total',
help: 'Number of rate limit hits',
labelNames: ['endpoint', 'user_tier'],
});
// Cache metrics
const cacheHits = new Counter({
name: 'cache_hits_total',
help: 'Number of cache hits',
labelNames: ['cache_name'],
});
const cacheMisses = new Counter({
name: 'cache_misses_total',
help: 'Number of cache misses',
labelNames: ['cache_name'],
});
// Database metrics
const dbQueryDuration = new Histogram({
name: 'db_query_duration_seconds',
help: 'Database query duration',
labelNames: ['operation', 'table'],
buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1],
});
const dbConnectionPool = new Gauge({
name: 'db_connection_pool_size',
help: 'Database connection pool size',
labelNames: ['state'], // active, idle, waiting
});
// Queue metrics
const queueSize = new Gauge({
name: 'queue_size',
help: 'Number of items in queue',
labelNames: ['queue_name'],
});
const jobDuration = new Histogram({
name: 'job_duration_seconds',
help: 'Job processing duration',
labelNames: ['job_type', 'status'],
});Alerting
Alert Rules (Prometheus)
# prometheus/alerts.yml
groups:
- name: application
rules:
# High error rate
- alert: HighErrorRate
expr: |
sum(rate(http_requests_total{status=~"5.."}[5m]))
/ sum(rate(http_requests_total[5m])) > 0.05
for: 5m
labels:
severity: critical
annotations:
summary: "High error rate detected"
description: "Error rate is {{ $value | humanizePercentage }}"
# High latency
- alert: HighLatency
expr: |
histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))
> 1
for: 5m
labels:
severity: warning
annotations:
summary: "High latency detected"
description: "95th percentile latency is {{ $value }}s"
# Service down
- alert: ServiceDown
expr: up == 0
for: 1m
labels:
severity: critical
annotations:
summary: "Service {{ $labels.instance }} is down"
# High memory usage
- alert: HighMemoryUsage
expr: |
process_resident_memory_bytes / 1024 / 1024 / 1024 > 4
for: 10m
labels:
severity: warning
annotations:
summary: "High memory usage"
description: "Memory usage is {{ $value | humanize }}GB"
- name: business
rules:
# Low order rate
- alert: LowOrderRate
expr: |
sum(rate(orders_total[1h])) < 10
for: 30m
labels:
severity: warning
annotations:
summary: "Order rate is below normal"
# Payment failures
- alert: HighPaymentFailures
expr: |
sum(rate(orders_total{status="failed"}[15m]))
/ sum(rate(orders_total[15m])) > 0.1
for: 10m
labels:
severity: critical
annotations:
summary: "High payment failure rate"PagerDuty Integration
import axios from 'axios';
interface Alert {
severity: 'critical' | 'error' | 'warning' | 'info';
summary: string;
source: string;
details?: Record<string, any>;
}
async function sendPagerDutyAlert(alert: Alert) {
const event = {
routing_key: process.env.PAGERDUTY_ROUTING_KEY,
event_action: 'trigger',
dedup_key: `${alert.source}-${alert.summary}`,
payload: {
summary: alert.summary,
severity: alert.severity,
source: alert.source,
custom_details: alert.details,
timestamp: new Date().toISOString(),
},
};
await axios.post(
'https://events.pagerduty.com/v2/enqueue',
event
);
}
// Resolve alert
async function resolvePagerDutyAlert(dedupKey: string) {
await axios.post('https://events.pagerduty.com/v2/enqueue', {
routing_key: process.env.PAGERDUTY_ROUTING_KEY,
event_action: 'resolve',
dedup_key: dedupKey,
});
}Dashboards
Grafana Dashboard JSON
{
"title": "Application Overview",
"panels": [
{
"title": "Request Rate",
"type": "graph",
"targets": [
{
"expr": "sum(rate(http_requests_total[5m])) by (status)",
"legendFormat": "{{status}}"
}
]
},
{
"title": "Latency (p95)",
"type": "graph",
"targets": [
{
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, path))",
"legendFormat": "{{path}}"
}
]
},
{
"title": "Error Rate",
"type": "stat",
"targets": [
{
"expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) / sum(rate(http_requests_total[5m])) * 100"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 1 },
{ "color": "red", "value": 5 }
]
}
}
}
},
{
"title": "Active Users",
"type": "stat",
"targets": [
{ "expr": "active_users" }
]
}
]
}Health Checks
import { Router } from 'express';
const healthRouter = Router();
// Liveness probe - is the app running?
healthRouter.get('/health/live', (req, res) => {
res.json({ status: 'ok' });
});
// Readiness probe - is the app ready to serve traffic?
healthRouter.get('/health/ready', async (req, res) => {
const checks = await Promise.allSettled([
checkDatabase(),
checkRedis(),
checkExternalApi(),
]);
const results = {
database: checks[0].status === 'fulfilled' ? 'ok' : 'error',
redis: checks[1].status === 'fulfilled' ? 'ok' : 'error',
externalApi: checks[2].status === 'fulfilled' ? 'ok' : 'error',
};
const allHealthy = Object.values(results).every(s => s === 'ok');
res.status(allHealthy ? 200 : 503).json({
status: allHealthy ? 'ok' : 'degraded',
checks: results,
timestamp: new Date().toISOString(),
});
});
async function checkDatabase() {
const start = Date.now();
await db.query('SELECT 1');
return { latency: Date.now() - start };
}
async function checkRedis() {
const start = Date.now();
await redis.ping();
return { latency: Date.now() - start };
}
async function checkExternalApi() {
const start = Date.now();
await fetch('https://api.example.com/health', { timeout: 5000 });
return { latency: Date.now() - start };
}Related Skills
- [[reliability-engineering]] - SRE practices
- [[devops-cicd]] - CI/CD monitoring
- [[cloud-platforms]] - Cloud monitoring