Skip to content

Nominate Logging Cheatsheet

All Nominate services log to journald for centralized, interleaved viewing.

Quick Start

# View recent logs (last 100 lines)
./nominate-logs.sh

# Follow all logs live
./nominate-logs.sh -f

# Show only errors
./nominate-logs.sh -e

nominate-logs.sh Commands

View Modes

Command Description
./nominate-logs.sh Recent logs (last 100 lines)
./nominate-logs.sh -f Follow logs live (Ctrl+C to stop)
./nominate-logs.sh -e Errors only (priority: err and above)
./nominate-logs.sh -1h Last hour
./nominate-logs.sh -today Today's logs
./nominate-logs.sh -json JSON output (for parsing)
./nominate-logs.sh -h Show help

Service Filters

Command Services Included
./nominate-logs.sh tenant tenant-manager, tenant-manager-frontend
./nominate-logs.sh workflow cbworkflow
./nominate-logs.sh tenants All tenant sites (ky04, mi20, testsite)
./nominate-logs.sh ky04 ky04-api, ky04-frontend
./nominate-logs.sh mi20 mi20-clevenger-api, mi20-clevenger-frontend
./nominate-logs.sh testsite testsite-api, testsite-frontend
./nominate-logs.sh nginx nginx

Combined Examples

# Follow tenant manager logs
./nominate-logs.sh tenant -f

# Errors from tenant sites only
./nominate-logs.sh tenants -e

# KY04 logs from last hour
./nominate-logs.sh ky04 --since "1 hour ago"

# Follow workflow with grep
./nominate-logs.sh workflow -f | grep -i error

Direct journalctl Commands

For more control, use journalctl directly.

Basic Usage

# Single service
sudo journalctl -u tenant-manager

# Multiple services (interleaved by timestamp)
sudo journalctl -u tenant-manager -u cbworkflow -u nginx

# Follow mode
sudo journalctl -u tenant-manager -f

# Last N lines
sudo journalctl -u tenant-manager -n 50

Time Filters

# Since specific time
sudo journalctl -u tenant-manager --since "2025-12-09 14:00"

# Last N minutes/hours
sudo journalctl -u tenant-manager --since "30 min ago"
sudo journalctl -u tenant-manager --since "2 hours ago"

# Time range
sudo journalctl -u tenant-manager --since "09:00" --until "12:00"

# Today only
sudo journalctl -u tenant-manager --since today

# Yesterday
sudo journalctl -u tenant-manager --since yesterday --until today

Priority Filters

Priority Flag Description
Emergency -p 0 System unusable
Alert -p 1 Immediate action needed
Critical -p 2 Critical conditions
Error -p 3 Error conditions
Warning -p 4 Warning conditions
Notice -p 5 Normal but significant
Info -p 6 Informational
Debug -p 7 Debug messages 
# Errors and above (0-3)
sudo journalctl -u tenant-manager -p err

# Warnings and above (0-4)
sudo journalctl -u tenant-manager -p warning

# Range: errors to warnings only
sudo journalctl -u tenant-manager -p err..warning

Output Formats

# Default (with metadata)
sudo journalctl -u tenant-manager

# Short (timestamp + message)
sudo journalctl -u tenant-manager -o short

# Raw message only (no metadata)
sudo journalctl -u tenant-manager -o cat

# JSON (for parsing)
sudo journalctl -u tenant-manager -o json

# JSON pretty-printed
sudo journalctl -u tenant-manager -o json-pretty

# Verbose (all fields)
sudo journalctl -u tenant-manager -o verbose

Searching & Grep

# Grep within journalctl
sudo journalctl -u tenant-manager | grep "ERROR"

# Case insensitive
sudo journalctl -u tenant-manager | grep -i "database"

# Grep with context
sudo journalctl -u tenant-manager | grep -B2 -A2 "Exception"

# journalctl native grep (faster for large logs)
sudo journalctl -u tenant-manager -g "ERROR|WARN"

All Nominate Services

Core Services

Service Port URL
tenant-manager 32201 https://tenant.nominate.ai/api
tenant-manager-frontend 32200 https://tenant.nominate.ai
cbworkflow 8234 https://workflow.nominate.ai

Tenant Deployments

Service Port URL
ky04-api 32321 https://ky04api.nominate.ai
ky04-frontend 32320 https://ky04.nominate.ai
mi20-clevenger-api 32311 https://mi20api.nominate.ai
mi20-clevenger-frontend 32310 https://mi20.nominate.ai
testsite-api 32301 https://testsiteapi.nominate.ai
testsite-frontend 32300 https://testsite.nominate.ai

Full Service List for journalctl

# All services in one command
sudo journalctl \
  -u tenant-manager \
  -u tenant-manager-frontend \
  -u cbworkflow \
  -u ky04-api \
  -u ky04-frontend \
  -u mi20-clevenger-api \
  -u mi20-clevenger-frontend \
  -u testsite-api \
  -u testsite-frontend \
  -u nginx \
  -f

Common Scenarios

Debugging a Failed Request

# 1. Check nginx access/error logs
./nominate-logs.sh nginx --since "5 min ago"

# 2. Check the specific service
./nominate-logs.sh tenant --since "5 min ago"

# 3. Look for errors
./nominate-logs.sh -e --since "5 min ago"

Service Won't Start

# Check service status
sudo systemctl status tenant-manager

# View startup logs
sudo journalctl -u tenant-manager -n 50

# Follow while restarting
sudo journalctl -u tenant-manager -f &
sudo systemctl restart tenant-manager

Finding Specific Errors

# Database errors
sudo journalctl -u tenant-manager -u ky04-api | grep -i "database\|sqlite\|duckdb"

# Authentication issues
sudo journalctl -u tenant-manager | grep -i "auth\|token\|401\|403"

# Connection errors
sudo journalctl -u nginx | grep -i "upstream\|connect\|timeout"

Performance Investigation

# Slow requests (look for timing info)
sudo journalctl -u tenant-manager --since "1 hour ago" | grep -E "[0-9]{4,}ms"

# Memory issues
sudo journalctl -u tenant-manager | grep -i "memory\|oom"

# High load times
sudo journalctl -u nginx | grep -E "request_time.*[0-9]\.[0-9]{2}"

Log Rotation & Maintenance

journald handles rotation automatically based on /etc/systemd/journald.conf.

# Check disk usage
sudo journalctl --disk-usage

# Rotate now (vacuum by time)
sudo journalctl --vacuum-time=7d

# Rotate by size
sudo journalctl --vacuum-size=1G

# View journald config
cat /etc/systemd/journald.conf

Tips & Tricks

Create Aliases

Add to ~/.bashrc:

alias nlogs='~/utils/nominate-logs.sh'
alias nfollow='~/utils/nominate-logs.sh -f'
alias nerrors='~/utils/nominate-logs.sh -e'

Watch for Specific Patterns

# Alert on errors (with sound)
./nominate-logs.sh -f | grep --line-buffered -i error | while read line; do
  echo "$line"
  # paplay /usr/share/sounds/alert.ogg  # uncomment for sound
done

Export Logs

# Export to file
sudo journalctl -u tenant-manager --since today > tenant-logs-$(date +%F).txt

# Export as JSON
sudo journalctl -u tenant-manager -o json --since today > tenant-logs-$(date +%F).json

Boot-Specific Logs

# Current boot only
sudo journalctl -u tenant-manager -b

# Previous boot
sudo journalctl -u tenant-manager -b -1

# List boots
sudo journalctl --list-boots