AI Pipelines

# 3-stage research pipeline
# Stage 1: Scrape
curl -s https://news.ycombinator.com/ | \
# Stage 2: Summarise
  openclaw run "Extract the top 10 article titles and URLs, output as JSON" | \
# Stage 3: Analyse
  openclaw run "From this JSON list of articles, identify the main tech trends and write a 150-word analysis"

#!/bin/bash
# parallel-pipeline.sh — analyse 3 data sources simultaneously
TMPDIR=$(mktemp -d)

# Run all 3 stages in parallel using & and wait
openclaw run "Summarise Q1 sales trends" --file q1-sales.csv \
  --output $TMPDIR/q1.md &
openclaw run "Summarise Q2 sales trends" --file q2-sales.csv \
  --output $TMPDIR/q2.md &
openclaw run "Summarise Q3 sales trends" --file q3-sales.csv \
  --output $TMPDIR/q3.md &

# Wait for all to complete
wait

# Stage 2: Merge results
cat $TMPDIR/q1.md $TMPDIR/q2.md $TMPDIR/q3.md | \
  openclaw run "Synthesise these quarterly summaries into an annual report with YoY comparisons" \
  --output annual-report.md

rm -rf $TMPDIR
echo "Annual report written to annual-report.md"

#!/bin/bash
# triage-tickets.sh
while IFS= read -r ticket_file; do
  priority=$(openclaw run \
    "Classify this support ticket as: critical, high, medium, or low priority. Reply with one word only." \
    --file "$ticket_file")

  case "$priority" in
    critical)
      openclaw run "Draft an urgent response and page on-call engineer" --file "$ticket_file" \
        --output "responses/urgent-$(basename $ticket_file)"
      ;;
    high|medium)
      openclaw run "Draft a helpful response" --file "$ticket_file" \
        --output "responses/$(basename $ticket_file)"
      ;;
    low)
      openclaw run "Draft a brief acknowledgement linking to the FAQ" --file "$ticket_file" \
        --output "responses/faq-$(basename $ticket_file)"
      ;;
  esac
done < <(find ./tickets/incoming -name "*.txt")

#!/bin/bash
# process-documents.sh — resumable checkpoint loop
CHECKPOINT_FILE=".pipeline-checkpoint"
FILES=($(find ./input -name "*.pdf"))
DONE=()

# Load existing checkpoint
[ -f "$CHECKPOINT_FILE" ] && mapfile -t DONE < "$CHECKPOINT_FILE"

for f in "${FILES[@]}"; do
  # Skip already processed
  [[ " ${DONE[*]} " == *" $f "* ]] && continue

  openclaw run "Extract key facts: title, author, date, main conclusions" \
    --file "$f" --format json \
    >> output.jsonl

  # Save checkpoint
  echo "$f" >> "$CHECKPOINT_FILE"
  echo "Processed: $f"
done

echo "Pipeline complete. Processed ${#FILES[@]} files."

#!/usr/bin/env bash
# Pipeline with retry logic

run_with_retry() {
  local task="$1"
  local file="$2"
  local max_retries=3
  local attempt=0

  while [ $attempt -lt $max_retries ]; do
    if openclaw run "$task" --file "$file" --format json; then
      return 0
    fi
    attempt=$((attempt + 1))
    echo "Attempt $attempt failed, retrying in 10s..."
    sleep 10
  done

  echo "ERROR: Failed after $max_retries attempts: $file" >> errors.log
  return 1
}

for f in ./documents/*.pdf; do
  run_with_retry "Extract summary" "$f" >> results.jsonl
done

#!/usr/bin/env bash
# pipeline-monitor.sh

LOG_FILE="/var/log/openclaw/pipeline-$(date +%F).log"
START_TIME=$(date +%s)
PROCESSED=0
FAILED=0

log() { echo "[$(date +%T)] $1" | tee -a "$LOG_FILE"; }

log "Pipeline started: $# files to process"

for f in "$@"; do
  if openclaw run "Summarize and extract entities" --file "$f" >> output.jsonl; then
    PROCESSED=$((PROCESSED + 1))
    log "OK: $f"
  else
    FAILED=$((FAILED + 1))
    log "FAIL: $f"
  fi
done

END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))
log "Done: $PROCESSED processed, $FAILED failed in ${DURATION}s"

# Send summary to Telegram if available
if command -v openclaw >/dev/null; then
  openclaw telegram send "Pipeline complete: $PROCESSED OK, $FAILED failed (${DURATION}s)"
fi

# Step 1: Clean raw CSV data
openclaw run "Clean this CSV: fix encoding, standardize dates to ISO 8601, remove duplicates" \
  --file raw-data.csv \
  --format csv \
  --output cleaned.csv

# Step 2: Enrich with categorization
openclaw run "Categorize each row by topic: tech, finance, health, other" \
  --file cleaned.csv \
  --format json \
  --output enriched.json

# Step 3: Generate summary report
openclaw run "Summarize the distribution across categories and highlight anomalies" \
  --file enriched.json \
  --format markdown \
  --output report.md

echo "Pipeline complete. See report.md"

#!/usr/bin/env bash
# Simple 2-step pipeline

# Step 1: Summarise a document
openclaw run "Summarise the following text in 3 bullet points:"   --file input.txt   --output summary.txt

# Step 2: Translate the summary to Spanish
openclaw run "Translate this English summary to Spanish:"   --file summary.txt   --output summary-es.txt

echo "Pipeline complete: summary-es.txt ready"

#!/usr/bin/env bash
# Run 3 analyses in parallel, then merge results

openclaw run "Analyse security vulnerabilities in this code" --file app.py > sec.txt &
openclaw run "Analyse performance bottlenecks in this code" --file app.py > perf.txt &
openclaw run "Check for code style issues in this code" --file app.py > style.txt &

wait  # Wait for all three

# Merge all findings into one report
cat sec.txt perf.txt style.txt | openclaw run "Merge these three code review reports into one prioritised list of issues." > final-review.txt
echo "Review saved to final-review.txt"