firefly-importer/watch-imports.sh

#!/usr/bin/env bash
# watch-imports.sh - Poll for new CSV files, flip sign on 4th column for
# specific accounts, then stage them in the imports/ directory.
#
# Uses polling — compatible with NFS and other network filesystems.
#
# Usage:
#   ./watch-imports.sh          # poll continuously
#   ./watch-imports.sh --once   # process existing files in INCOMING_DIR and exit
#
# Requires: python3 (standard on Ubuntu)

set -euo pipefail

# ---------------------------------------------------------------------------
# Args
# ---------------------------------------------------------------------------
ONCE_MODE=false
for arg in "$@"; do
  case "$arg" in
    --once) ONCE_MODE=true ;;
    *) echo "Unknown argument: $arg"; exit 1 ;;
  esac
done

# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

if [[ -f "$SCRIPT_DIR/.env" ]]; then
  # shellcheck source=/dev/null
  source "$SCRIPT_DIR/.env"
fi

INCOMING_DIR="${INCOMING_DIR:-$SCRIPT_DIR/incoming}"
IMPORT_DIR="${IMPORT_DIR:-$SCRIPT_DIR/imports}"
AUTO_IMPORT="${AUTO_IMPORT:-false}"
POLL_INTERVAL="${POLL_INTERVAL:-10}"     # seconds between directory scans
FILE_STABLE_AGE="${FILE_STABLE_AGE:-3}" # seconds a file must be unmodified before processing

# Files whose 4th column values should have their sign flipped
FLIP_FILES=(
  "jerickdiscover.csv"
  "paigediscover.csv"
)

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m'

pass() { echo -e "${GREEN}[OK]${NC}  $*"; }
info() { echo -e "${YELLOW}[INFO]${NC} $*"; }
step() { echo -e "${CYAN}[>>]${NC}  $*"; }

needs_flip() {
  local filename="$1"
  for name in "${FLIP_FILES[@]}"; do
    [[ "$filename" == "$name" ]] && return 0
  done
  return 1
}

# Returns true if the file hasn't been modified in the last FILE_STABLE_AGE seconds.
# Prevents processing a file that's still being uploaded over the network.
is_stable() {
  local filepath="$1"
  local mtime now age
  mtime=$(stat -c%Y "$filepath" 2>/dev/null) || return 1
  now=$(date +%s)
  age=$(( now - mtime ))
  [[ "$age" -ge "$FILE_STABLE_AGE" ]]
}

# Flip the sign of all numeric values in the 4th column using python3.
# Handles quoted CSV fields correctly.
flip_fourth_column() {
  local filepath="$1"
  python3 - "$filepath" <<'PYEOF'
import csv, sys, os, tempfile

filepath = sys.argv[1]
col_idx = 3  # 4th column (0-indexed)

rows = []
with open(filepath, 'r', newline='', encoding='utf-8-sig') as f:
    rows = list(csv.reader(f))

if len(rows) < 2:
    sys.exit(0)

output = [rows[0]]
for row in rows[1:]:
    if len(row) > col_idx:
        try:
            val = float(row[col_idx])
            flipped = -val
            # Preserve integer formatting when there's no fractional part
            row[col_idx] = f"{flipped:.2f}" if flipped != int(flipped) else str(int(flipped))
        except ValueError:
            pass
    output.append(row)

# Write atomically via a temp file in the same directory
dir_ = os.path.dirname(filepath)
fd, tmp = tempfile.mkstemp(dir=dir_, suffix='.tmp')
try:
    with os.fdopen(fd, 'w', newline='', encoding='utf-8') as f:
        csv.writer(f).writerows(output)
    os.replace(tmp, filepath)
except Exception:
    os.unlink(tmp)
    raise
PYEOF
}

process_csv() {
  local src="$1"
  local filename
  filename="$(basename "$src")"
  local dest="$IMPORT_DIR/$filename"

  if needs_flip "$filename"; then
    step "Flipping 4th column: $filename"
    flip_fourth_column "$src"
  fi

  mv -f "$src" "$dest"
  pass "Staged: $filename"

  if [[ "$AUTO_IMPORT" == "true" ]]; then
    info "Running import..."
    "$SCRIPT_DIR/import.sh"
  fi
}

poll_once() {
  while IFS= read -r filepath; do
    if ! is_stable "$filepath"; then
      info "Still writing: $(basename "$filepath") — will retry next poll"
      continue
    fi
    process_csv "$filepath"
  done < <(find "$INCOMING_DIR" -maxdepth 1 -name '*.csv' 2>/dev/null | sort)
}

# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
mkdir -p "$INCOMING_DIR" "$IMPORT_DIR"

if $ONCE_MODE; then
  poll_once
  exit 0
fi

info "Polling $INCOMING_DIR every ${POLL_INTERVAL}s... (Ctrl+C to stop)"

while true; do
  poll_once
  sleep "$POLL_INTERVAL"
done