159 lines
4.2 KiB
Bash
159 lines
4.2 KiB
Bash
#!/usr/bin/env bash
|
|
# watch-imports.sh - Watch for new CSV files, flip sign on 4th column for
|
|
# specific accounts, then stage them in the imports/ directory.
|
|
#
|
|
# Usage:
|
|
# ./watch-imports.sh # watch continuously (requires inotify-tools)
|
|
# ./watch-imports.sh --once # process existing files in INCOMING_DIR and exit
|
|
#
|
|
# Requires: inotify-tools (sudo apt-get install inotify-tools)
|
|
# python3 (standard on Ubuntu)
|
|
|
|
set -euo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Args
|
|
# ---------------------------------------------------------------------------
|
|
ONCE_MODE=false
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--once) ONCE_MODE=true ;;
|
|
*) echo "Unknown argument: $arg"; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config
|
|
# ---------------------------------------------------------------------------
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
if [[ -f "$SCRIPT_DIR/.env" ]]; then
|
|
# shellcheck source=/dev/null
|
|
source "$SCRIPT_DIR/.env"
|
|
fi
|
|
|
|
INCOMING_DIR="${INCOMING_DIR:-$SCRIPT_DIR/incoming}"
|
|
IMPORT_DIR="${IMPORT_DIR:-$SCRIPT_DIR/imports}"
|
|
AUTO_IMPORT="${AUTO_IMPORT:-false}"
|
|
|
|
# Files whose 4th column values should have their sign flipped
|
|
FLIP_FILES=(
|
|
"jerickdiscover.csv"
|
|
"paigediscover.csv"
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m'
|
|
|
|
pass() { echo -e "${GREEN}[OK]${NC} $*"; }
|
|
info() { echo -e "${YELLOW}[INFO]${NC} $*"; }
|
|
step() { echo -e "${CYAN}[>>]${NC} $*"; }
|
|
|
|
needs_flip() {
|
|
local filename="$1"
|
|
for name in "${FLIP_FILES[@]}"; do
|
|
[[ "$filename" == "$name" ]] && return 0
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Flip the sign of all numeric values in the 4th column using python3.
|
|
# Handles quoted CSV fields correctly.
|
|
flip_fourth_column() {
|
|
local filepath="$1"
|
|
python3 - "$filepath" <<'PYEOF'
|
|
import csv, sys, os, tempfile
|
|
|
|
filepath = sys.argv[1]
|
|
col_idx = 3 # 4th column (0-indexed)
|
|
|
|
rows = []
|
|
with open(filepath, 'r', newline='', encoding='utf-8-sig') as f:
|
|
rows = list(csv.reader(f))
|
|
|
|
if len(rows) < 2:
|
|
sys.exit(0)
|
|
|
|
output = [rows[0]]
|
|
for row in rows[1:]:
|
|
if len(row) > col_idx:
|
|
try:
|
|
val = float(row[col_idx])
|
|
flipped = -val
|
|
# Preserve integer formatting when there's no fractional part
|
|
row[col_idx] = f"{flipped:.2f}" if flipped != int(flipped) else str(int(flipped))
|
|
except ValueError:
|
|
pass
|
|
output.append(row)
|
|
|
|
# Write atomically via a temp file in the same directory
|
|
dir_ = os.path.dirname(filepath)
|
|
fd, tmp = tempfile.mkstemp(dir=dir_, suffix='.tmp')
|
|
try:
|
|
with os.fdopen(fd, 'w', newline='', encoding='utf-8') as f:
|
|
csv.writer(f).writerows(output)
|
|
os.replace(tmp, filepath)
|
|
except Exception:
|
|
os.unlink(tmp)
|
|
raise
|
|
PYEOF
|
|
}
|
|
|
|
process_csv() {
|
|
local src="$1"
|
|
local filename
|
|
filename="$(basename "$src")"
|
|
local dest="$IMPORT_DIR/$filename"
|
|
|
|
if needs_flip "$filename"; then
|
|
step "Flipping 4th column: $filename"
|
|
flip_fourth_column "$src"
|
|
fi
|
|
|
|
mv -f "$src" "$dest"
|
|
pass "Staged: $filename"
|
|
|
|
if [[ "$AUTO_IMPORT" == "true" ]]; then
|
|
info "Running import..."
|
|
"$SCRIPT_DIR/import.sh"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
mkdir -p "$INCOMING_DIR" "$IMPORT_DIR"
|
|
|
|
if $ONCE_MODE; then
|
|
mapfile -t existing < <(find "$INCOMING_DIR" -maxdepth 1 -name '*.csv' | sort)
|
|
if [[ ${#existing[@]} -eq 0 ]]; then
|
|
info "No CSV files found in $INCOMING_DIR"
|
|
exit 0
|
|
fi
|
|
for f in "${existing[@]}"; do
|
|
process_csv "$f"
|
|
done
|
|
exit 0
|
|
fi
|
|
|
|
# Continuous watch mode
|
|
if ! command -v inotifywait &>/dev/null; then
|
|
echo "inotify-tools not found. Install with:"
|
|
echo " sudo apt-get install inotify-tools"
|
|
exit 1
|
|
fi
|
|
|
|
info "Watching $INCOMING_DIR for new CSV files... (Ctrl+C to stop)"
|
|
|
|
inotifywait -m -e close_write -e moved_to --format '%f' "$INCOMING_DIR" \
|
|
| while IFS= read -r filename; do
|
|
if [[ "$filename" == *.csv ]]; then
|
|
process_csv "$INCOMING_DIR/$filename"
|
|
fi
|
|
done
|