diff --git a/.env.example b/.env.example index c96e865..a0574ee 100644 --- a/.env.example +++ b/.env.example @@ -20,3 +20,10 @@ INCOMING_DIR=./incoming # Set to "true" to automatically run import.sh after watch-imports.sh stages a file AUTO_IMPORT=false + +# Polling interval in seconds — how often watch-imports.sh scans INCOMING_DIR (default: 10) +POLL_INTERVAL=10 + +# Seconds a file must be unmodified before it is processed (default: 3) +# Increase this if files are large or your network is slow +FILE_STABLE_AGE=3 diff --git a/README.md b/README.md index 8f03132..81706e9 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,8 @@ CSV files are paired with their FIDI config by base name: `checking.json` + `che - bash - curl - python3 (standard on Ubuntu) -- inotify-tools — for continuous file watching: `sudo apt-get install inotify-tools` + +> `watch-imports.sh` uses polling rather than inotify, so it works on NFS and other network filesystems. ## Setup @@ -53,14 +54,16 @@ chmod +x import.sh watch-imports.sh All config lives in `.env` (never committed to git): -| Variable | Required | Default | Description | -|---------------------|----------|---------------|----------------------------------------------------------| -| `FIDI_URL` | Yes | — | URL of your FIDI instance, e.g. `http://localhost:8080` | -| `FIDI_SECRET` | No | — | FIDI `AUTO_IMPORT_SECRET` value | -| `FIDI_ACCESS_TOKEN` | No | — | Firefly III Personal Access Token | -| `IMPORT_DIR` | No | `./imports` | Directory with JSON configs and staged CSVs | -| `INCOMING_DIR` | No | `./incoming` | Drop zone for raw CSV files | -| `AUTO_IMPORT` | No | `false` | Run `import.sh` automatically after a file is staged | +| Variable | Required | Default | Description | +|---------------------|----------|---------------|--------------------------------------------------------------------| +| `FIDI_URL` | Yes | — | URL of your FIDI instance, e.g. `http://localhost:8080` | +| `FIDI_SECRET` | No | — | FIDI `AUTO_IMPORT_SECRET` value | +| `FIDI_ACCESS_TOKEN` | No | — | Firefly III Personal Access Token | +| `IMPORT_DIR` | No | `./imports` | Directory with JSON configs and staged CSVs | +| `INCOMING_DIR` | No | `./incoming` | Drop zone for raw CSV files | +| `AUTO_IMPORT` | No | `false` | Run `import.sh` automatically after a file is staged | +| `POLL_INTERVAL` | No | `10` | Seconds between directory scans | +| `FILE_STABLE_AGE` | No | `3` | Seconds a file must be unmodified before processing (upload guard) | ## Usage diff --git a/watch-imports.sh b/watch-imports.sh index 9a9ba46..9f4c1db 100644 --- a/watch-imports.sh +++ b/watch-imports.sh @@ -1,13 +1,14 @@ #!/usr/bin/env bash -# watch-imports.sh - Watch for new CSV files, flip sign on 4th column for +# watch-imports.sh - Poll for new CSV files, flip sign on 4th column for # specific accounts, then stage them in the imports/ directory. # +# Uses polling — compatible with NFS and other network filesystems. +# # Usage: -# ./watch-imports.sh # watch continuously (requires inotify-tools) +# ./watch-imports.sh # poll continuously # ./watch-imports.sh --once # process existing files in INCOMING_DIR and exit # -# Requires: inotify-tools (sudo apt-get install inotify-tools) -# python3 (standard on Ubuntu) +# Requires: python3 (standard on Ubuntu) set -euo pipefail @@ -35,6 +36,8 @@ fi INCOMING_DIR="${INCOMING_DIR:-$SCRIPT_DIR/incoming}" IMPORT_DIR="${IMPORT_DIR:-$SCRIPT_DIR/imports}" AUTO_IMPORT="${AUTO_IMPORT:-false}" +POLL_INTERVAL="${POLL_INTERVAL:-10}" # seconds between directory scans +FILE_STABLE_AGE="${FILE_STABLE_AGE:-3}" # seconds a file must be unmodified before processing # Files whose 4th column values should have their sign flipped FLIP_FILES=( @@ -62,6 +65,17 @@ needs_flip() { return 1 } +# Returns true if the file hasn't been modified in the last FILE_STABLE_AGE seconds. +# Prevents processing a file that's still being uploaded over the network. +is_stable() { + local filepath="$1" + local mtime now age + mtime=$(stat -c%Y "$filepath" 2>/dev/null) || return 1 + now=$(date +%s) + age=$(( now - mtime )) + [[ "$age" -ge "$FILE_STABLE_AGE" ]] +} + # Flip the sign of all numeric values in the 4th column using python3. # Handles quoted CSV fields correctly. flip_fourth_column() { @@ -124,35 +138,29 @@ process_csv() { fi } +poll_once() { + while IFS= read -r filepath; do + if ! is_stable "$filepath"; then + info "Still writing: $(basename "$filepath") — will retry next poll" + continue + fi + process_csv "$filepath" + done < <(find "$INCOMING_DIR" -maxdepth 1 -name '*.csv' 2>/dev/null | sort) +} + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- mkdir -p "$INCOMING_DIR" "$IMPORT_DIR" if $ONCE_MODE; then - mapfile -t existing < <(find "$INCOMING_DIR" -maxdepth 1 -name '*.csv' | sort) - if [[ ${#existing[@]} -eq 0 ]]; then - info "No CSV files found in $INCOMING_DIR" - exit 0 - fi - for f in "${existing[@]}"; do - process_csv "$f" - done + poll_once exit 0 fi -# Continuous watch mode -if ! command -v inotifywait &>/dev/null; then - echo "inotify-tools not found. Install with:" - echo " sudo apt-get install inotify-tools" - exit 1 -fi +info "Polling $INCOMING_DIR every ${POLL_INTERVAL}s... (Ctrl+C to stop)" -info "Watching $INCOMING_DIR for new CSV files... (Ctrl+C to stop)" - -inotifywait -m -e close_write -e moved_to --format '%f' "$INCOMING_DIR" \ - | while IFS= read -r filename; do - if [[ "$filename" == *.csv ]]; then - process_csv "$INCOMING_DIR/$filename" - fi - done +while true; do + poll_once + sleep "$POLL_INTERVAL" +done