From 7e17ec1fea5809308214d04cbeef760fa127041d Mon Sep 17 00:00:00 2001 From: jerick Date: Fri, 6 Mar 2026 20:49:37 -0500 Subject: [PATCH] New import methodology, done from static directory mapped in FireFly --- .env.example | 5 +++ README.md | 44 +++++++++++++++++--- import.sh | 112 ++++++++++++++------------------------------------- 3 files changed, 75 insertions(+), 86 deletions(-) diff --git a/.env.example b/.env.example index a0574ee..7234b1b 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,11 @@ FIDI_ACCESS_TOKEN= # JSON configs live here permanently; processed CSVs are staged here by watch-imports.sh IMPORT_DIR=./imports +# Path to the imports directory as seen by FIDI *inside its container* (default: /imports) +# Must match the volume mount target in docker-compose.yml and be listed in +# IMPORT_DIR_WHITELIST in your .fidi.env +FIDI_IMPORT_DIR=/imports + # Directory where raw CSV files are dropped (default: ./incoming) # watch-imports.sh monitors this and moves files to IMPORT_DIR after processing INCOMING_DIR=./incoming diff --git a/README.md b/README.md index 81706e9..128ce59 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,15 @@ imports/ checking.csv ↓ import.sh - ↓ (POST each JSON+CSV pair to FIDI /autoimport) + ↓ (POST to FIDI /autoimport — FIDI reads the directory itself) Firefly III ✓ ``` CSV files are paired with their FIDI config by base name: `checking.json` + `checking.csv`. +FIDI scans the mounted import directory and imports all JSON+CSV pairs it finds. + ## Requirements - bash @@ -50,16 +52,48 @@ chmod +x import.sh watch-imports.sh # e.g.: imports/checking.json, imports/jerickdiscover.json ``` +## FIDI Docker configuration + +FIDI must be able to read the `imports/` directory. Mount it as a volume and whitelist it: + +**docker-compose.yml:** +```yaml +fidi: + image: fireflyiii/data-importer:latest + restart: always + env_file: .fidi.env + volumes: + - /path/to/.fidi.env:/var/www/html/.env:ro + - /var/vault/Finances/firefly-importer/imports:/imports:ro + ports: + - 8082:8080 +``` + +**.fidi.env** (add these lines): +``` +CAN_POST_AUTOIMPORT=true +AUTO_IMPORT_SECRET= +IMPORT_DIR_WHITELIST=/imports +``` + +> **Note:** PHP-FPM does not inherit Docker container environment variables. The `.fidi.env` file must be volume-mounted as `/var/www/html/.env` inside the container (in addition to being referenced by `env_file`). + +After updating docker-compose.yml, restart FIDI: +```bash +docker compose up -d fidi +``` + ## Configuration All config lives in `.env` (never committed to git): | Variable | Required | Default | Description | -|---------------------|----------|---------------|--------------------------------------------------------------------| +|---------------------|----------|---------------|-----------------------------------------------------------------------| | `FIDI_URL` | Yes | — | URL of your FIDI instance, e.g. `http://localhost:8080` | | `FIDI_SECRET` | No | — | FIDI `AUTO_IMPORT_SECRET` value | | `FIDI_ACCESS_TOKEN` | No | — | Firefly III Personal Access Token | -| `IMPORT_DIR` | No | `./imports` | Directory with JSON configs and staged CSVs | +| `FIDI_IMPORT_DIR` | No | `/imports` | Path to imports dir *inside the FIDI container* (must match volume mount target and `IMPORT_DIR_WHITELIST`) | +| `IMPORT_DIR` | No | `./imports` | Local directory with JSON configs and staged CSVs | | `INCOMING_DIR` | No | `./incoming` | Drop zone for raw CSV files | | `AUTO_IMPORT` | No | `false` | Run `import.sh` automatically after a file is staged | | `POLL_INTERVAL` | No | `10` | Seconds between directory scans | @@ -85,7 +119,7 @@ Useful for batch runs or cron jobs: ### Run the importer -Posts all staged JSON+CSV pairs to FIDI: +Tells FIDI to import all staged JSON+CSV pairs from the mounted directory: ```bash ./import.sh @@ -138,7 +172,7 @@ sudo journalctl -u firefly-importer -f ``` firefly-importer/ -├── import.sh # Batch importer — POSTs JSON+CSV pairs to FIDI +├── import.sh # Batch importer — triggers FIDI /autoimport for the mounted directory ├── watch-imports.sh # File watcher — processes and stages incoming CSVs ├── firefly-importer.service # systemd unit file ├── .env.example # Config template diff --git a/import.sh b/import.sh index 6318feb..556ce14 100644 --- a/import.sh +++ b/import.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# Firefly III Data Importer - batch auto-import -# Pairs .json + .csv files by base name and POSTs each to FIDI /autoimport +# Firefly III Data Importer - trigger FIDI directory auto-import +# POSTs to FIDI /autoimport with the container-side import directory path. +# FIDI scans that directory for JSON+CSV pairs and imports them all. # # Usage: ./import.sh [--dry-run|-n] @@ -30,7 +31,8 @@ fi FIDI_URL="${FIDI_URL:?Set FIDI_URL in .env or environment (e.g. http://localhost:8080)}" FIDI_SECRET="${FIDI_SECRET:-}" FIDI_ACCESS_TOKEN="${FIDI_ACCESS_TOKEN:-}" -IMPORT_DIR="${IMPORT_DIR:-$SCRIPT_DIR/imports}" +# Path to the imports directory as seen by FIDI inside its container +FIDI_IMPORT_DIR="${FIDI_IMPORT_DIR:-/imports}" # --------------------------------------------------------------------------- # Helpers @@ -48,95 +50,43 @@ dry() { echo -e "${CYAN}[DRY]${NC} $*"; } build_url() { local url="${FIDI_URL%/}/autoimport" - [[ -n "$FIDI_SECRET" ]] && url="${url}?secret=${FIDI_SECRET}" - echo "$url" -} - -import_pair() { - local json_file="$1" - local csv_file="$2" - local base - base="$(basename "$json_file" .json)" - local url - url="$(build_url)" - - if $DRY_RUN; then - dry "$base" - dry " POST $url" - dry " csv → $csv_file" - dry " json → $json_file" - [[ -n "$FIDI_ACCESS_TOKEN" ]] && dry " auth → Bearer ***" - return 0 - fi - - info "Importing: $base" - - # Build auth args as an array to safely handle spaces/special chars - local curl_args=(-s -w "\n%{http_code}") - [[ -n "$FIDI_ACCESS_TOKEN" ]] && curl_args+=(-H "Authorization: Bearer $FIDI_ACCESS_TOKEN") - curl_args+=( - -F "csv=@${csv_file};type=text/csv" - -F "json=@${json_file};type=application/json" - "$url" - ) - - local response http_code body - response=$(curl "${curl_args[@]}") - http_code=$(echo "$response" | tail -n1) - body=$(echo "$response" | head -n -1) - - if [[ "$http_code" =~ ^2 ]]; then - pass "$base (HTTP $http_code)" - return 0 - else - fail "$base (HTTP $http_code)" - echo " Response: $body" - return 1 - fi + local qs="directory=${FIDI_IMPORT_DIR}" + [[ -n "$FIDI_SECRET" ]] && qs="${qs}&secret=${FIDI_SECRET}" + echo "${url}?${qs}" } # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- -if [[ ! -d "$IMPORT_DIR" ]]; then - echo "Import directory not found: $IMPORT_DIR" - exit 1 -fi +url="$(build_url)" -$DRY_RUN && info "Dry-run mode — no requests will be sent" - -mapfile -t json_files < <(find "$IMPORT_DIR" -maxdepth 1 -name '*.json' | sort) - -if [[ ${#json_files[@]} -eq 0 ]]; then - info "No .json files found in $IMPORT_DIR" +if $DRY_RUN; then + info "Dry-run mode — no requests will be sent" + dry "POST ${url//secret=*/secret=***}" + [[ -n "$FIDI_ACCESS_TOKEN" ]] && dry "auth → Bearer ***" exit 0 fi -success=0 -skipped=0 -failed=0 +info "Triggering FIDI auto-import from ${FIDI_IMPORT_DIR}..." -for json_file in "${json_files[@]}"; do - base="$(basename "$json_file" .json)" - csv_file="${IMPORT_DIR}/${base}.csv" +curl_args=(-s -w "\n%{http_code}" -X POST) +[[ -n "$FIDI_ACCESS_TOKEN" ]] && curl_args+=(-H "Authorization: Bearer $FIDI_ACCESS_TOKEN") +curl_args+=("$url") - if [[ ! -f "$csv_file" ]]; then - info "Skipping $base — no matching .csv found" - ((skipped++)) - continue - fi - - if import_pair "$json_file" "$csv_file"; then - ((success++)) - else - ((failed++)) - fi -done +response=$(curl "${curl_args[@]}") +http_code=$(echo "$response" | tail -n1) +body=$(echo "$response" | head -n -1) echo "" echo "----------------------------------------" -$DRY_RUN && echo " (dry run — nothing was imported)" -echo " Done: ${success} succeeded, ${failed} failed, ${skipped} skipped" -echo "----------------------------------------" - -[[ "$failed" -eq 0 ]] +if [[ "$http_code" =~ ^2 ]]; then + pass "Import triggered (HTTP $http_code)" + echo " Response: $body" + echo "----------------------------------------" + exit 0 +else + fail "Import failed (HTTP $http_code)" + echo " Response: $body" + echo "----------------------------------------" + exit 1 +fi