New import methodology, done from static directory mapped in FireFly

.env.example

@@ -14,6 +14,11 @@ FIDI_ACCESS_TOKEN=
 # JSON configs live here permanently; processed CSVs are staged here by watch-imports.sh
 IMPORT_DIR=./imports
 
+# Path to the imports directory as seen by FIDI *inside its container* (default: /imports)
+# Must match the volume mount target in docker-compose.yml and be listed in
+# IMPORT_DIR_WHITELIST in your .fidi.env
+FIDI_IMPORT_DIR=/imports
+
 # Directory where raw CSV files are dropped (default: ./incoming)
 # watch-imports.sh monitors this and moves files to IMPORT_DIR after processing
 INCOMING_DIR=./incoming

README.md

@@ -19,13 +19,15 @@ imports/
   checking.csv
 
         ↓  import.sh
-        ↓  (POST each JSON+CSV pair to FIDI /autoimport)
+        ↓  (POST to FIDI /autoimport — FIDI reads the directory itself)
 
 Firefly III ✓
 ```
 
 CSV files are paired with their FIDI config by base name: `checking.json` + `checking.csv`.
 
+FIDI scans the mounted import directory and imports all JSON+CSV pairs it finds.
+
 ## Requirements
 
 - bash
@@ -50,16 +52,48 @@ chmod +x import.sh watch-imports.sh
 # e.g.: imports/checking.json, imports/jerickdiscover.json
 ```
 
+## FIDI Docker configuration
+
+FIDI must be able to read the `imports/` directory. Mount it as a volume and whitelist it:
+
+**docker-compose.yml:**
+```yaml
+fidi:
+  image: fireflyiii/data-importer:latest
+  restart: always
+  env_file: .fidi.env
+  volumes:
+    - /path/to/.fidi.env:/var/www/html/.env:ro
+    - /var/vault/Finances/firefly-importer/imports:/imports:ro
+  ports:
+    - 8082:8080
+```
+
+**.fidi.env** (add these lines):
+```
+CAN_POST_AUTOIMPORT=true
+AUTO_IMPORT_SECRET=<your-secret>
+IMPORT_DIR_WHITELIST=/imports
+```
+
+> **Note:** PHP-FPM does not inherit Docker container environment variables. The `.fidi.env` file must be volume-mounted as `/var/www/html/.env` inside the container (in addition to being referenced by `env_file`).
+
+After updating docker-compose.yml, restart FIDI:
+```bash
+docker compose up -d fidi
+```
+
 ## Configuration
 
 All config lives in `.env` (never committed to git):
 
 | Variable            | Required | Default       | Description                                                        |
-|---------------------|----------|---------------|--------------------------------------------------------------------|
+|---------------------|----------|---------------|-----------------------------------------------------------------------|
 | `FIDI_URL`          | Yes      | —             | URL of your FIDI instance, e.g. `http://localhost:8080`            |
 | `FIDI_SECRET`       | No       | —             | FIDI `AUTO_IMPORT_SECRET` value                                    |
 | `FIDI_ACCESS_TOKEN` | No       | —             | Firefly III Personal Access Token                                  |
-| `IMPORT_DIR`        | No       | `./imports`   | Directory with JSON configs and staged CSVs                        |
+| `FIDI_IMPORT_DIR`   | No       | `/imports`    | Path to imports dir *inside the FIDI container* (must match volume mount target and `IMPORT_DIR_WHITELIST`) |
+| `IMPORT_DIR`        | No       | `./imports`   | Local directory with JSON configs and staged CSVs                  |
 | `INCOMING_DIR`      | No       | `./incoming`  | Drop zone for raw CSV files                                        |
 | `AUTO_IMPORT`       | No       | `false`       | Run `import.sh` automatically after a file is staged               |
 | `POLL_INTERVAL`     | No       | `10`          | Seconds between directory scans                                    |
@@ -85,7 +119,7 @@ Useful for batch runs or cron jobs:
 
 ### Run the importer
 
-Posts all staged JSON+CSV pairs to FIDI:
+Tells FIDI to import all staged JSON+CSV pairs from the mounted directory:
 
 ```bash
 ./import.sh
@@ -138,7 +172,7 @@ sudo journalctl -u firefly-importer -f
 
 ```
 firefly-importer/
-├── import.sh                   # Batch importer — POSTs JSON+CSV pairs to FIDI
+├── import.sh                   # Batch importer — triggers FIDI /autoimport for the mounted directory
 ├── watch-imports.sh            # File watcher — processes and stages incoming CSVs
 ├── firefly-importer.service    # systemd unit file
 ├── .env.example                # Config template

import.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
-# Firefly III Data Importer - batch auto-import
+# Firefly III Data Importer - trigger FIDI directory auto-import
-# Pairs .json + .csv files by base name and POSTs each to FIDI /autoimport
+# POSTs to FIDI /autoimport with the container-side import directory path.
+# FIDI scans that directory for JSON+CSV pairs and imports them all.
 #
 # Usage: ./import.sh [--dry-run|-n]
 
@@ -30,7 +31,8 @@ fi
 FIDI_URL="${FIDI_URL:?Set FIDI_URL in .env or environment (e.g. http://localhost:8080)}"
 FIDI_SECRET="${FIDI_SECRET:-}"
 FIDI_ACCESS_TOKEN="${FIDI_ACCESS_TOKEN:-}"
-IMPORT_DIR="${IMPORT_DIR:-$SCRIPT_DIR/imports}"
+# Path to the imports directory as seen by FIDI inside its container
+FIDI_IMPORT_DIR="${FIDI_IMPORT_DIR:-/imports}"
 
 # ---------------------------------------------------------------------------
 # Helpers
@@ -48,95 +50,43 @@ dry()  { echo -e "${CYAN}[DRY]${NC}  $*"; }
 
 build_url() {
   local url="${FIDI_URL%/}/autoimport"
-  [[ -n "$FIDI_SECRET" ]] && url="${url}?secret=${FIDI_SECRET}"
+  local qs="directory=${FIDI_IMPORT_DIR}"
-  echo "$url"
+  [[ -n "$FIDI_SECRET" ]] && qs="${qs}&secret=${FIDI_SECRET}"
-}
+  echo "${url}?${qs}"
-
-import_pair() {
-  local json_file="$1"
-  local csv_file="$2"
-  local base
-  base="$(basename "$json_file" .json)"
-  local url
-  url="$(build_url)"
-
-  if $DRY_RUN; then
-    dry "$base"
-    dry "  POST $url"
-    dry "  csv  → $csv_file"
-    dry "  json → $json_file"
-    [[ -n "$FIDI_ACCESS_TOKEN" ]] && dry "  auth → Bearer ***"
-    return 0
-  fi
-
-  info "Importing: $base"
-
-  # Build auth args as an array to safely handle spaces/special chars
-  local curl_args=(-s -w "\n%{http_code}")
-  [[ -n "$FIDI_ACCESS_TOKEN" ]] && curl_args+=(-H "Authorization: Bearer $FIDI_ACCESS_TOKEN")
-  curl_args+=(
-    -F "csv=@${csv_file};type=text/csv"
-    -F "json=@${json_file};type=application/json"
-    "$url"
-  )
-
-  local response http_code body
-  response=$(curl "${curl_args[@]}")
-  http_code=$(echo "$response" | tail -n1)
-  body=$(echo "$response" | head -n -1)
-
-  if [[ "$http_code" =~ ^2 ]]; then
-    pass "$base (HTTP $http_code)"
-    return 0
-  else
-    fail "$base (HTTP $http_code)"
-    echo "     Response: $body"
-    return 1
-  fi
 }
 
 # ---------------------------------------------------------------------------
 # Main
 # ---------------------------------------------------------------------------
-if [[ ! -d "$IMPORT_DIR" ]]; then
+url="$(build_url)"
-  echo "Import directory not found: $IMPORT_DIR"
-  exit 1
-fi
 
-$DRY_RUN && info "Dry-run mode — no requests will be sent"
+if $DRY_RUN; then
-
+  info "Dry-run mode — no requests will be sent"
-mapfile -t json_files < <(find "$IMPORT_DIR" -maxdepth 1 -name '*.json' | sort)
+  dry "POST ${url//secret=*/secret=***}"
-
+  [[ -n "$FIDI_ACCESS_TOKEN" ]] && dry "auth → Bearer ***"
-if [[ ${#json_files[@]} -eq 0 ]]; then
-  info "No .json files found in $IMPORT_DIR"
   exit 0
 fi
 
-success=0
+info "Triggering FIDI auto-import from ${FIDI_IMPORT_DIR}..."
-skipped=0
-failed=0
 
-for json_file in "${json_files[@]}"; do
+curl_args=(-s -w "\n%{http_code}" -X POST)
-  base="$(basename "$json_file" .json)"
+[[ -n "$FIDI_ACCESS_TOKEN" ]] && curl_args+=(-H "Authorization: Bearer $FIDI_ACCESS_TOKEN")
-  csv_file="${IMPORT_DIR}/${base}.csv"
+curl_args+=("$url")
 
-  if [[ ! -f "$csv_file" ]]; then
+response=$(curl "${curl_args[@]}")
-    info "Skipping $base — no matching .csv found"
+http_code=$(echo "$response" | tail -n1)
-    ((skipped++))
+body=$(echo "$response" | head -n -1)
-    continue
-  fi
-
-  if import_pair "$json_file" "$csv_file"; then
-    ((success++))
-  else
-    ((failed++))
-  fi
-done
 
 echo ""
 echo "----------------------------------------"
-$DRY_RUN && echo "  (dry run — nothing was imported)"
+if [[ "$http_code" =~ ^2 ]]; then
-echo "  Done: ${success} succeeded, ${failed} failed, ${skipped} skipped"
+  pass "Import triggered (HTTP $http_code)"
-echo "----------------------------------------"
+  echo "  Response: $body"
-
+  echo "----------------------------------------"
-[[ "$failed" -eq 0 ]]
+  exit 0
+else
+  fail "Import failed (HTTP $http_code)"
+  echo "  Response: $body"
+  echo "----------------------------------------"
+  exit 1
+fi