jerick/firefly-importer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New import methodology, done from static directory mapped in FireFly

Browse Source
This commit is contained in:
jerick
2026-03-06 20:49:37 -05:00
parent 0c39445d71
commit 7e17ec1fea
3 changed files with 75 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.env.example
View File

@@ -14,6 +14,11 @@ FIDI_ACCESS_TOKEN=
# JSON configs live here permanently; processed CSVs are staged here by watch-imports.sh
IMPORT_DIR=./imports
# Path to the imports directory as seen by FIDI *inside its container* (default: /imports)
# Must match the volume mount target in docker-compose.yml and be listed in
# IMPORT_DIR_WHITELIST in your .fidi.env
FIDI_IMPORT_DIR=/imports
# Directory where raw CSV files are dropped (default: ./incoming)
# watch-imports.sh monitors this and moves files to IMPORT_DIR after processing
INCOMING_DIR=./incoming

44
README.md
View File

@@ -19,13 +19,15 @@ imports/
checking.csv
↓ import.sh
↓ (POST each JSON+CSV pair to FIDI /autoimport)
↓ (POST to FIDI /autoimport — FIDI reads the directory itself)
Firefly III ✓
```
CSV files are paired with their FIDI config by base name: `checking.json` + `checking.csv`.
FIDI scans the mounted import directory and imports all JSON+CSV pairs it finds.
## Requirements
- bash
@@ -50,16 +52,48 @@ chmod +x import.sh watch-imports.sh
# e.g.: imports/checking.json, imports/jerickdiscover.json
```
## FIDI Docker configuration
FIDI must be able to read the `imports/` directory. Mount it as a volume and whitelist it:
**docker-compose.yml:**
```yaml
fidi:
image: fireflyiii/data-importer:latest
restart: always
env_file: .fidi.env
volumes:
- /path/to/.fidi.env:/var/www/html/.env:ro
- /var/vault/Finances/firefly-importer/imports:/imports:ro
ports:
- 8082:8080
```
**.fidi.env** (add these lines):
```
CAN_POST_AUTOIMPORT=true
AUTO_IMPORT_SECRET=<your-secret>
IMPORT_DIR_WHITELIST=/imports
```
> **Note:** PHP-FPM does not inherit Docker container environment variables. The `.fidi.env` file must be volume-mounted as `/var/www/html/.env` inside the container (in addition to being referenced by `env_file`).
After updating docker-compose.yml, restart FIDI:
```bash
docker compose up -d fidi
```
## Configuration
All config lives in `.env` (never committed to git):
| Variable | Required | Default | Description |
|---------------------|----------|---------------|--------------------------------------------------------------------|
|---------------------|----------|---------------|-----------------------------------------------------------------------|
| `FIDI_URL` | Yes | — | URL of your FIDI instance, e.g. `http://localhost:8080` |
| `FIDI_SECRET` | No | — | FIDI `AUTO_IMPORT_SECRET` value |
| `FIDI_ACCESS_TOKEN` | No | — | Firefly III Personal Access Token |
| `IMPORT_DIR` | No | `./imports` | Directory with JSON configs and staged CSVs |
| `FIDI_IMPORT_DIR` | No | `/imports` | Path to imports dir *inside the FIDI container* (must match volume mount target and `IMPORT_DIR_WHITELIST`) |
| `IMPORT_DIR` | No | `./imports` | Local directory with JSON configs and staged CSVs |
| `INCOMING_DIR` | No | `./incoming` | Drop zone for raw CSV files |
| `AUTO_IMPORT` | No | `false` | Run `import.sh` automatically after a file is staged |
| `POLL_INTERVAL` | No | `10` | Seconds between directory scans |
@@ -85,7 +119,7 @@ Useful for batch runs or cron jobs:
### Run the importer
Posts all staged JSON+CSV pairs to FIDI:
Tells FIDI to import all staged JSON+CSV pairs from the mounted directory:
```bash
./import.sh
@@ -138,7 +172,7 @@ sudo journalctl -u firefly-importer -f
```
firefly-importer/
├── import.sh # Batch importer — POSTs JSON+CSV pairs to FIDI
├── import.sh # Batch importer — triggers FIDI /autoimport for the mounted directory
├── watch-imports.sh # File watcher — processes and stages incoming CSVs
├── firefly-importer.service # systemd unit file
├── .env.example # Config template

112
import.sh
View File

@@ -1,6 +1,7 @@
#!/usr/bin/env bash
# Firefly III Data Importer - batch auto-import
# Pairs .json + .csv files by base name and POSTs each to FIDI /autoimport
# Firefly III Data Importer - trigger FIDI directory auto-import
# POSTs to FIDI /autoimport with the container-side import directory path.
# FIDI scans that directory for JSON+CSV pairs and imports them all.
#
# Usage: ./import.sh [--dry-run|-n]
@@ -30,7 +31,8 @@ fi
FIDI_URL="${FIDI_URL:?Set FIDI_URL in .env or environment (e.g. http://localhost:8080)}"
FIDI_SECRET="${FIDI_SECRET:-}"
FIDI_ACCESS_TOKEN="${FIDI_ACCESS_TOKEN:-}"
IMPORT_DIR="${IMPORT_DIR:-$SCRIPT_DIR/imports}"
# Path to the imports directory as seen by FIDI inside its container
FIDI_IMPORT_DIR="${FIDI_IMPORT_DIR:-/imports}"
# ---------------------------------------------------------------------------
# Helpers
@@ -48,95 +50,43 @@ dry() { echo -e "${CYAN}[DRY]${NC} $*"; }
build_url() {
local url="${FIDI_URL%/}/autoimport"
[[ -n "$FIDI_SECRET" ]] && url="${url}?secret=${FIDI_SECRET}"
echo "$url"
}
import_pair() {
local json_file="$1"
local csv_file="$2"
local base
base="$(basename "$json_file" .json)"
local url
url="$(build_url)"
if $DRY_RUN; then
dry "$base"
dry " POST $url"
dry " csv → $csv_file"
dry " json → $json_file"
[[ -n "$FIDI_ACCESS_TOKEN" ]] && dry " auth → Bearer ***"
return 0
fi
info "Importing: $base"
# Build auth args as an array to safely handle spaces/special chars
local curl_args=(-s -w "\n%{http_code}")
[[ -n "$FIDI_ACCESS_TOKEN" ]] && curl_args+=(-H "Authorization: Bearer $FIDI_ACCESS_TOKEN")
curl_args+=(
-F "csv=@${csv_file};type=text/csv"
-F "json=@${json_file};type=application/json"
"$url"
)
local response http_code body
response=$(curl "${curl_args[@]}")
http_code=$(echo "$response" | tail -n1)
body=$(echo "$response" | head -n -1)
if [[ "$http_code" =~ ^2 ]]; then
pass "$base (HTTP $http_code)"
return 0
else
fail "$base (HTTP $http_code)"
echo " Response: $body"
return 1
fi
local qs="directory=${FIDI_IMPORT_DIR}"
[[ -n "$FIDI_SECRET" ]] && qs="${qs}&secret=${FIDI_SECRET}"
echo "${url}?${qs}"
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
if [[ ! -d "$IMPORT_DIR" ]]; then
echo "Import directory not found: $IMPORT_DIR"
exit 1
fi
url="$(build_url)"
$DRY_RUN && info "Dry-run mode — no requests will be sent"
mapfile -t json_files < <(find "$IMPORT_DIR" -maxdepth 1 -name '*.json' | sort)
if [[ ${#json_files[@]} -eq 0 ]]; then
info "No .json files found in $IMPORT_DIR"
if $DRY_RUN; then
info "Dry-run mode — no requests will be sent"
dry "POST ${url//secret=*/secret=***}"
[[ -n "$FIDI_ACCESS_TOKEN" ]] && dry "auth → Bearer ***"
exit 0
fi
success=0
skipped=0
failed=0
info "Triggering FIDI auto-import from ${FIDI_IMPORT_DIR}..."
for json_file in "${json_files[@]}"; do
base="$(basename "$json_file" .json)"
csv_file="${IMPORT_DIR}/${base}.csv"
curl_args=(-s -w "\n%{http_code}" -X POST)
[[ -n "$FIDI_ACCESS_TOKEN" ]] && curl_args+=(-H "Authorization: Bearer $FIDI_ACCESS_TOKEN")
curl_args+=("$url")
if [[ ! -f "$csv_file" ]]; then
info "Skipping $base — no matching .csv found"
((skipped++))
continue
fi
if import_pair "$json_file" "$csv_file"; then
((success++))
else
((failed++))
fi
done
response=$(curl "${curl_args[@]}")
http_code=$(echo "$response" | tail -n1)
body=$(echo "$response" | head -n -1)
echo ""
echo "----------------------------------------"
$DRY_RUN && echo " (dry run — nothing was imported)"
echo " Done: ${success} succeeded, ${failed} failed, ${skipped} skipped"
echo "----------------------------------------"
[[ "$failed" -eq 0 ]]
if [[ "$http_code" =~ ^2 ]]; then
pass "Import triggered (HTTP $http_code)"
echo " Response: $body"
echo "----------------------------------------"
exit 0
else
fail "Import failed (HTTP $http_code)"
echo " Response: $body"
echo "----------------------------------------"
exit 1
fi