From 9599867d243949f8452bc251fa3fc4e3a7f843bc Mon Sep 17 00:00:00 2001 From: dhaan7 Date: Sat, 26 Sep 2026 01:08:50 -0700 Subject: [PATCH] WIP --- .env.example | 2 +- .gitignore | 4 + compose.yml | 1 + firecrawl.yml | 233 ++++++++++++++++++++++++++++++++++++++++ homepage/bookmarks.yaml | 15 +-- homepage/services.yaml | 13 ++- infra.yml | 17 --- media.yml | 33 ------ startup.sh | 4 +- web.yml | 11 +- 10 files changed, 264 insertions(+), 69 deletions(-) create mode 100644 firecrawl.yml diff --git a/.env.example b/.env.example index 5137894..450dc21 100644 --- a/.env.example +++ b/.env.example @@ -47,7 +47,7 @@ BAZARR_PORT=8787 VLLM_PORT=5081 SPOTIZERR_PORT=7171 SPOTIZERR_REDIS_PORT=6379 -TDARR_NODE_ID=ShaanNode +TDARR_NODE_ID=TdarrTranscoder # ----- Docker Images ----- # get with command: diff --git a/.gitignore b/.gitignore index 4e80662..7da0b3f 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ homepage/* !homepage/*.css !homepage/*.js !homepage/images/ + +# Firecrawl (self-hosted web scraper for Hermes) — reference repo clone + app data +firecrawl/ +firecrawl-data/ diff --git a/compose.yml b/compose.yml index 55fe5ac..962985e 100755 --- a/compose.yml +++ b/compose.yml @@ -29,3 +29,4 @@ include: - media.yml - infra.yml - web.yml + - firecrawl.yml diff --git a/firecrawl.yml b/firecrawl.yml new file mode 100644 index 0000000..3fefdb8 --- /dev/null +++ b/firecrawl.yml @@ -0,0 +1,233 @@ +# ============================================================================= +# Firecrawl — self-hosted web scraper for Hermes (scrape/extract) + vLLM (AI) +# Runs entirely on a private `firecrawl-net` bridge; only the API port +# (FIRECRAWL_PORT=3002) is published to the host. The homepage's host :3000 is +# untouched: playwright binds 3000 INTERNALLY only (firecrawl-net), and the +# api's in-container workers bind the api container's own namespace — so +# nothing firecrawl runs can collide with host :3000. No docker volumes: all +# state lives under ./firecrawl-data/. Images are pulled (never +# built) — see the FIRECRAWL_*_IMAGE digest pins in .env. +# +# Port layout: +# host :3002 -> api container :3002 (the ONLY published port) +# container-internal (firecrawl-net): +# api (express) :3002 +# playwright-service :3000 +# extract-worker :3004 +# queue-worker (liveness) :3005 +# nuq-worker x5 :3006-3010 +# nuq-prefetch-worker :3011 +# nuq-reconciler-worker :3012 +# cclog-worker :3013 +# postgres :5432 / rabbitmq :5672 / redis :6380 — internal, not published +# +# Env: each firecrawl service loads ONLY its own env file — NEVER the global +# stack .env (it would leak spotizerr's REDIS_PASSWORD into firecrawl-redis, +# whose entrypoint turns it into requirepass and break the passwordless +# REDIS_URL). .firecrawl.env carries the api+postgres contract; the two keys +# needing ${} interpolation (OPENAI_BASE_URL -> vLLM on the LAN, +# SEARXNG_ENDPOINT -> SearXNG on the LAN) live in `environment:` because +# env_file does not interpolate. LAN reach: extra_hosts host-gateway. +# +# Key ordering per service (matches compose.yml): +# image → container_name → env_file → networks/network_mode +# → depends_on → cap_add → ports → volumes → environment +# → labels → healthcheck → security_opt → mem_limit → cpus +# → devices → restart +# ============================================================================= + +networks: + firecrawl-net: + driver: bridge + +services: + # --------------------------------------------------------------------------- + # API + in-container workers (api, queue-worker, extract-worker, nuq workers) + # --------------------------------------------------------------------------- + firecrawl-api: + image: ${FIRECRAWL_IMAGE} + container_name: firecrawl-api + env_file: + - ./env/.firecrawl.env + networks: + - firecrawl-net + depends_on: + firecrawl-postgres: + condition: service_healthy + firecrawl-rabbitmq: + condition: service_healthy + firecrawl-redis: + condition: service_healthy + firecrawl-playwright: + condition: service_healthy + ulimits: + nofile: + soft: 65535 + hard: 65535 + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "${FIRECRAWL_PORT}:3002" + # ${} expansion (env_file does not interpolate): vLLM is host-networked, so + # LAN IP + VLLM_PORT reach it from inside; same for SearXNG. + environment: + - "OPENAI_BASE_URL=http://${LOCAL_IPV4}:${VLLM_PORT}/v1" + - "SEARXNG_ENDPOINT=http://${LOCAL_IPV4}:${SEARXNG_PORT}" + labels: + - "autoheal=true" + healthcheck: + test: ["CMD", "curl", "-sf", "http://127.0.0.1:3002/v0/health/liveness"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 60s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + compress: "true" + security_opt: + - no-new-privileges:true + mem_limit: 6g + memswap_limit: 8g + cpus: 2.0 + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Playwright browser microservice — internal only (port 3000 on firecrawl-net) + # --------------------------------------------------------------------------- + firecrawl-playwright: + image: ${FIRECRAWL_PLAYWRIGHT_IMAGE} + container_name: firecrawl-playwright + env_file: + - ./env/.firecrawl-playwright.env + networks: + - firecrawl-net + cap_drop: + - ALL + labels: + - "autoheal=true" + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3000/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"] + interval: 20s + timeout: 5s + retries: 5 + start_period: 30s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + compress: "true" + tmpfs: + - /tmp/.cache:noexec,nosuid,size=1g + security_opt: + - no-new-privileges:true + mem_limit: 4g + memswap_limit: 4g + cpus: 1.0 + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Redis — BullMQ job queues + rate limiting (internal, not host-published) + # Port 6380 (not 6379) so it can never be confused with spotizerr's redis. + # --------------------------------------------------------------------------- + firecrawl-redis: + image: ${FIRECRAWL_REDIS_IMAGE} + container_name: firecrawl-redis + networks: + - firecrawl-net + command: redis-server --bind 0.0.0.0 --port 6380 --maxmemory 256mb --maxmemory-policy noeviction + # NOTE: no `cap_drop: ALL` here — the entrypoint must chown the bind + # mount on first boot (same reason postgres has no cap_drop). Dropping ALL + # strips CAP_CHOWN/DAC_OVERRIDE and makes the root entrypoint fail with EPERM. + volumes: + - ./firecrawl-data/redis:/data + labels: + - "autoheal=true" + healthcheck: + test: ["CMD", "redis-cli", "-p", "6380", "ping"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 5s + logging: + driver: json-file + options: + max-size: "5m" + max-file: "2" + compress: "true" + security_opt: + - no-new-privileges:true + mem_limit: 256m + cpus: 0.5 + restart: unless-stopped + + # --------------------------------------------------------------------------- + # RabbitMQ — NUQ transport (internal, not host-published). Default guest/guest + # works cross-container on the bridge (verified); no env file needed. + # --------------------------------------------------------------------------- + firecrawl-rabbitmq: + image: ${FIRECRAWL_RABBITMQ_IMAGE} + container_name: firecrawl-rabbitmq + networks: + - firecrawl-net + command: rabbitmq-server + # NOTE: no `cap_drop: ALL` — entrypoint chowns the bind mount on boot + # (same reason postgres has none). Dropping ALL causes EPERM on chown. + volumes: + - ./firecrawl-data/rabbitmq:/var/lib/rabbitmq + labels: + - "autoheal=true" + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s + logging: + driver: json-file + options: + max-size: "5m" + max-file: "2" + compress: "true" + security_opt: + - no-new-privileges:true + mem_limit: 512m + cpus: 0.5 + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Postgres (NUQ queue store, pg_cron) — internal, not host-published. + # DB must stay named 'postgres' (see .firecrawl.env note: the baked + # pg_cron extension in this image is configured for cron.database_name='postgres'). + # --------------------------------------------------------------------------- + firecrawl-postgres: + image: ${FIRECRAWL_POSTGRES_IMAGE} + container_name: firecrawl-postgres + env_file: + - ./env/.firecrawl.env + networks: + - firecrawl-net + volumes: + - ./firecrawl-data/postgres:/var/lib/postgresql/data + labels: + - "autoheal=true" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + compress: "true" + security_opt: + - no-new-privileges:true + mem_limit: 512m + cpus: 0.5 + restart: unless-stopped diff --git a/homepage/bookmarks.yaml b/homepage/bookmarks.yaml index a12fcfa..62d3642 100755 --- a/homepage/bookmarks.yaml +++ b/homepage/bookmarks.yaml @@ -44,15 +44,12 @@ - "qBittorrent": - icon: sh-qbittorrent href: "{{HOMEPAGE_VAR_QBITTORRENT_HOST}}" - - "Scrutiny": - - icon: sh-scrutiny-light - href: "{{HOMEPAGE_VAR_SCRUTINY_HOST}}" + - "Spotizerr": + - icon: sh-spotify-light + href: http://shaan-server:7171 - "Router Gateway": - icon: "http://192.168.50.1/images/favicon.png" href: http://192.168.50.1 - - "Tailscale": - - icon: sh-tailscale-light - href: https://login.tailscale.com - "Portainer": - icon: sh-portainer-light href: "{{HOMEPAGE_VAR_PORTAINER_HOST}}" @@ -62,6 +59,9 @@ - "Cloudflare": - icon: sh-cloudflare href: https://dash.cloudflare.com + - "Tailscale": + - icon: sh-tailscale-light + href: https://login.tailscale.com - Bookmarks: - "Youtube": @@ -85,3 +85,6 @@ - "Discord": - icon: sh-discord href: https://discord.com/channels/@me + - "Spotify": + - icon: sh-spotify + href: https://open.spotify.com diff --git a/homepage/services.yaml b/homepage/services.yaml index 949c30f..5a58dd4 100755 --- a/homepage/services.yaml +++ b/homepage/services.yaml @@ -34,6 +34,11 @@ deviceid: "{{HOMEPAGE_VAR_TAILSCALE_DEVICE_KEY}}" key: "{{HOMEPAGE_VAR_TAILSCALE_KEY}}" fields: [address] + - "Caddy": + icon: sh-caddy + widget: + type: caddy + url: "http://localhost:2019" - "Media": - "Media_1": - "Open WebUI": @@ -195,8 +200,8 @@ when: gt value: 0 - "Spotizerr": - icon: sh-spotify - href: http://shaan-server:7171 + icon: sh-spotify-light + href: http://shaan-server:7171 # "{{HOMEPAGE_VAR_SPOTIZERR_HOST}}" server: my-docker container: spotizerr - "Media_Bottom": @@ -226,13 +231,13 @@ - "Network_1": - "Portainer": icon: sh-portainer-light - href: https://shaan-server:9443 + href: "{{HOMEPAGE_VAR_PORTAINER_HOST}}" description: Containers server: my-docker container: portainer widget: type: portainer - url: https://portainer:9443 + url: https://localhost:9443 env: 3 key: "{{HOMEPAGE_VAR_PORTAINER_KEY}}" - "Pi-Hole Adblocker": diff --git a/infra.yml b/infra.yml index b872f8f..eaf0082 100644 --- a/infra.yml +++ b/infra.yml @@ -10,19 +10,12 @@ # ============================================================================= networks: - #caddy_net: - # external: true tdarr-net: driver: bridge default: name: portainer_network volumes: - #caddy_config: - #caddy_data: - # external: true - #myspeed: - # external: true portainer_data: name: portainer_data @@ -72,8 +65,6 @@ services: container_name: autoheal volumes: - /var/run/docker.sock:/var/run/docker.sock - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - AUTOHEAL_CONTAINER_LABEL=all - AUTOHEAL_INTERVAL=30 @@ -98,8 +89,6 @@ services: - /run/udev:/run/udev:ro - ./scrutiny/config:/opt/scrutiny/config - ./scrutiny/influxdb:/opt/scrutiny/influxdb - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - TZ=${TIMEZONE} security_opt: @@ -126,14 +115,10 @@ services: volumes: - ./etc-pihole:/etc/pihole - ./etc-dnsmasq.d:/etc/dnsmasq.d - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - TZ=${TIMEZONE} - FTLCONF_misc_etc_dnsmasq_d=true - FTLCONF_webserver_port=${PIHOLE_PORT:-80} - # security_opt: - # - no-new-privileges:true mem_limit: 256m cpus: 1.0 restart: unless-stopped @@ -146,8 +131,6 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock - portainer_data:/data - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - TZ=${TIMEZONE} security_opt: diff --git a/media.yml b/media.yml index cce24f9..47b8fe2 100644 --- a/media.yml +++ b/media.yml @@ -10,19 +10,12 @@ # ============================================================================= networks: - #caddy_net: - # external: true tdarr-net: driver: bridge default: name: portainer_network volumes: - #caddy_config: - #caddy_data: - # external: true - #myspeed: - # external: true portainer_data: name: portainer_data redis-data: @@ -51,8 +44,6 @@ services: - ${BAZARR_PORT}:${BAZARR_PORT} volumes: - ./gluetun-data:/tmp/gluetun:rw - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - VPN_SERVICE_PROVIDER=protonvpn - VPN_TYPE=wireguard @@ -97,8 +88,6 @@ services: - ${DISK1}:${DISK1} - ${DISK2}:${DISK2} - ./gluetun-data:/tmp/gluetun - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - PUID=${HOST_PUID:-1000} - PGID=${HOST_PGID:-1000} @@ -188,8 +177,6 @@ services: volumes: - ./prowlarr:/config - ./notify.sh:/notify.sh:ro - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - PUID=${HOST_PUID:-1000} - PGID=${HOST_PGID:-1000} @@ -223,8 +210,6 @@ services: - ./sonarr:/config - ${DISK1}:${DISK1} - ${DISK2}:${DISK2} - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - PUID=${HOST_PUID:-1000} - PGID=${HOST_PGID:-1000} @@ -258,8 +243,6 @@ services: - ./radarr:/config - ${DISK1}:${DISK1} - ${DISK2}:${DISK2} - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - PUID=${HOST_PUID:-1000} - PGID=${HOST_PGID:-1000} @@ -294,8 +277,6 @@ services: - ./lidarr:/config - ${DISK1}:${DISK1} - ${DISK2}:${DISK2} - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - PUID=${HOST_PUID:-1000} - PGID=${HOST_PGID:-1000} @@ -329,8 +310,6 @@ services: - ./bazarr:/config - ${DISK1}:${DISK1} - ${DISK2}:${DISK2} - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - PUID=${HOST_PUID:-1000} - PGID=${HOST_PGID:-1000} @@ -360,14 +339,10 @@ services: tautulli: image: ${TAUTULLI_IMAGE} container_name: tautulli - #networks: - # - caddy_net ports: - ${TAUTULLI_PORT}:${TAUTULLI_PORT} volumes: - ./tautulli:/config - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - PUID=${HOST_PUID:-1000} - PGID=${HOST_PGID:-1000} @@ -392,8 +367,6 @@ services: - ./tdarr/temp:/temp - ${DISK1}:${DISK1} - ${DISK2}:${DISK2} - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - TZ=${TIMEZONE} - PUID=${HOST_PUID:-1000} @@ -417,8 +390,6 @@ services: - ./tdarr/temp:/temp - ${DISK1}:${DISK1} - ${DISK2}:${DISK2} - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - TZ=${TIMEZONE} - PUID=${HOST_PUID:-1000} @@ -438,14 +409,10 @@ services: seerr: image: ${SEERR_IMAGE} container_name: seerr - # networks: - # - caddy_net ports: - ${SEERR_PORT}:${SEERR_PORT} volumes: - ./seerr:/app/config - #- /etc/localtime:/etc/localtime:ro - #- /etc/timezone:/etc/timezone:ro environment: - LOG_LEVEL=info - TZ=${TIMEZONE} diff --git a/startup.sh b/startup.sh index c2cca25..895b371 100755 --- a/startup.sh +++ b/startup.sh @@ -2,5 +2,5 @@ sleep 15 cd /home/shaan/torrent-stack || exit -docker compose config ||> /tmp/startup.log && exit 1 -docker compose up -d 2>&1 | tee /tmp/startup.log +/usr/bin/docker compose config || >/tmp/startup.log && exit 1 +/usr/bin/docker compose up -d 2>&1 | tee /tmp/startup.log diff --git a/web.yml b/web.yml index 17f994e..ae96ed2 100644 --- a/web.yml +++ b/web.yml @@ -38,8 +38,7 @@ services: env_file: - .env - ./env/.homepage.env - #networks: - # - caddy_net + network_mode: host ports: - ${HOMEPAGE_PORT}:${HOMEPAGE_PORT} volumes: @@ -49,7 +48,7 @@ services: #- /etc/localtime:/etc/localtime:ro #- /etc/timezone:/etc/timezone:ro environment: - - "HOMEPAGE_ALLOWED_HOSTS=gethomepage.dev,${LOCAL_IPV4}:${HOMEPAGE_PORT},${SERVER_NAME}:${HOMEPAGE_PORT},${SERVER_NAME}.${TAILNET_NAME}:${HOMEPAGE_PORT},host.docker.internal:{HOMEPAGE_PORT}" + - "HOMEPAGE_ALLOWED_HOSTS=gethomepage.dev,${LOCAL_IPV4}:${HOMEPAGE_PORT},${SERVER_NAME}:${HOMEPAGE_PORT},${SERVER_NAME}.${TAILNET_NAME}:${HOMEPAGE_PORT},host.docker.internal:{HOMEPAGE_PORT},${SERVER_NAME}.local:${HOMEPAGE_PORT},${SERVER_NAME}.local" - "HOMEPAGE_VAR_DISK1=${DISK1}" - "HOMEPAGE_VAR_DISK2=${DISK2}" - "HOMEPAGE_VAR_DISK3=${DISK3}" @@ -70,7 +69,7 @@ services: - "HOMEPAGE_VAR_SEERR_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${SEERR_PORT}" - "HOMEPAGE_VAR_OPENWEBUI_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${OPENWEBUI_PORT}" - "HOMEPAGE_VAR_PIHOLE_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${PIHOLE_PORT}" - - "HOMEPAGE_VAR_PORTAINER_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${PORTAINER_PORT}" + - "HOMEPAGE_VAR_PORTAINER_ADDRESS=${WEB_PROTOCOL}s://${LOCAL_IPV4}:${PORTAINER_PORT}" - "HOMEPAGE_VAR_SCRUTINY_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${SCRUTINY_PORT}" - "HOMEPAGE_VAR_KARAKEEP_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${KARAKEEP_PORT}" - "HOMEPAGE_VAR_VLLM_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${VLLM_PORT}" @@ -97,8 +96,6 @@ services: - no-new-privileges:true mem_limit: 256m cpus: 1 - # extra_hosts: - # - "host.docker.internal:${LOCAL_IPV4}" restart: unless-stopped # --------------------------------------------------------------------------- @@ -225,6 +222,8 @@ services: --enable-prefix-caching \ --enable-chunked-prefill restart: on-failure:5 + cpus: 4.0 + mem_limit: 16g # --------------------------------------------------------------------------- # Karakeep