WIP

2026-09-26 01:08:50 -07:00
parent fba0546eff
commit 9599867d24
10 changed files with 264 additions and 69 deletions
@@ -47,7 +47,7 @@ BAZARR_PORT=8787
 VLLM_PORT=5081
 SPOTIZERR_PORT=7171
 SPOTIZERR_REDIS_PORT=6379
-TDARR_NODE_ID=ShaanNode
+TDARR_NODE_ID=TdarrTranscoder

 # ----- Docker Images -----
 # get with command:
@@ -43,3 +43,7 @@ homepage/*
 !homepage/*.css
 !homepage/*.js
 !homepage/images/
+
+# Firecrawl (self-hosted web scraper for Hermes) — reference repo clone + app data
+firecrawl/
+firecrawl-data/
@@ -29,3 +29,4 @@ include:
  - media.yml
  - infra.yml
  - web.yml
+  - firecrawl.yml
@@ -0,0 +1,233 @@
+# =============================================================================
+# Firecrawl — self-hosted web scraper for Hermes (scrape/extract) + vLLM (AI)
+# Runs entirely on a private `firecrawl-net` bridge; only the API port
+# (FIRECRAWL_PORT=3002) is published to the host. The homepage's host :3000 is
+# untouched: playwright binds 3000 INTERNALLY only (firecrawl-net), and the
+# api's in-container workers bind the api container's own namespace — so
+# nothing firecrawl runs can collide with host :3000. No docker volumes: all
+# state lives under ./firecrawl-data/<service>. Images are pulled (never
+# built) — see the FIRECRAWL_*_IMAGE digest pins in .env.
+#
+# Port layout:
+#   host  :3002  -> api container :3002   (the ONLY published port)
+#   container-internal (firecrawl-net):
+#     api (express)            :3002
+#     playwright-service       :3000
+#     extract-worker           :3004
+#     queue-worker (liveness)  :3005
+#     nuq-worker x5            :3006-3010
+#     nuq-prefetch-worker      :3011
+#     nuq-reconciler-worker    :3012
+#     cclog-worker             :3013
+#   postgres :5432 / rabbitmq :5672 / redis :6380 — internal, not published
+#
+# Env: each firecrawl service loads ONLY its own env file — NEVER the global
+# stack .env (it would leak spotizerr's REDIS_PASSWORD into firecrawl-redis,
+# whose entrypoint turns it into requirepass and break the passwordless
+# REDIS_URL). .firecrawl.env carries the api+postgres contract; the two keys
+# needing ${} interpolation (OPENAI_BASE_URL -> vLLM on the LAN,
+# SEARXNG_ENDPOINT -> SearXNG on the LAN) live in `environment:` because
+# env_file does not interpolate. LAN reach: extra_hosts host-gateway.
+#
+# Key ordering per service (matches compose.yml):
+#   image → container_name → env_file → networks/network_mode
+#   → depends_on → cap_add → ports → volumes → environment
+#   → labels → healthcheck → security_opt → mem_limit → cpus
+#   → devices → restart
+# =============================================================================
+
+networks:
+  firecrawl-net:
+    driver: bridge
+
+services:
+  # ---------------------------------------------------------------------------
+  # API + in-container workers (api, queue-worker, extract-worker, nuq workers)
+  # ---------------------------------------------------------------------------
+  firecrawl-api:
+    image: ${FIRECRAWL_IMAGE}
+    container_name: firecrawl-api
+    env_file:
+      - ./env/.firecrawl.env
+    networks:
+      - firecrawl-net
+    depends_on:
+      firecrawl-postgres:
+        condition: service_healthy
+      firecrawl-rabbitmq:
+        condition: service_healthy
+      firecrawl-redis:
+        condition: service_healthy
+      firecrawl-playwright:
+        condition: service_healthy
+    ulimits:
+      nofile:
+        soft: 65535
+        hard: 65535
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    ports:
+      - "${FIRECRAWL_PORT}:3002"
+    # ${} expansion (env_file does not interpolate): vLLM is host-networked, so
+    # LAN IP + VLLM_PORT reach it from inside; same for SearXNG.
+    environment:
+      - "OPENAI_BASE_URL=http://${LOCAL_IPV4}:${VLLM_PORT}/v1"
+      - "SEARXNG_ENDPOINT=http://${LOCAL_IPV4}:${SEARXNG_PORT}"
+    labels:
+      - "autoheal=true"
+    healthcheck:
+      test: ["CMD", "curl", "-sf", "http://127.0.0.1:3002/v0/health/liveness"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+      start_period: 60s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+        compress: "true"
+    security_opt:
+      - no-new-privileges:true
+    mem_limit: 6g
+    memswap_limit: 8g
+    cpus: 2.0
+    restart: unless-stopped
+
+  # ---------------------------------------------------------------------------
+  # Playwright browser microservice — internal only (port 3000 on firecrawl-net)
+  # ---------------------------------------------------------------------------
+  firecrawl-playwright:
+    image: ${FIRECRAWL_PLAYWRIGHT_IMAGE}
+    container_name: firecrawl-playwright
+    env_file:
+      - ./env/.firecrawl-playwright.env
+    networks:
+      - firecrawl-net
+    cap_drop:
+      - ALL
+    labels:
+      - "autoheal=true"
+    healthcheck:
+      test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3000/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"]
+      interval: 20s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+        compress: "true"
+    tmpfs:
+      - /tmp/.cache:noexec,nosuid,size=1g
+    security_opt:
+      - no-new-privileges:true
+    mem_limit: 4g
+    memswap_limit: 4g
+    cpus: 1.0
+    restart: unless-stopped
+
+  # ---------------------------------------------------------------------------
+  # Redis — BullMQ job queues + rate limiting (internal, not host-published)
+  # Port 6380 (not 6379) so it can never be confused with spotizerr's redis.
+  # ---------------------------------------------------------------------------
+  firecrawl-redis:
+    image: ${FIRECRAWL_REDIS_IMAGE}
+    container_name: firecrawl-redis
+    networks:
+      - firecrawl-net
+    command: redis-server --bind 0.0.0.0 --port 6380 --maxmemory 256mb --maxmemory-policy noeviction
+    # NOTE: no `cap_drop: ALL` here — the entrypoint must chown the bind
+    # mount on first boot (same reason postgres has no cap_drop). Dropping ALL
+    # strips CAP_CHOWN/DAC_OVERRIDE and makes the root entrypoint fail with EPERM.
+    volumes:
+      - ./firecrawl-data/redis:/data
+    labels:
+      - "autoheal=true"
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "6380", "ping"]
+      interval: 10s
+      timeout: 3s
+      retries: 5
+      start_period: 5s
+    logging:
+      driver: json-file
+      options:
+        max-size: "5m"
+        max-file: "2"
+        compress: "true"
+    security_opt:
+      - no-new-privileges:true
+    mem_limit: 256m
+    cpus: 0.5
+    restart: unless-stopped
+
+  # ---------------------------------------------------------------------------
+  # RabbitMQ — NUQ transport (internal, not host-published). Default guest/guest
+  # works cross-container on the bridge (verified); no env file needed.
+  # ---------------------------------------------------------------------------
+  firecrawl-rabbitmq:
+    image: ${FIRECRAWL_RABBITMQ_IMAGE}
+    container_name: firecrawl-rabbitmq
+    networks:
+      - firecrawl-net
+    command: rabbitmq-server
+    # NOTE: no `cap_drop: ALL` — entrypoint chowns the bind mount on boot
+    # (same reason postgres has none). Dropping ALL causes EPERM on chown.
+    volumes:
+      - ./firecrawl-data/rabbitmq:/var/lib/rabbitmq
+    labels:
+      - "autoheal=true"
+    healthcheck:
+      test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 20s
+    logging:
+      driver: json-file
+      options:
+        max-size: "5m"
+        max-file: "2"
+        compress: "true"
+    security_opt:
+      - no-new-privileges:true
+    mem_limit: 512m
+    cpus: 0.5
+    restart: unless-stopped
+
+  # ---------------------------------------------------------------------------
+  # Postgres (NUQ queue store, pg_cron) — internal, not host-published.
+  # DB must stay named 'postgres' (see .firecrawl.env note: the baked
+  # pg_cron extension in this image is configured for cron.database_name='postgres').
+  # ---------------------------------------------------------------------------
+  firecrawl-postgres:
+    image: ${FIRECRAWL_POSTGRES_IMAGE}
+    container_name: firecrawl-postgres
+    env_file:
+      - ./env/.firecrawl.env
+    networks:
+      - firecrawl-net
+    volumes:
+      - ./firecrawl-data/postgres:/var/lib/postgresql/data
+    labels:
+      - "autoheal=true"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 20s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+        compress: "true"
+    security_opt:
+      - no-new-privileges:true
+    mem_limit: 512m
+    cpus: 0.5
+    restart: unless-stopped
@@ -44,15 +44,12 @@
    - "qBittorrent":
        - icon: sh-qbittorrent
          href: "{{HOMEPAGE_VAR_QBITTORRENT_HOST}}"
-    - "Scrutiny":
-        - icon: sh-scrutiny-light
-          href: "{{HOMEPAGE_VAR_SCRUTINY_HOST}}"
+    - "Spotizerr":
+        - icon: sh-spotify-light
+          href: http://shaan-server:7171
    - "Router Gateway":
        - icon: "http://192.168.50.1/images/favicon.png"
          href: http://192.168.50.1
-    - "Tailscale":
-        - icon: sh-tailscale-light
-          href: https://login.tailscale.com
    - "Portainer":
        - icon: sh-portainer-light
          href: "{{HOMEPAGE_VAR_PORTAINER_HOST}}"
@@ -62,6 +59,9 @@
    - "Cloudflare":
        - icon: sh-cloudflare
          href: https://dash.cloudflare.com
+    - "Tailscale":
+        - icon: sh-tailscale-light
+          href: https://login.tailscale.com

 - Bookmarks:
    - "Youtube":
@@ -85,3 +85,6 @@
    - "Discord":
        - icon: sh-discord
          href: https://discord.com/channels/@me
+    - "Spotify":
+        - icon: sh-spotify
+          href: https://open.spotify.com
@@ -34,6 +34,11 @@
          deviceid: "{{HOMEPAGE_VAR_TAILSCALE_DEVICE_KEY}}"
          key: "{{HOMEPAGE_VAR_TAILSCALE_KEY}}"
          fields: [address]
+    - "Caddy":
+        icon: sh-caddy
+        widget:
+          type: caddy
+          url: "http://localhost:2019"
 - "Media":
    - "Media_1":
        - "Open WebUI":
@@ -195,8 +200,8 @@
                      when: gt
                      value: 0
        - "Spotizerr":
-            icon: sh-spotify
-            href: http://shaan-server:7171
+            icon: sh-spotify-light
+            href: http://shaan-server:7171 # "{{HOMEPAGE_VAR_SPOTIZERR_HOST}}"
            server: my-docker
            container: spotizerr
    - "Media_Bottom":
@@ -226,13 +231,13 @@
    - "Network_1":
        - "Portainer":
            icon: sh-portainer-light
-            href: https://shaan-server:9443
+            href: "{{HOMEPAGE_VAR_PORTAINER_HOST}}"
            description: Containers
            server: my-docker
            container: portainer
            widget:
              type: portainer
-              url: https://portainer:9443
+              url: https://localhost:9443
              env: 3
              key: "{{HOMEPAGE_VAR_PORTAINER_KEY}}"
        - "Pi-Hole Adblocker":
@@ -10,19 +10,12 @@
 # =============================================================================

 networks:
-  #caddy_net:
-  # external: true
  tdarr-net:
    driver: bridge
  default:
    name: portainer_network

 volumes:
-  #caddy_config:
-  #caddy_data:
-  #  external: true
-  #myspeed:
-  #  external: true
  portainer_data:
    name: portainer_data

@@ -72,8 +65,6 @@ services:
    container_name: autoheal
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - AUTOHEAL_CONTAINER_LABEL=all
      - AUTOHEAL_INTERVAL=30
@@ -98,8 +89,6 @@ services:
      - /run/udev:/run/udev:ro
      - ./scrutiny/config:/opt/scrutiny/config
      - ./scrutiny/influxdb:/opt/scrutiny/influxdb
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - TZ=${TIMEZONE}
    security_opt:
@@ -126,14 +115,10 @@ services:
    volumes:
      - ./etc-pihole:/etc/pihole
      - ./etc-dnsmasq.d:/etc/dnsmasq.d
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - TZ=${TIMEZONE}
      - FTLCONF_misc_etc_dnsmasq_d=true
      - FTLCONF_webserver_port=${PIHOLE_PORT:-80}
-    # security_opt:
-    #   - no-new-privileges:true
    mem_limit: 256m
    cpus: 1.0
    restart: unless-stopped
@@ -146,8 +131,6 @@ services:
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - portainer_data:/data
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - TZ=${TIMEZONE}
    security_opt:
@@ -10,19 +10,12 @@
 # =============================================================================

 networks:
-  #caddy_net:
-  # external: true
  tdarr-net:
    driver: bridge
  default:
    name: portainer_network

 volumes:
-  #caddy_config:
-  #caddy_data:
-  #  external: true
-  #myspeed:
-  #  external: true
  portainer_data:
    name: portainer_data
  redis-data:
@@ -51,8 +44,6 @@ services:
      - ${BAZARR_PORT}:${BAZARR_PORT}
    volumes:
      - ./gluetun-data:/tmp/gluetun:rw
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - VPN_SERVICE_PROVIDER=protonvpn
      - VPN_TYPE=wireguard
@@ -97,8 +88,6 @@ services:
      - ${DISK1}:${DISK1}
      - ${DISK2}:${DISK2}
      - ./gluetun-data:/tmp/gluetun
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - PUID=${HOST_PUID:-1000}
      - PGID=${HOST_PGID:-1000}
@@ -188,8 +177,6 @@ services:
    volumes:
      - ./prowlarr:/config
      - ./notify.sh:/notify.sh:ro
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - PUID=${HOST_PUID:-1000}
      - PGID=${HOST_PGID:-1000}
@@ -223,8 +210,6 @@ services:
      - ./sonarr:/config
      - ${DISK1}:${DISK1}
      - ${DISK2}:${DISK2}
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - PUID=${HOST_PUID:-1000}
      - PGID=${HOST_PGID:-1000}
@@ -258,8 +243,6 @@ services:
      - ./radarr:/config
      - ${DISK1}:${DISK1}
      - ${DISK2}:${DISK2}
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - PUID=${HOST_PUID:-1000}
      - PGID=${HOST_PGID:-1000}
@@ -294,8 +277,6 @@ services:
      - ./lidarr:/config
      - ${DISK1}:${DISK1}
      - ${DISK2}:${DISK2}
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - PUID=${HOST_PUID:-1000}
      - PGID=${HOST_PGID:-1000}
@@ -329,8 +310,6 @@ services:
      - ./bazarr:/config
      - ${DISK1}:${DISK1}
      - ${DISK2}:${DISK2}
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - PUID=${HOST_PUID:-1000}
      - PGID=${HOST_PGID:-1000}
@@ -360,14 +339,10 @@ services:
  tautulli:
    image: ${TAUTULLI_IMAGE}
    container_name: tautulli
-    #networks:
-    #  - caddy_net
    ports:
      - ${TAUTULLI_PORT}:${TAUTULLI_PORT}
    volumes:
      - ./tautulli:/config
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - PUID=${HOST_PUID:-1000}
      - PGID=${HOST_PGID:-1000}
@@ -392,8 +367,6 @@ services:
      - ./tdarr/temp:/temp
      - ${DISK1}:${DISK1}
      - ${DISK2}:${DISK2}
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - TZ=${TIMEZONE}
      - PUID=${HOST_PUID:-1000}
@@ -417,8 +390,6 @@ services:
      - ./tdarr/temp:/temp
      - ${DISK1}:${DISK1}
      - ${DISK2}:${DISK2}
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - TZ=${TIMEZONE}
      - PUID=${HOST_PUID:-1000}
@@ -438,14 +409,10 @@ services:
  seerr:
    image: ${SEERR_IMAGE}
    container_name: seerr
-    # networks:
-    #  - caddy_net
    ports:
      - ${SEERR_PORT}:${SEERR_PORT}
    volumes:
      - ./seerr:/app/config
-      #- /etc/localtime:/etc/localtime:ro
-      #- /etc/timezone:/etc/timezone:ro
    environment:
      - LOG_LEVEL=info
      - TZ=${TIMEZONE}
@@ -2,5 +2,5 @@
 sleep 15

 cd /home/shaan/torrent-stack || exit
-docker compose config ||> /tmp/startup.log && exit 1
-docker compose up -d 2>&1 | tee /tmp/startup.log
+/usr/bin/docker compose config || >/tmp/startup.log && exit 1
+/usr/bin/docker compose up -d 2>&1 | tee /tmp/startup.log
@@ -38,8 +38,7 @@ services:
    env_file:
      - .env
      - ./env/.homepage.env
-    #networks:
-    #  - caddy_net
+    network_mode: host
    ports:
      - ${HOMEPAGE_PORT}:${HOMEPAGE_PORT}
    volumes:
@@ -49,7 +48,7 @@ services:
      #- /etc/localtime:/etc/localtime:ro
      #- /etc/timezone:/etc/timezone:ro
    environment:
-      - "HOMEPAGE_ALLOWED_HOSTS=gethomepage.dev,${LOCAL_IPV4}:${HOMEPAGE_PORT},${SERVER_NAME}:${HOMEPAGE_PORT},${SERVER_NAME}.${TAILNET_NAME}:${HOMEPAGE_PORT},host.docker.internal:{HOMEPAGE_PORT}"
+      - "HOMEPAGE_ALLOWED_HOSTS=gethomepage.dev,${LOCAL_IPV4}:${HOMEPAGE_PORT},${SERVER_NAME}:${HOMEPAGE_PORT},${SERVER_NAME}.${TAILNET_NAME}:${HOMEPAGE_PORT},host.docker.internal:{HOMEPAGE_PORT},${SERVER_NAME}.local:${HOMEPAGE_PORT},${SERVER_NAME}.local"
      - "HOMEPAGE_VAR_DISK1=${DISK1}"
      - "HOMEPAGE_VAR_DISK2=${DISK2}"
      - "HOMEPAGE_VAR_DISK3=${DISK3}"
@@ -70,7 +69,7 @@ services:
      - "HOMEPAGE_VAR_SEERR_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${SEERR_PORT}"
      - "HOMEPAGE_VAR_OPENWEBUI_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${OPENWEBUI_PORT}"
      - "HOMEPAGE_VAR_PIHOLE_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${PIHOLE_PORT}"
-      - "HOMEPAGE_VAR_PORTAINER_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${PORTAINER_PORT}"
+      - "HOMEPAGE_VAR_PORTAINER_ADDRESS=${WEB_PROTOCOL}s://${LOCAL_IPV4}:${PORTAINER_PORT}"
      - "HOMEPAGE_VAR_SCRUTINY_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${SCRUTINY_PORT}"
      - "HOMEPAGE_VAR_KARAKEEP_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${KARAKEEP_PORT}"
      - "HOMEPAGE_VAR_VLLM_ADDRESS=${WEB_PROTOCOL}://${LOCAL_IPV4}:${VLLM_PORT}"
@@ -97,8 +96,6 @@ services:
      - no-new-privileges:true
    mem_limit: 256m
    cpus: 1
-    # extra_hosts:
-    #  - "host.docker.internal:${LOCAL_IPV4}"
    restart: unless-stopped
  # ---------------------------------------------------------------------------

@@ -225,6 +222,8 @@ services:
          --enable-prefix-caching \
          --enable-chunked-prefill
    restart: on-failure:5
+    cpus: 4.0
+    mem_limit: 16g

  # ---------------------------------------------------------------------------
  # Karakeep