{
    "archive_path": "archive/1766121593.807781",
    "base_url": "blog.jakesaunders.dev/my-server-started-mining-monero-this-morning",
    "basename": "",
    "bookmarked_date": "2025-12-19 05:19",
    "canonical": {
        "archive_org_path": "https://web.archive.org/web/blog.jakesaunders.dev/my-server-started-mining-monero-this-morning",
        "dom_path": "output.html",
        "favicon_path": "favicon.ico",
        "git_path": "git/",
        "google_favicon_path": "https://www.google.com/s2/favicons?domain=blog.jakesaunders.dev",
        "headers_path": "headers.json",
        "htmltotext_path": "htmltotext.txt",
        "index_path": "index.html",
        "media_path": "media/",
        "mercury_path": "mercury/content.html",
        "pdf_path": "output.pdf",
        "readability_path": "readability/content.html",
        "screenshot_path": "screenshot.png",
        "singlefile_path": "singlefile.html",
        "warc_path": "warc/",
        "wget_path": null
    },
    "domain": "blog.jakesaunders.dev",
    "downloaded_at": "2025-12-19T05:20:00.669453+00:00",
    "downloaded_datestr": "2025-12-19 05:20",
    "extension": "",
    "hash": "A00TCP20TPNVZ8F79E22",
    "history": {
        "archive_org": [
            {
                "cmd": [
                    "/usr/bin/curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--proxy",
                    "socks5://tor-socks-proxy:9150",
                    "--head",
                    "--max-time",
                    "60",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)",
                    "https://web.archive.org/save/https://blog.jakesaunders.dev/my-server-started-mining-monero-this-morning/"
                ],
                "cmd_version": "8.10.1",
                "end_ts": "2025-12-19T05:21:49.202236+00:00",
                "index_texts": null,
                "output": "TimeoutExpired: Command '['/usr/bin/curl', '--silent', '--location', '--compressed', '--proxy', 'socks5://tor-socks-proxy:9150', '--head', '--max-time', '60', '--user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)', 'https://web.archive.org/save/https://blog.jakesaunders.dev/my-server-started-mining-monero-this-morning/']' timed out after 60 seconds",
                "pwd": "/data/archive/1766121593.807781",
                "schema": "ArchiveResult",
                "start_ts": "2025-12-19T05:20:49.134366+00:00",
                "status": "failed"
            }
        ],
        "dom": [
            {
                "cmd": [
                    "/usr/bin/chromium-browser",
                    "--proxy-server=socks5://tor-socks-proxy:9150",
                    "--disable-features=DarkMode",
                    "--run-all-compositor-stages-before-draw",
                    "--hide-scrollbars",
                    "--autoplay-policy=no-user-gesture-required",
                    "--no-first-run",
                    "--use-fake-ui-for-media-stream",
                    "--use-fake-device-for-media-stream",
                    "--simulate-outdated-no-au='Tue, 31 Dec 2099 23:59:59 GMT'",
                    "--headless=new",
                    "--no-sandbox",
                    "--no-zygote",
                    "--disable-dev-shm-usage",
                    "--disable-software-rasterizer",
                    "--disable-sync",
                    "--window-size=1440,2000",
                    "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)",
                    "--user-data-dir=/data/personas/Default/chrome_profile",
                    "--profile-directory=Default",
                    "--dump-dom",
                    "https://blog.jakesaunders.dev/my-server-started-mining-monero-this-morning/"
                ],
                "cmd_version": "131.0.6778",
                "end_ts": "2025-12-19T05:20:27.990192+00:00",
                "index_texts": null,
                "output": "output.html",
                "pwd": "/data/archive/1766121593.807781",
                "schema": "ArchiveResult",
                "start_ts": "2025-12-19T05:20:13.773604+00:00",
                "status": "succeeded"
            }
        ],
        "favicon": [
            {
                "cmd": [
                    "/usr/bin/curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--proxy",
                    "socks5://tor-socks-proxy:9150",
                    "--max-time",
                    "60",
                    "--output",
                    "favicon.ico",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)",
                    "https://www.google.com/s2/favicons?domain=blog.jakesaunders.dev"
                ],
                "cmd_version": "8.10.1",
                "end_ts": "2025-12-19T05:20:04.482552+00:00",
                "index_texts": null,
                "output": "favicon.ico",
                "pwd": "/data/archive/1766121593.807781",
                "schema": "ArchiveResult",
                "start_ts": "2025-12-19T05:20:01.106499+00:00",
                "status": "succeeded"
            }
        ],
        "git": [],
        "headers": [
            {
                "cmd": [
                    "/usr/bin/curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--proxy",
                    "socks5://tor-socks-proxy:9150",
                    "--head",
                    "--max-time",
                    "60",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)",
                    "https://blog.jakesaunders.dev/my-server-started-mining-monero-this-morning/"
                ],
                "cmd_version": "8.10.1",
                "end_ts": "2025-12-19T05:20:04.642980+00:00",
                "index_texts": null,
                "output": "headers.json",
                "pwd": "/data/archive/1766121593.807781",
                "schema": "ArchiveResult",
                "start_ts": "2025-12-19T05:20:04.562582+00:00",
                "status": "succeeded"
            }
        ],
        "htmltotext": [
            {
                "cmd": [
                    "(internal) archivebox.extractors.htmltotext",
                    "./{singlefile,dom}.html"
                ],
                "cmd_version": "0.8.5rc51",
                "end_ts": "2025-12-19T05:20:42.799735+00:00",
                "index_texts": [
                    "(/assets/images/logo.png) I got hacked, my server started mining Monero this morning. | Unfinished Side Projects I got hacked, my server started mining Monero this morning. | Unfinished Side Projects (https://blog.jakesaunders.dev/my-server-started-mining-monero-this-morning/) (https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css) (/assets/css/screen.css) (/assets/css/main.css)  (/) (Unfinished Side Projects)    (/index.html) Blog  (/about) About  (https://github.com/JakeWritesCode)  GitHub  (https://www.linkedin.com/in/jake-saunders-83617741/)  LinkedIn  () (Type and enter...)        Unfinished Side Projects Jake Saunders personal blog.   I got hacked, my server started mining Monero this morning.  (I got hacked, my server started mining Monero this morning.) Edit: This got way more attention than I was ever expecting. I originally asked claude to draft it from a transcript\nof my panicked messages and the feedback was clear, nobody likes AI slop. As a result I\u2019ve redrafted it this morning\nto fix some inaccuracies and make it sound human. - Jake  Or: How I learned that \u201cI don\u2019t use Next.js\u201d doesn\u2019t mean your dependencies don\u2019t use Next.js  8:25 AM: The Email I woke up to this beauty from Hetzner: Dear Mr Jake Saunders,  We have indications that there was an attack from your server.\nPlease take all necessary measures to avoid this in the future and to solve the issue.  We also request that you send a short response to us. This response should contain information about how this could\nhave happened and what you intend to do about it.\nIn the event that the following steps are not completed successfully, your server can be blocked at any time after the\n2025-12-17 12:46:15 +0100.  Attached was evidence of network scanning from my server to some IP range in Thailand. Great. Nothing says \u201cgood\nmorning\u201d like an abuse report and the threat of getting your infrastructure shut down in 4 hours.  Background: I run a Hetzner server with Coolify. It runs all my stuff , like my little corner of the internet: (https://inventronix.club/connect) My IoT Side Project  This Blog Analytics My dads site (he\u2019s an electrician)  8:30 AM: Oh Fuck First thing I did was SSH in and check the load average: 1\n2   $  w\n 08:25:17 up 55 days, 17:23,  5 users ,  load average: 15.35, 15.44, 15.60          I run a bunch of Go backend services and some SvelteKit frontend stuff on there. My grand total of daily users peaks at\n20, so something was very wrong. I ran ps aux to see what was eating my CPU: 1\n2\n3\n4\n5\n6   USER         PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND\n1001      714822  819  3.6 2464788 2423424 ?     Sl   Dec16 9385:36 /tmp/.XIN-unix/javae\n1001       35035  760  0.0      0     0 ?        Z    Dec14 31638:25 [javae] <defunct>\n1001     3687838  586  0.0      0     0 ?        Z    Dec07 82103:58 [runnv] <defunct>\n1001     4011270  125  0.0      0     0 ?        Z    Dec11 10151:54 [xmrig] <defunct>\n1001       35652 62.3  0.0      0     0 ?        Z    Dec12 4405:17 [xmrig] <defunct>          819% CPU usage. On a process called javae running from /tmp/.XIN-unix/ . And multiple xmrig processes - that\u2019s\nliterally cryptocurrency mining software (Monero, specifically). Looks like I\u2019d been mining cryptocurrency for someone since December 7th. For ten days . Brilliant. The Investigation My first thought was \u201cI\u2019m completely fucked.\u201d My host had been running a crypto miner for a week, the whole think was\nborked. Time to just nuke it from orbit and rebuild. Fortunately, I had the foresight to do a little detective work beforehand to at least learn how I\u2019d been compromised so\nI could learn for the future. I set out to do this with the help of Claude (this is not my speciality). First, I noticed something interesting. All these processes were running as user 1001 . Not root. Not a system user.\nUID 1001. Let me check what\u2019s actually running: 1   $  docker ps          1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n21\n22\n23\n24\n25   CONTAINER ID   IMAGE                                                               CREATED       STATUS                 PORTS            NAMES\nc604f579efd5   dsw80g4w8g0kgog8oskc0sks:63e3be6167b43de47663445dd72f92f97887b843   2 days ago    Up 2 days ( healthy) [ DELETED]       dsw80g4w8g0kgog8oskc0sks-075301203997\n00aec82c2650   o4wk8gsckwgkcgcgkcw8gcsc:40497e7208602d31d7b5e58af4f2e86611b9850c   2 days ago    Up 2 days [ DELETED]       o4wk8gsckwgkcgcgkcw8gcsc-072326337252\na42f72cb1bc5   ghcr.io/umami-software/umami:postgresql-latest                      9 days ago    Up 9 days ( healthy) [ DELETED]       umami-bkc4kkss848cc4kw4gkw8s44\n7c365a792902   postgres:16-alpine                                                  9 days ago    Up 9 days ( healthy) [ DELETED]       postgresql-bkc4kkss848cc4kw4gkw8s44\naf077d142471   ghcr.io/coollabsio/coolify:4.0.0-beta.452                           10 days ago   Up 10 days ( healthy) [ DELETED]       coolify\nfdc3cc9b926b   ghcr.io/coollabsio/coolify-realtime:1.0.10                          10 days ago   Up 10 days ( healthy) [ DELETED]       coolify-realtime\nd3dc2af3ff4d   postgres:15-alpine                                                  10 days ago   Up 10 days ( healthy) [ DELETED]       coolify-db\ndc77adba40bb   redis:7-alpine                                                      10 days ago   Up 10 days ( healthy) [ DELETED]       coolify-redis\n4962dd18bed7   ghcr.io/coollabsio/sentinel:0.0.18                                  3 weeks ago   Up 7 hours ( healthy) [ DELETED]       coolify-sentinel\n5ec997e35140   nginx:stable-alpine                                                 6 weeks ago   Up 6 weeks ( healthy) [ DELETED]       kcwsosksw084swoog04g0w0k-proxy\n5da5e2f2052b   prom/prometheus:latest                                              6 weeks ago   Up 6 weeks [ DELETED]       yg400wo4wok8k0cgo8844gcg-155648790718\n32815a5e2e52   twakedrive/tdrive-frontend                                          7 weeks ago   Up 7 weeks [ DELETED]       frontend-ssowscwgccgk8k0k8oos8w40-120609116307\n5d6bc828fe7f   twakedrive/tdrive-node                                              7 weeks ago   Up 7 weeks [ DELETED]       tdrive_node-ssowscwgccgk8k0k8oos8w40-120609108796\n3e727b84415d   mongo                                                               7 weeks ago   Up 7 weeks [ DELETED]       mongo-ssowscwgccgk8k0k8oos8w40-120609102533\n3506728b808b   a4c00g0ggkk4cww4scsw8scw:682dfd679845535f873d3c5b4599295f4d855ba5   7 weeks ago   Up 7 weeks [ DELETED]       a4c00g0ggkk4cww4scsw8scw-113711308615\n736d9f03d152   rccwscgosk48gs0844sogsgw:51d68c7e7665371569aacc5f044c82ec1f06fa4c   7 weeks ago   Up 7 weeks [ DELETED]       rccwscgosk48gs0844sogsgw-111702410410\n8f79e6f4c981   grafana/grafana-oss                                                 7 weeks ago   Up 7 weeks ( healthy) [ DELETED]       grafana-ik8wokwgowow8gksok8k40sc\n09d013497f9f   24a90047f2d2                                                        7 weeks ago   Up 7 weeks ( healthy) [ DELETED]       postgresql-ik8wokwgowow8gksok8k40sc\nbf8b6a969b19   gcr.io/cadvisor/cadvisor:latest                                     7 weeks ago   Up 7 weeks ( healthy) [ DELETED]       k0gkw4koc8swo4wkg44w408g-211926055160\n30e4d6edf675   prom/node-exporter:latest                                           7 weeks ago   Up 7 weeks [ DELETED]       yc4c4ckg80ogggc4ck8gwgww-211604215046\nb227504e8787   rabbitmq:3-management                                               7 weeks ago   Up 7 weeks ( healthy) [ DELETED]       rabbitmq-xscowck8kgc0wssokoggcskc\nb260ad24c434   d741b3768746                                                        7 weeks ago   Up 7 weeks ( healthy) [ DELETED]       kcwsosksw084swoog04g0w0k\n6d038254e9ef   grafana/loki:latest                                                 7 weeks ago   Up 7 weeks [ DELETED]       b88cwo8ckwo0gw840oo444kk-193205274080\nfe2aad5d9704   traefik:v3.1                                                        7 weeks ago   Up 7 weeks ( healthy) [ DELETED]       coolify-proxy          Note: Deleted ports from this list as i feel like the might expose some inner workings.  Crucially, I was running Umami - a privacy-focused analytics tool I\u2019d re-deployed 9 days ago to track traffic on my\nblog. I redeployed it because it had started acting up and I wasn\u2019t sure why. The timing was suspicious to me. Let me check which container has user 1001: 1\n2\n3\n4   $  docker ps -q | while  read  container; do  echo \"===  $container ===\" docker exec $container ls -la /app/node_modules/next/dist/server/lib/ 2>/dev/null | grep  xmrig done          Output: 1\n2   === a42f72cb1bc5 ===\ndrwxr-xr-x    2 nextjs   nogroup       4096 Dec 17 05:11 xmrig-6.24.0          There it is. Container a42f72cb1bc5 - that\u2019s my Umami analytics container. And it\u2019s got a whole xmrig-6.24.0 directory sitting in what should be Next.js server internals. The mining command in the process list confirmed it: 1\n2\n3\n4\n5   /app/node_modules/next/dist/server/lib/xmrig-6.24.0/xmrig \n  --url auto.c3pool.org:443 \n  --user 8Bt9BEG98SbBPNTp1svQtDQs7PMztqzGoNQHo58eaUYdf8apDkbzp8HbLJH89fMzzciFQ7fb4ZiqUbymDZR6S9asKHZR6wn \n  --pass WUZHRkYOHh1GW1RZWBxaWENRX0ZBWVtdSRxQWkBWHg== \n  --donate-level 0          Someone had exploited my analytics container and was mining Monero using my CPU. Nice. Wait, I Don\u2019t Use Next.js I\u2019d actually seen a post on HN referencing this (https://www.reddit.com/r/nextjs/comments/1pgiaj3/i_got_hacked_and_traced_how_much_money_hacker/) Reddit post about a\ncritical Next.js (CVE-2025-66478). My immediate reaction was \u201clol who cares, I don\u2019t run Next.js.\u201d Oh my sweet summer child.  Except\u2026 Umami is built with Next.js . I did not know this, nor did I bother looking. Oops. The vulnerability (CVE-2025-66478) was in Next.js\u2019s React Server Components deserialization. The \u201cFlight\u201d protocol that\nRSC uses to serialize/deserialize data between client and server had an unsafe deserialization flaw. An attacker could\nsend a specially crafted HTTP request with a malicious payload to any App Router endpoint, and when deserialized, it\nwould execute arbitrary code on the server. Attacker sends crafted HTTP request to Umami\u2019s Next.js endpoint RSC deserializes the malicious payload RCE achieved via unsafe deserialization Download and install cryptominers Profit (for them)  So much for \u201cI don\u2019t use Next.js.\u201d The Panic: Has It Escaped the Container? This is where I started to properly panic. Looking at that process list: 1   1001      714822  819  3.6 2464788 2423424 ?     Sl   Dec16 9385:36 /tmp/.XIN-unix/javae          That path - /tmp/.XIN-unix/javae - looks like it\u2019s on the host filesystem , not inside a container. That means it\ncan get access to my database, all my environment variables, the works. Claude was telling me I\u2019d need to: Assume everything is compromised Check for rootkits, backdoors, persistence mechanisms Probably rebuild from scratch Spend my entire day unfucking this  I checked for persistence mechanisms: 1\n2\n3\n4\n5   $  crontab -l no crontab for  root $  systemctl list-unit-files | grep  enabled # ... all legitimate system services, nothing suspicious          No malicious cron jobs. No fake systemd services pretending to be nginxs or apaches . That\u2019s\u2026 good? But I still needed to know: Did the malware actually escape the container or not?  The Moment of Truth The test was, if /tmp/.XIN-unix/javae exists on the host, I\u2019m fucked. If it doesn\u2019t exist, then apparently what I\u2019m\nseeing is\njust Docker\u2019s default behavior of showing container processes in the host\u2019s ps output, but they\u2019re actually isolated. 1\n2   $  ls -la /tmp/.XIN-unix/javae ls : cannot access '/tmp/.XIN-unix/javae' : No such file or directory          IT NEVER ESCAPED.  Or at least it doesn\u2019t look like it. We can downgrade this incident from DEFCON1 to \u2018point a gun at it and do some more\nchecks, but no guillotine yet\u2019. The malware was entirely contained within the Umami container. Apparently, when you run ps aux on a Docker host, you\nsee processes\nfrom all containers because they share the same kernel. But those processes are in their own mount namespace - they\ncan\u2019t see or touch the host filesystem. I verified what user that container was actually running as: 1\n2\n3\n4\n5\n6\n7\n8   $  docker inspect umami-bkc4kkss848cc4kw4gkw8s44 | grep '\"User\"' \"User\" : \"nextjs\" , $  docker inspect umami-bkc4kkss848cc4kw4gkw8s44 | grep '\"Privileged\"' \"Privileged\" : false , $  docker inspect umami-bkc4kkss848cc4kw4gkw8s44 | grep -A 30 \"Mounts\" \"Mounts\" : [] ,          So here\u2019s what I now know, and why I\u2019m not totally fucked:  Container ran as user nextjs (UID 1001), not root. Container was not privileged. Container had zero volume mounts .  Which means: Run processes inside the container Mine cryptocurrency Scan networks (hence the Hetzner abuse report) Consume 100% CPU  The malware could NOT: Access the host filesystem Install cron jobs Create systemd services Persist across container restarts Escape to other containers Install rootkits  Container isolation actually worked. Nice. Dockerfiles vs. Auto-Generated Images There were a couple of things which saved me in this case IMO compared to the Reddit post I linked: I write all my own dockerfiles for my applications. This isn;t a silver bullet on it\u2019s own but compared to\nautogenerated ones you have a better idea of what\u2019s in there. Coolify and Dockers approach to containerization in general. I\u2019ve since learned that we can\u2019t rely on container\nseparation for security but honestly it seems better than running everything on the host.  The Reddit post I\u2019d seen earlier? That guy got completely screwed because his container was running as root. The malware\ncould: Install cron jobs for persistence Create systemd services Write anywhere on the filesystem Survive reboots  So in this case, container isolation had worked! What I did not do, was keep track of the tolling I was using and what tooling that was using. In fact, I installed\nUmami from Coolify\u2019s services screen. I didn\u2019t even configure it. Obviously none of this is Umami\u2019s fault by the way. They released a fix for their free software like a week ago. I\njust\ndidn\u2019t think to do anything about it.  The Fix 1\n2\n3\n4\n5\n6\n7   # Stop and remove the compromised container $  docker stop umami-bkc4kkss848cc4kw4gkw8s44 $  docker rm  umami-bkc4kkss848cc4kw4gkw8s44 # Check CPU usage $  uptime  08:45:17 up 55 days, 17:43,  1 user,  load average: 0.52, 1.24, 4.83          CPU back to normal. It\u2019s been two days since and my CPU is just chilling at like 5%. I also enabled UFW (which I should have done ages ago): 1\n2\n3\n4\n5\n6   $  sudo  ufw default deny incoming $  sudo  ufw default allow outgoing $  sudo  ufw allow ssh $  sudo  ufw allow 80/tcp $  sudo  ufw allow 443/tcp $  sudo  ufw enable          This blocks all inbound connections except SSH, HTTP, and HTTPS. No more exposed PostgreSQL ports, no more RabbitMQ\nports open to the internet. In my mind this shouldn\u2019t be too big a deal because 5432 wasn\u2019t open to he host from the\ndocker container. But worth doing. I sent Hetzner a brief explanation: Investigation complete. The scanning originated from a compromised Umami analytics container (CVE-2025-66478). The container ran as non-root user with no privileged access or host mounts, so the compromise was fully contained.\nContainer has been removed and firewall hardened.  They closed the ticket within an hour. Lessons Learned 1. \u201cI don\u2019t use X\u201d doesn\u2019t mean your dependencies don\u2019t use X I don\u2019t write Next.js applications. But I run third-party tools that are built with Next.js. When CVE-2025-66478 was\ndisclosed, I thought \u201cnot my problem.\u201d Wrong. Know what your dependencies are actually built with. That \u201csimple analytics tool\u201d is a full web application with a\ncomplex stack. 2. Container isolation works (when configured properly) This could have been so much worse. If that container had been running as root, or had volume mounts to sensitive\ndirectories, or had access to the Docker socket, I\u2019d be writing a very different blog post about rebuilding my entire\ninfrastructure. Instead, I deleted one container and moved on with my day. Write your own Dockerfiles. Understand what user your processes run as. Avoid USER root unless you have a very\ngood reason. Don\u2019t mount volumes you don\u2019t need. Don\u2019t give containers --privileged access. 3. The sophistication gap This malware wasn\u2019t like those people who auto-poll for /wpadmin every time I make a DNS change. This was spicy. Disguised itself in legitimate-looking paths (/app/node_modules/next/dist/server/lib/ ) Used process names that blend in (javae , runnv ) Attempted to establish persistence According to other reports, even had \u201ckiller scripts\u201d to murder competing miners  But it was still limited by container isolation. Good security practices beat sophisticated malware. 4. Defense in depth matters Even though the container isolation held, I still should have: Had a firewall enabled from day one (not \u201cI\u2019ll do it later\u201d) Been running fail2ban to stop those SSH brute force attempts Had proper monitoring/alerting (I only noticed because of the Hetzner email) Updated Umami when the CVE was disclosed  I got lucky. Container isolation saved me from my own laziness. What I\u2019m Doing Differently No more Umami. - Meh, I\u2019ve gone back on this. This wasn\u2019t Umami\u2019s fault, and their open source software is super\ncool. I\u2019ve rebooted a fresh version of Umami.  Audit all third-party containers. Going through everything I run and checking: What user does it run as? What volumes does it have? When was it last updated? Do I actually need it?   SSH hardening. Moving to key-based authentication only, disabling password auth, and setting up fail2ban.  Proper monitoring. Setting up alerts for CPU usage, load average, and suspicious network activity. I shouldn\u2019t\nfind out about compromises from my hosting provider. I actually have grafana and Node exporter set up, but it\u2019s not\ngood unless I go look at it!  Regular security updates. No more \u201cI\u2019ll update it later.\u201d If there\u2019s a CVE, I patch or I remove the service.  The Silver Lining This was actually a pretty good learning experience. I got to: Practice incident response on a real compromise (never done this before!) Prove that container isolation actually works Learn about Docker namespaces, user mapping, and privilege boundaries Harden my infrastructure without the pressure of active data loss  And I only lost about 2 hours of my morning before work. Could\u2019ve been way worse. Though I do wonder how much Monero I mined for that dickhead. Based on the CPU usage and duration\u2026 probably enough for\nthem to have a nice lunch. You\u2019re welcome, mysterious attacker. Hope you enjoyed it. TL;DR Umami analytics (built with Next.js) had an RCE vulnerability. Got exploited, installed cryptominers Mined Monero for 10 days at 1000%+ CPU Container isolation saved me because it ran as non-root with no mounts Fix: docker rm umami and enable firewall Lesson: Know what your dependencies are built with, and configure containers properly   17 Dec 2025    (/categories#Self-Hosting) Self Hosting  (/categories#Work) Work    (/tags#Engineering) #Engineering  (/tags#Hetzner) #Hetzner  (/tags#Self-Hosting) #Self Hosting  (/tags#Software-Development) #Software Development    (//fix-google-sitemap-could-not-be-read/) \u00ab Fix for google search console 'sitemap could not be read'       Please enable JavaScript to view the (http://disqus.com/?ref_noscript) comments powered by Disqus.  (http://disqus.com) comments powered by Disqus        Explore \u2192    (/categories#Work) Work (6) (/categories#Ramblings) Ramblings (3) (/categories#Projects) Projects (2) (/categories#Arduino) Arduino (2) (/categories#Hydroponics) Hydroponics (2) (/categories#Rant) Rant (1) (/categories#Webdev) Webdev (2) (/categories#Guides) Guides (2) (/categories#Software-Development) Software Development (2) (/categories#DIY) DIY (1) (/categories#Self-Hosting) Self Hosting (1)    Copyright \u00a9 2025 Unfinished Side Projects  (https://www.wowthemes.net/mediumish-free-jekyll-template/) Mediumish Jekyll\n                        Theme by WowThemes.net     (https://fonts.googleapis.com/css?family=Righteous%7CMerriweather:300,300i,400,400i,700,700i) (https://use.fontawesome.com/releases/v5.0.13/css/all.css)    "
                ],
                "output": "htmltotext.txt",
                "pwd": "/data/archive/1766121593.807781",
                "schema": "ArchiveResult",
                "start_ts": "2025-12-19T05:20:42.736798+00:00",
                "status": "succeeded"
            }
        ],
        "media": [
            {
                "cmd": [
                    "/usr/local/bin/yt-dlp",
                    "--restrict-filenames",
                    "--trim-filenames",
                    "128",
                    "--write-description",
                    "--write-info-json",
                    "--write-annotations",
                    "--write-thumbnail",
                    "--no-call-home",
                    "--write-sub",
                    "--write-auto-subs",
                    "--convert-subs=srt",
                    "--yes-playlist",
                    "--continue",
                    "--no-abort-on-error",
                    "--ignore-errors",
                    "--geo-bypass",
                    "--add-metadata",
                    "--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)",
                    "--skip-download",
                    "--cache-dir=/data/yt-dlp-cache/",
                    "--cookies=/data/yt-dlp-cache/cookies.txt",
                    "--proxy=socks5://tor-socks-proxy:9150",
                    "--no-playlist",
                    "https://blog.jakesaunders.dev/my-server-started-mining-monero-this-morning/"
                ],
                "cmd_version": "2024.10.7",
                "end_ts": "2025-12-19T05:20:49.100014+00:00",
                "index_texts": [],
                "output": "media/",
                "pwd": "/data/archive/1766121593.807781",
                "schema": "ArchiveResult",
                "start_ts": "2025-12-19T05:20:45.069379+00:00",
                "status": "succeeded"
            }
        ],
        "mercury": [
            {
                "cmd": [
                    "/home/archivebox/.npm/bin/postlight-parser",
                    "https://blog.jakesaunders.dev/my-server-started-mining-monero-this-morning/"
                ],
                "cmd_version": "2.2.3",
                "end_ts": "2025-12-19T05:20:42.706726+00:00",
                "index_texts": null,
                "output": "mercury/",
                "pwd": "/data/archive/1766121593.807781",
                "schema": "ArchiveResult",
                "start_ts": "2025-12-19T05:20:39.923455+00:00",
                "status": "succeeded"
            }
        ],
        "pdf": [],
        "readability": [
            {
                "cmd": [
                    "/home/archivebox/.npm/bin/readability-extractor",
                    "/tmp/tmpr10tayck",
                    "https://blog.jakesaunders.dev/my-server-started-mining-monero-this-morning/"
                ],
                "cmd_version": "0.0.11",
                "end_ts": "2025-12-19T05:20:31.969213+00:00",
                "index_texts": [
                    "I got hacked, my server started mining Monero this morning.\n            \n                \n                \n                \n                \n  Edit: This got way more attention than I was ever expecting. I originally asked claude to draft it from a transcript\nof my panicked messages and the feedback was clear, nobody likes AI slop. As a result I\u2019ve redrafted it this morning\nto fix some inaccuracies and make it sound human. - Jake\n\n\nOr: How I learned that \u201cI don\u2019t use Next.js\u201d doesn\u2019t mean your dependencies don\u2019t use Next.js\n\n8:25 AM: The Email\n\nI woke up to this beauty from Hetzner:\n\n\n  Dear Mr Jake Saunders,\n\n\n\n  We have indications that there was an attack from your server.\nPlease take all necessary measures to avoid this in the future and to solve the issue.\n\n\n\n  We also request that you send a short response to us. This response should contain information about how this could\nhave happened and what you intend to do about it.\nIn the event that the following steps are not completed successfully, your server can be blocked at any time after the\n2025-12-17 12:46:15 +0100.\n\n\n\n  Attached was evidence of network scanning from my server to some IP range in Thailand. Great. Nothing says \u201cgood\nmorning\u201d like an abuse report and the threat of getting your infrastructure shut down in 4 hours.\n\n\nBackground: I run a Hetzner server with Coolify. It runs all my stuff, like my little corner of the internet:\n\n\n  My IoT Side Project\n  This Blog\n  Analytics\n  My dads site (he\u2019s an electrician)\n\n\n8:30 AM: Oh Fuck\n\nFirst thing I did was SSH in and check the load average:\n\n1\n2\n$ w\n 08:25:17 up 55 days, 17:23,  5 users,  load average: 15.35, 15.44, 15.60\n\n\nI run a bunch of Go backend services and some SvelteKit frontend stuff on there. My grand total of daily users peaks at\n20, so something was very wrong.\n\nI ran ps aux to see what was eating my CPU:\n\n1\n2\n3\n4\n5\n6\nUSER         PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND\n1001      714822  819  3.6 2464788 2423424 ?     Sl   Dec16 9385:36 /tmp/.XIN-unix/javae\n1001       35035  760  0.0      0     0 ?        Z    Dec14 31638:25 [javae] <defunct>\n1001     3687838  586  0.0      0     0 ?        Z    Dec07 82103:58 [runnv] <defunct>\n1001     4011270  125  0.0      0     0 ?        Z    Dec11 10151:54 [xmrig] <defunct>\n1001       35652 62.3  0.0      0     0 ?        Z    Dec12 4405:17 [xmrig] <defunct>\n\n\n819% CPU usage. On a process called javae running from /tmp/.XIN-unix/. And multiple xmrig processes - that\u2019s\nliterally cryptocurrency mining software (Monero, specifically).\n\nLooks like I\u2019d been mining cryptocurrency for someone since December 7th. For ten days. Brilliant.\n\nThe Investigation\n\nMy first thought was \u201cI\u2019m completely fucked.\u201d My host had been running a crypto miner for a week, the whole think was\nborked. Time to just nuke it from orbit and rebuild.\n\nFortunately, I had the foresight to do a little detective work beforehand to at least learn how I\u2019d been compromised so\nI could learn for the future. I set out to do this with the help of Claude (this is not my speciality).\n\nFirst, I noticed something interesting. All these processes were running as user 1001. Not root. Not a system user.\nUID 1001.\n\nLet me check what\u2019s actually running:\n\n\n\n1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n21\n22\n23\n24\n25\nCONTAINER ID   IMAGE                                                               CREATED       STATUS                 PORTS            NAMES\nc604f579efd5   dsw80g4w8g0kgog8oskc0sks:63e3be6167b43de47663445dd72f92f97887b843   2 days ago    Up 2 days (healthy)     [DELETED]       dsw80g4w8g0kgog8oskc0sks-075301203997\n00aec82c2650   o4wk8gsckwgkcgcgkcw8gcsc:40497e7208602d31d7b5e58af4f2e86611b9850c   2 days ago    Up 2 days               [DELETED]       o4wk8gsckwgkcgcgkcw8gcsc-072326337252\na42f72cb1bc5   ghcr.io/umami-software/umami:postgresql-latest                      9 days ago    Up 9 days (healthy)     [DELETED]       umami-bkc4kkss848cc4kw4gkw8s44\n7c365a792902   postgres:16-alpine                                                  9 days ago    Up 9 days (healthy)     [DELETED]       postgresql-bkc4kkss848cc4kw4gkw8s44\naf077d142471   ghcr.io/coollabsio/coolify:4.0.0-beta.452                           10 days ago   Up 10 days (healthy)    [DELETED]       coolify\nfdc3cc9b926b   ghcr.io/coollabsio/coolify-realtime:1.0.10                          10 days ago   Up 10 days (healthy)    [DELETED]       coolify-realtime\nd3dc2af3ff4d   postgres:15-alpine                                                  10 days ago   Up 10 days (healthy)    [DELETED]       coolify-db\ndc77adba40bb   redis:7-alpine                                                      10 days ago   Up 10 days (healthy)    [DELETED]       coolify-redis\n4962dd18bed7   ghcr.io/coollabsio/sentinel:0.0.18                                  3 weeks ago   Up 7 hours (healthy)    [DELETED]       coolify-sentinel\n5ec997e35140   nginx:stable-alpine                                                 6 weeks ago   Up 6 weeks (healthy)    [DELETED]       kcwsosksw084swoog04g0w0k-proxy\n5da5e2f2052b   prom/prometheus:latest                                              6 weeks ago   Up 6 weeks              [DELETED]       yg400wo4wok8k0cgo8844gcg-155648790718\n32815a5e2e52   twakedrive/tdrive-frontend                                          7 weeks ago   Up 7 weeks              [DELETED]       frontend-ssowscwgccgk8k0k8oos8w40-120609116307\n5d6bc828fe7f   twakedrive/tdrive-node                                              7 weeks ago   Up 7 weeks              [DELETED]       tdrive_node-ssowscwgccgk8k0k8oos8w40-120609108796\n3e727b84415d   mongo                                                               7 weeks ago   Up 7 weeks              [DELETED]       mongo-ssowscwgccgk8k0k8oos8w40-120609102533\n3506728b808b   a4c00g0ggkk4cww4scsw8scw:682dfd679845535f873d3c5b4599295f4d855ba5   7 weeks ago   Up 7 weeks              [DELETED]       a4c00g0ggkk4cww4scsw8scw-113711308615\n736d9f03d152   rccwscgosk48gs0844sogsgw:51d68c7e7665371569aacc5f044c82ec1f06fa4c   7 weeks ago   Up 7 weeks              [DELETED]       rccwscgosk48gs0844sogsgw-111702410410\n8f79e6f4c981   grafana/grafana-oss                                                 7 weeks ago   Up 7 weeks (healthy)    [DELETED]       grafana-ik8wokwgowow8gksok8k40sc\n09d013497f9f   24a90047f2d2                                                        7 weeks ago   Up 7 weeks (healthy)    [DELETED]       postgresql-ik8wokwgowow8gksok8k40sc\nbf8b6a969b19   gcr.io/cadvisor/cadvisor:latest                                     7 weeks ago   Up 7 weeks (healthy)    [DELETED]       k0gkw4koc8swo4wkg44w408g-211926055160\n30e4d6edf675   prom/node-exporter:latest                                           7 weeks ago   Up 7 weeks              [DELETED]       yc4c4ckg80ogggc4ck8gwgww-211604215046\nb227504e8787   rabbitmq:3-management                                               7 weeks ago   Up 7 weeks (healthy)    [DELETED]       rabbitmq-xscowck8kgc0wssokoggcskc\nb260ad24c434   d741b3768746                                                        7 weeks ago   Up 7 weeks (healthy)    [DELETED]       kcwsosksw084swoog04g0w0k\n6d038254e9ef   grafana/loki:latest                                                 7 weeks ago   Up 7 weeks              [DELETED]       b88cwo8ckwo0gw840oo444kk-193205274080\nfe2aad5d9704   traefik:v3.1                                                        7 weeks ago   Up 7 weeks (healthy)    [DELETED]       coolify-proxy\n\n\n\n  Note: Deleted ports from this list as i feel like the might expose some inner workings.\n\n\nCrucially, I was running Umami - a privacy-focused analytics tool I\u2019d re-deployed 9 days ago to track traffic on my\nblog. I redeployed it because it had started acting up and I wasn\u2019t sure why. The timing was suspicious to me.\n\nLet me check which container has user 1001:\n\n1\n2\n3\n4\n$ docker ps -q | while read container; do\n  echo \"=== $container ===\"\n  docker exec $container ls -la /app/node_modules/next/dist/server/lib/ 2>/dev/null | grep xmrig\ndone\n\n\nOutput:\n\n1\n2\n=== a42f72cb1bc5 ===\ndrwxr-xr-x    2 nextjs   nogroup       4096 Dec 17 05:11 xmrig-6.24.0\n\n\nThere it is. Container a42f72cb1bc5 - that\u2019s my Umami analytics container. And it\u2019s got a whole xmrig-6.24.0\ndirectory sitting in what should be Next.js server internals.\n\nThe mining command in the process list confirmed it:\n\n1\n2\n3\n4\n5\n/app/node_modules/next/dist/server/lib/xmrig-6.24.0/xmrig \n  --url auto.c3pool.org:443 \n  --user 8Bt9BEG98SbBPNTp1svQtDQs7PMztqzGoNQHo58eaUYdf8apDkbzp8HbLJH89fMzzciFQ7fb4ZiqUbymDZR6S9asKHZR6wn \n  --pass WUZHRkYOHh1GW1RZWBxaWENRX0ZBWVtdSRxQWkBWHg== \n  --donate-level 0\n\n\nSomeone had exploited my analytics container and was mining Monero using my CPU. Nice.\n\nWait, I Don\u2019t Use Next.js\n\nI\u2019d actually seen a post on HN referencing this\nReddit post about a\ncritical Next.js (CVE-2025-66478). My immediate reaction was \u201clol who cares, I don\u2019t run Next.js.\u201d\n\n\n  Oh my sweet summer child.\n\n\nExcept\u2026 Umami is built with Next.js. I did not know this, nor did I bother looking. Oops.\n\nThe vulnerability (CVE-2025-66478) was in Next.js\u2019s React Server Components deserialization. The \u201cFlight\u201d protocol that\nRSC uses to serialize/deserialize data between client and server had an unsafe deserialization flaw. An attacker could\nsend a specially crafted HTTP request with a malicious payload to any App Router endpoint, and when deserialized, it\nwould execute arbitrary code on the server.\n\n\n  Attacker sends crafted HTTP request to Umami\u2019s Next.js endpoint\n  RSC deserializes the malicious payload\n  RCE achieved via unsafe deserialization\n  Download and install cryptominers\n  Profit (for them)\n\n\nSo much for \u201cI don\u2019t use Next.js.\u201d\n\nThe Panic: Has It Escaped the Container?\n\nThis is where I started to properly panic. Looking at that process list:\n\n1\n1001      714822  819  3.6 2464788 2423424 ?     Sl   Dec16 9385:36 /tmp/.XIN-unix/javae\n\n\nThat path - /tmp/.XIN-unix/javae - looks like it\u2019s on the host filesystem, not inside a container. That means it\ncan get access to my database, all my environment variables, the works. Claude was telling me I\u2019d need to:\n\n\n  Assume everything is compromised\n  Check for rootkits, backdoors, persistence mechanisms\n  Probably rebuild from scratch\n  Spend my entire day unfucking this\n\n\nI checked for persistence mechanisms:\n\n1\n2\n3\n4\n5\n$ crontab -l\nno crontab for root\n\n$ systemctl list-unit-files | grep enabled\n# ... all legitimate system services, nothing suspicious\n\n\nNo malicious cron jobs. No fake systemd services pretending to be nginxs or apaches. That\u2019s\u2026 good?\n\nBut I still needed to know: Did the malware actually escape the container or not?\n\nThe Moment of Truth\n\nThe test was, if /tmp/.XIN-unix/javae exists on the host, I\u2019m fucked. If it doesn\u2019t exist, then apparently what I\u2019m\nseeing is\njust Docker\u2019s default behavior of showing container processes in the host\u2019s ps output, but they\u2019re actually isolated.\n\n1\n2\n$ ls -la /tmp/.XIN-unix/javae\nls: cannot access '/tmp/.XIN-unix/javae': No such file or directory\n\n\nIT NEVER ESCAPED.\n\nOr at least it doesn\u2019t look like it. We can downgrade this incident from DEFCON1 to \u2018point a gun at it and do some more\nchecks, but no guillotine yet\u2019.\n\nThe malware was entirely contained within the Umami container. Apparently, when you run ps aux on a Docker host, you\nsee processes\nfrom all containers because they share the same kernel. But those processes are in their own mount namespace - they\ncan\u2019t see or touch the host filesystem.\n\nI verified what user that container was actually running as:\n\n1\n2\n3\n4\n5\n6\n7\n8\n$ docker inspect umami-bkc4kkss848cc4kw4gkw8s44 | grep '\"User\"'\n\"User\": \"nextjs\",\n\n$ docker inspect umami-bkc4kkss848cc4kw4gkw8s44 | grep '\"Privileged\"'\n\"Privileged\": false,\n\n$ docker inspect umami-bkc4kkss848cc4kw4gkw8s44 | grep -A 30 \"Mounts\"\n\"Mounts\": [],\n\n\nSo here\u2019s what I now know, and why I\u2019m not totally fucked:\n\n\n  Container ran as user nextjs (UID 1001), not root.\n  Container was not privileged.\n  Container had zero volume mounts.\n\n\nWhich means:\n\n\n  Run processes inside the container\n  Mine cryptocurrency\n  Scan networks (hence the Hetzner abuse report)\n  Consume 100% CPU\n\n\nThe malware could NOT:\n\n\n  Access the host filesystem\n  Install cron jobs\n  Create systemd services\n  Persist across container restarts\n  Escape to other containers\n  Install rootkits\n\n\nContainer isolation actually worked. Nice.\n\nDockerfiles vs. Auto-Generated Images\n\nThere were a couple of things which saved me in this case IMO compared to the Reddit post I linked:\n\n\n  I write all my own dockerfiles for my applications. This isn;t a silver bullet on it\u2019s own but compared to\nautogenerated ones you have a better idea of what\u2019s in there.\n  Coolify and Dockers approach to containerization in general. I\u2019ve since learned that we can\u2019t rely on container\nseparation for security but honestly it seems better than running everything on the host.\n\n\nThe Reddit post I\u2019d seen earlier? That guy got completely screwed because his container was running as root. The malware\ncould:\n\n\n  Install cron jobs for persistence\n  Create systemd services\n  Write anywhere on the filesystem\n  Survive reboots\n\n\nSo in this case, container isolation had worked!\n\nWhat I did not do, was keep track of the tolling I was using and what tooling that was using. In fact, I installed\nUmami from Coolify\u2019s services screen. I didn\u2019t even configure it.\n\n\n  Obviously none of this is Umami\u2019s fault by the way. They released a fix for their free software like a week ago. I\njust\ndidn\u2019t think to do anything about it.\n\n\nThe Fix\n\n1\n2\n3\n4\n5\n6\n7\n# Stop and remove the compromised container\n$ docker stop umami-bkc4kkss848cc4kw4gkw8s44\n$ docker rm umami-bkc4kkss848cc4kw4gkw8s44\n\n# Check CPU usage\n$ uptime\n 08:45:17 up 55 days, 17:43,  1 user,  load average: 0.52, 1.24, 4.83\n\n\nCPU back to normal. It\u2019s been two days since and my CPU is just chilling at like 5%.\n\nI also enabled UFW (which I should have done ages ago):\n\n1\n2\n3\n4\n5\n6\n$ sudo ufw default deny incoming\n$ sudo ufw default allow outgoing\n$ sudo ufw allow ssh\n$ sudo ufw allow 80/tcp\n$ sudo ufw allow 443/tcp\n$ sudo ufw enable\n\n\nThis blocks all inbound connections except SSH, HTTP, and HTTPS. No more exposed PostgreSQL ports, no more RabbitMQ\nports open to the internet. In my mind this shouldn\u2019t be too big a deal because 5432 wasn\u2019t open to he host from the\ndocker container. But worth doing.\n\nI sent Hetzner a brief explanation:\n\n\n  Investigation complete. The scanning originated from a compromised Umami analytics container (CVE-2025-66478).\n\n  The container ran as non-root user with no privileged access or host mounts, so the compromise was fully contained.\nContainer has been removed and firewall hardened.\n\n\nThey closed the ticket within an hour.\n\nLessons Learned\n\n1. \u201cI don\u2019t use X\u201d doesn\u2019t mean your dependencies don\u2019t use X\n\nI don\u2019t write Next.js applications. But I run third-party tools that are built with Next.js. When CVE-2025-66478 was\ndisclosed, I thought \u201cnot my problem.\u201d Wrong.\n\nKnow what your dependencies are actually built with. That \u201csimple analytics tool\u201d is a full web application with a\ncomplex stack.\n\n2. Container isolation works (when configured properly)\n\nThis could have been so much worse. If that container had been running as root, or had volume mounts to sensitive\ndirectories, or had access to the Docker socket, I\u2019d be writing a very different blog post about rebuilding my entire\ninfrastructure.\n\nInstead, I deleted one container and moved on with my day.\n\nWrite your own Dockerfiles. Understand what user your processes run as. Avoid USER root unless you have a very\ngood reason. Don\u2019t mount volumes you don\u2019t need. Don\u2019t give containers --privileged access.\n\n3. The sophistication gap\n\nThis malware wasn\u2019t like those people who auto-poll for /wpadmin every time I make a DNS change. This was spicy.\n\n\n  Disguised itself in legitimate-looking paths (/app/node_modules/next/dist/server/lib/)\n  Used process names that blend in (javae, runnv)\n  Attempted to establish persistence\n  According to other reports, even had \u201ckiller scripts\u201d to murder competing miners\n\n\nBut it was still limited by container isolation. Good security practices beat sophisticated malware.\n\n4. Defense in depth matters\n\nEven though the container isolation held, I still should have:\n\n\n  Had a firewall enabled from day one (not \u201cI\u2019ll do it later\u201d)\n  Been running fail2ban to stop those SSH brute force attempts\n  Had proper monitoring/alerting (I only noticed because of the Hetzner email)\n  Updated Umami when the CVE was disclosed\n\n\nI got lucky. Container isolation saved me from my own laziness.\n\nWhat I\u2019m Doing Differently\n\n\n  \n    No more Umami. - Meh, I\u2019ve gone back on this. This wasn\u2019t Umami\u2019s fault, and their open source software is super\ncool. I\u2019ve rebooted a fresh version of Umami.\n  \n  Audit all third-party containers. Going through everything I run and checking:\n    \n      What user does it run as?\n      What volumes does it have?\n      When was it last updated?\n      Do I actually need it?\n    \n  \n  \n    SSH hardening. Moving to key-based authentication only, disabling password auth, and setting up fail2ban.\n  \n  \n    Proper monitoring. Setting up alerts for CPU usage, load average, and suspicious network activity. I shouldn\u2019t\nfind out about compromises from my hosting provider. I actually have grafana and Node exporter set up, but it\u2019s not\ngood unless I go look at it!\n  \n  Regular security updates. No more \u201cI\u2019ll update it later.\u201d If there\u2019s a CVE, I patch or I remove the service.\n\n\nThe Silver Lining\n\nThis was actually a pretty good learning experience. I got to:\n\n\n  Practice incident response on a real compromise (never done this before!)\n  Prove that container isolation actually works\n  Learn about Docker namespaces, user mapping, and privilege boundaries\n  Harden my infrastructure without the pressure of active data loss\n\n\nAnd I only lost about 2 hours of my morning before work. Could\u2019ve been way worse.\n\nThough I do wonder how much Monero I mined for that dickhead. Based on the CPU usage and duration\u2026 probably enough for\nthem to have a nice lunch. You\u2019re welcome, mysterious attacker. Hope you enjoyed it.\n\nTL;DR\n\n\n  Umami analytics (built with Next.js) had an RCE vulnerability.\n  Got exploited, installed cryptominers\n  Mined Monero for 10 days at 1000%+ CPU\n  Container isolation saved me because it ran as non-root with no mounts\n  Fix: docker rm umami and enable firewall\n  Lesson: Know what your dependencies are built with, and configure containers properly"
                ],
                "output": "readability/",
                "pwd": "/data/archive/1766121593.807781",
                "schema": "ArchiveResult",
                "start_ts": "2025-12-19T05:20:29.156221+00:00",
                "status": "succeeded"
            }
        ],
        "screenshot": [],
        "singlefile": [],
        "title": [
            {
                "cmd": [
                    "/usr/bin/curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--proxy",
                    "socks5://tor-socks-proxy:9150",
                    "--max-time",
                    "60",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)",
                    "https://blog.jakesaunders.dev/my-server-started-mining-monero-this-morning/"
                ],
                "cmd_version": "8.10.1",
                "end_ts": "2025-12-19T05:20:28.265016+00:00",
                "index_texts": null,
                "output": "I got hacked, my server started mining Monero this morning. | Unfinished Side Projects",
                "pwd": "/data/archive/1766121593.807781",
                "schema": "ArchiveResult",
                "start_ts": "2025-12-19T05:20:28.206926+00:00",
                "status": "succeeded"
            }
        ],
        "wget": []
    },
    "icons": null,
    "is_archived": true,
    "is_static": false,
    "latest": {
        "archive_org": "TimeoutExpired: Command '['/usr/bin/curl', '--silent', '--location', '--compressed', '--proxy', 'socks5://tor-socks-proxy:9150', '--head', '--max-time', '60', '--user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)', 'https://web.archive.org/save/https://blog.jakesaunders.dev/my-server-started-mining-monero-this-morning/']' timed out after 60 seconds",
        "dom": "output.html",
        "favicon": "favicon.ico",
        "git": null,
        "media": "media/",
        "pdf": null,
        "screenshot": null,
        "singlefile": null,
        "title": "I got hacked, my server started mining Monero this morning. | Unfinished Side Projects",
        "warc": null,
        "wget": null
    },
    "link_dir": "/data/archive/1766121593.807781",
    "newest_archive_date": "2025-12-19T05:20:49.134366+00:00",
    "num_failures": 1,
    "num_outputs": 8,
    "oldest_archive_date": "2025-12-19T05:20:01.106499+00:00",
    "path": "/my-server-started-mining-monero-this-morning/",
    "schema": "Link",
    "scheme": "https",
    "snapshot_abid": "snp_01KCTGSPZR80B5BF0A012KDZ23",
    "snapshot_id": "93186345-ba25-4bd2-b67e-ce64c536fc43",
    "sources": [
        "/data/sources/1766121592-import.txt"
    ],
    "tags": null,
    "tags_str": "",
    "timestamp": "1766121593.807781",
    "title": "I got hacked, my server started mining Monero this morning. | Unfinished Side Projects",
    "url": "https://blog.jakesaunders.dev/my-server-started-mining-monero-this-morning/"
}