{ "archive_path": "archive/1756879672.881544", "base_url": "specbranch.com/posts/one-big-server", "basename": "", "bookmarked_date": "2025-09-03 06:07", "canonical": { "archive_org_path": "https://web.archive.org/web/specbranch.com/posts/one-big-server", "dom_path": "output.html", "favicon_path": "favicon.ico", "git_path": "git/", "google_favicon_path": "https://www.google.com/s2/favicons?domain=specbranch.com", "headers_path": "headers.json", "htmltotext_path": "htmltotext.txt", "index_path": "index.html", "media_path": "media/", "mercury_path": "mercury/content.html", "pdf_path": "output.pdf", "readability_path": "readability/content.html", "screenshot_path": "screenshot.png", "singlefile_path": "singlefile.html", "warc_path": "warc/", "wget_path": null }, "domain": "specbranch.com", "downloaded_at": "2025-09-03T06:07:58.118283+00:00", "downloaded_datestr": "2025-09-03 06:07", "extension": "", "hash": "1BJ3P4WGXJ96R5JWQK5J", "history": { "archive_org": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--head", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://web.archive.org/save/https://specbranch.com/posts/one-big-server/" ], "cmd_version": "8.10.1", "end_ts": "2025-09-03T06:09:38.229746+00:00", "index_texts": null, "output": "TimeoutExpired: Command '['/usr/bin/curl', '--silent', '--location', '--compressed', '--proxy', 'socks5://tor-socks-proxy:9150', '--head', '--max-time', '60', '--user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)', 'https://web.archive.org/save/https://specbranch.com/posts/one-big-server/']' timed out after 60 seconds", "pwd": "/data/archive/1756879672.881544", "schema": "ArchiveResult", "start_ts": "2025-09-03T06:08:38.161969+00:00", "status": "failed" } ], "dom": [ { "cmd": [ "/usr/bin/chromium-browser", "--proxy-server=socks5://tor-socks-proxy:9150", "--disable-features=DarkMode", "--run-all-compositor-stages-before-draw", "--hide-scrollbars", "--autoplay-policy=no-user-gesture-required", "--no-first-run", "--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--simulate-outdated-no-au='Tue, 31 Dec 2099 23:59:59 GMT'", "--headless=new", "--no-sandbox", "--no-zygote", "--disable-dev-shm-usage", "--disable-software-rasterizer", "--disable-sync", "--window-size=1440,2000", "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "--user-data-dir=/data/personas/Default/chrome_profile", "--profile-directory=Default", "--dump-dom", "https://specbranch.com/posts/one-big-server/" ], "cmd_version": "131.0.6778", "end_ts": "2025-09-03T06:08:18.129971+00:00", "index_texts": null, "output": "output.html", "pwd": "/data/archive/1756879672.881544", "schema": "ArchiveResult", "start_ts": "2025-09-03T06:08:08.120571+00:00", "status": "succeeded" } ], "favicon": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--max-time", "60", "--output", "favicon.ico", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://www.google.com/s2/favicons?domain=specbranch.com" ], "cmd_version": "8.10.1", "end_ts": "2025-09-03T06:08:01.495322+00:00", "index_texts": null, "output": "favicon.ico", "pwd": "/data/archive/1756879672.881544", "schema": "ArchiveResult", "start_ts": "2025-09-03T06:07:58.369891+00:00", "status": "succeeded" } ], "git": [], "headers": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--head", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://specbranch.com/posts/one-big-server/" ], "cmd_version": "8.10.1", "end_ts": "2025-09-03T06:08:01.883046+00:00", "index_texts": null, "output": "headers.json", "pwd": "/data/archive/1756879672.881544", "schema": "ArchiveResult", "start_ts": "2025-09-03T06:08:01.520008+00:00", "status": "succeeded" } ], "htmltotext": [ { "cmd": [ "(internal) archivebox.extractors.htmltotext", "./{singlefile,dom}.html" ], "cmd_version": "0.8.5rc51", "end_ts": "2025-09-03T06:08:32.284164+00:00", "index_texts": [ "Use One Big Server | Speculative Branches (https://specbranch.com/icons/apple-touch-icon.png) (https://specbranch.com/icons/favicon-32x32.png) (https://specbranch.com/icons/site.webmanifest) (https://specbranch.com/posts/one-big-server/) (https://fonts.googleapis.com) (https://fonts.gstatic.com) (https://fonts.googleapis.com/css2?family=Inria+Serif:ital,wght@0,300;0,400;0,700;1,300;1,400;1,700&display=swap) (https://fonts.googleapis.com/css2?family=IBM+Plex+Serif:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;1,100;1,200;1,300;1,400;1,500;1,600;1,700&family=Inria+Serif:ital,wght@0,300;0,400;0,700;1,300;1,400;1,700&display=swap) (https://specbranch.com/css/styles.3c442ae879d92dac69af941d8592e0c975cac430ef74dc2a3840ebb221b5d75150c27f4455f680e13d66e273114bcefa6f19ecf6dcec1bcf72b9735e8aebb23e.css) (https://specbranch.com/en/js/bundle.88d5aa9a242fde1e761c8dc7bae9844e1d55d29a423039dc16a9cc5d396a5ef41cd383fecffac9724e3d9804a0e1befe056e5683eb368635148504b1ee528e14.js) (https://specbranch.com/css/styles.3c442ae879d92dac69af941d8592e0c975cac430ef74dc2a3840ebb221b5d75150c27f4455f680e13d66e273114bcefa6f19ecf6dcec1bcf72b9735e8aebb23e.css) (https://specbranch.com/) (Speculative Branches) (Speculative Branches) open-menu closeme (https://specbranch.com/) (Home) Home (https://specbranch.com/categories/) (Categories) Categories (https://specbranch.com/tags/) (Tags) Tags (https://specbranch.com/about/) (About) About (https://github.com/specbranch) github (https://x.com/specbranch) twitter (https://www.linkedin.com/in/nimabadi) linkedin (http://eepurl.com/h0tVFT) email (https://specbranch.com/index.xml) rss Use One Big Server calendar Jul 27, 2022 \u00b7 15 min read \u00b7 (https://specbranch.com/tags/software-engineering/) (Software Engineering) Software Engineering \u00b7 Share on: (https://twitter.com/intent/tweet?text=Use%20One%20Big%20Server&url=https%3a%2f%2fspecbranch.com%2fposts%2fone-big-server%2f&tw_p=tweetbutton) (Share on Twitter) twitter (https://www.facebook.com/sharer.php?u=https%3a%2f%2fspecbranch.com%2fposts%2fone-big-server%2f&t=Use%20One%20Big%20Server) (Share on Facebook) facebook (Share on LinkedIn) linkedin (https://specbranch.com/posts/one-big-server/) (Copy Link) copy A lot of ink is spent on the \"monoliths vs. microservices\" debate, but the real issue behind\nthis debate is about whether distributed system architecture is worth the developer time and\ncost overheads. By thinking about the real operational considerations of our systems, we can\nget some insight into whether we actually need distributed systems for most things. We have all gotten so familiar with virtualization and abstractions between our software\nand the servers that run it. These days, \"serverless\" computing is all the rage, and even\n\"bare metal\" is a class of virtual machine. However, every piece of software runs on a\nserver. Since we now live in a world of virtualization, most of these servers are a lot\nbigger and a lot cheaper than we actually think. Meet Your Server(https://specbranch.com/posts/one-big-server/#meet-your-server) () This is a picture of a server used by Microsoft Azure with AMD CPUs. Starting from the left,\nthe big metal fixture on the left (with the copper tubes) is a heatsink, and the metal boxes\nthat the copper tubes are attached to are heat exchangers on each CPU. The CPUs are AMD's\nthird generation server CPU, each of which has the following specifications: 64 cores 128 threads ~2-2.5 GHz clock Cores capable of 4-6 instructions per clock cycle 256 MB of L3 cache In total, this server has 128 cores with 256 simultaneous threads. With all of the cores working\ntogether, this server is capable of 4 TFLOPs of peak double precision computing performance. This\nserver would sit at the top of the top500 supercomputer list in early 2000. It would take until\n2007 for this server to leave the top500 list. Each CPU core is substantially more powerful than a\nsingle core from 10 years ago, and boasts a much wider computation pipeline. Above and below each CPU is the memory: 16 slots of DDR4-3200 RAM per socket. The largest\ncapacity \"cost effective\" DIMMs today are 64 GB. Populated cost-efficiently, this server can hold 1 TB of memory. Populated with specialized high-capacity DIMMs (which are generally slower\nthan the smaller DIMMs), this server supports up to 8 TB of memory total. At DDR4-3200, with\na total of 16 memory channels, this server will likely see ~200 Gbps of memory throughput across\nall of its cores. In terms of I/O, each CPU offers 64 PCIe gen 4 lanes. With 128 PCIe lanes total, this server is\ncapable of supporting 30 NVMe SSDs plus a network card. Typical configurations you can buy will\noffer slots for around 16 SSDs or disks. The final thing I wanted to point out in this picture is\nin the top right, the network card. This server is likely equipped with a 50-100 Gbps network\nconnection. The Capabilities of One Server(https://specbranch.com/posts/one-big-server/#the-capabilities-of-one-server) One server today is capable of: (https://people.freebsd.org/~gallatin/talks/euro2021.pdf) Serving video files at 400 Gbps (now (http://nabstreamingsummit.com/wp-content/uploads/2022/05/2022-Streaming-Summit-Netflix.pdf) 800 Gbps ) (https://www.scylladb.com/2017/05/10/faster-and-better-what-to-expect-running-scylla-on-aws-i3-instances/) 1 million IOPS on a NoSQL database (https://www.enterprisedb.com/blog/pgbench-performance-benchmark-postgresql-12-and-edb-advanced-server-12) 70k IOPS in PostgreSQL (https://openbenchmarking.org/test/pts/nginx) 500k requests per second to nginx (https://openbenchmarking.org/test/pts/build-linux-kernel-1.14.0) Compiling the linux kernel in 20 seconds (https://openbenchmarking.org/test/pts/x264-2.7.0) Rendering 4k video with x264 at 75 FPS Among other things. There are a lot of public benchmarks these days, and if you know how your\nservice behaves, you can probably find a similar benchmark. The Cost of One Server(https://specbranch.com/posts/one-big-server/#the-cost-of-one-server) In a large hosting provider, OVHCloud, you can rent an HGR-HCI-6 server with similar specifications\nto the above, with 128 physical cores (256 threads), 512 GB of memory, and 50 Gbps of bandwidth\nfor $1,318/month. Moving to the popular budget option, Hetzner, you can rent a smaller server with 32 physical cores\nand 128 GB of RAM for about \u20ac140.00/month. This is a smaller server than the one from OVHCloud\n(1/4 the size), but it gives you some idea of the price spread between hosting providers. In AWS, one of the largest servers you can rent is the m6a.metal server. It offers 50 Gbps\nof network bandwidth, 192 vCPUs (96 physical cores), and 768 GB of memory, and costs 8.2944 / h o u r i n t h e U S E a s t r e g i o n . T h i s c o m e s o u t t o 8.2944/hour\nin the US East region. This comes out to 8.2944/ h o u r in t h e U SE a s t re g i o n . T hi sco m eso u tt o 6,055/month. The cloud premium is real! A similar server, with 128 physical cores and 512 GB of memory (as well as appropriate NICs,\nSSDs, and support contracts), can be purchased from the Dell website for about $40,000. However,\nif you are going to spend this much on a server, you should probably chat with a salesperson to\nmake sure you are getting the best deal you can. You will also need to pay to host this server\nand connect it to a network, though. In comparison, buying servers takes about 8 months to break even compared to using cloud servers,\nand 30 months to break even compared to renting. Of course, buying servers has a lot of drawbacks,\nand so does renting, so going forward, we will think a little bit about the \"cloud premium\" and\nwhether you should be willing to pay it (spoiler alert: the answer is \"yes, but not as much as the\ncloud companies want you to pay\"). Thinking about the Cloud(https://specbranch.com/posts/one-big-server/#thinking-about-the-cloud) The \"cloud era\" began in earnest around 2010. At the time, the state of the art CPU was an\n8-core Intel Nehalem CPU. Hyperthreading had just begun, so that 8-core CPU offered a\nwhopping 16 threads. Hardware acceleration was about to arrive for AES encryption, and\nvectors were 128 bits wide. The largest CPUs had 24 MB of cache, and your server could fit a\nwhopping 256 GB of DDR3-1066 memory. If you wanted to store data, Seagate had just begun to\noffer a 3 TB hard drive. Each core offered 4 FLOPs per cycle, meaning that your 8-core\nserver running at 2.5 GHz offered a blazing fast 80 GFLOPs. The boom in distributed computing rode on this wave: if you wanted to do anything that\ninvolved retrieval of data, you needed a lot of disks to get the storage throughput you want.\nIf you wanted to do large computations, you generally needed a lot of CPUs. This meant that\nyou needed to coordinate between a lot of CPUs to get most things done. Since that time began, the size of servers has increased a lot, and SSDs have increased available\nIOPS by a factor of at least 100, but the size of mainstream VMs and containers hasn't increased\nmuch, and we still use virtualized drives that perform more like hard drives than SSDs (although\nthis gap is closing). One Server (Plus a Backup) is Usually Plenty(https://specbranch.com/posts/one-big-server/#one-server-plus-a-backup-is-usually-plenty) If you are doing anything short of video streaming, and you have under 10k QPS, one server\nwill generally be fine for most web services. For really simple services, one server could\neven make it to a million QPS or so. Very few web services get this much traffic - if you\nhave one, you know about it. Even if you're serving video, running only one server for your\ncontrol plane is very reasonable. A benchmark can help you determine where you are.\nAlternatively, you can use common benchmarks of similar applications, or (/posts/common-perf-numbers/) tables of common performance numbers to estimate how big of a\nmachine you might need. Tall is Better than Wide(https://specbranch.com/posts/one-big-server/#tall-is-better-than-wide) When you need a cluster of computers, if one server is not enough, using fewer larger servers\nwill often be better than using a large fleet of small machines. There is non-zero overhead\nto coordinate a cluster, and that overhead is frequently O(n) on each server. To reduce this\noverhead, you should generally prefer to use a few large servers than to use many small servers.\nIn the case of things like serverless computing, where you allocate tiny short-lived containers,\nthis overhead accounts for a large fraction of the cost of use. On the other extreme end,\ncoordinating a cluster of one computer is trivial. Big Servers and Availability(https://specbranch.com/posts/one-big-server/#big-servers-and-availability) The big drawback of using a single big server is availability. Your server is going to need\ndowntime, and it is going to break. Running a primary and a backup server is usually enough,\nkeeping them in different datacenters. A 2x2 configuration should appease the truly paranoid: two\nservers in a primary datacenter (or cloud provider) and two servers in a backup datacenter will\ngive you a lot of redundancy. If you want a third backup deployment, you can often make that\nsmaller than your primary and secondary. However, you may still have to be concerned about correlated hardware failures. Hard drives\n(and now SSDs) have been known to occasionally have correlated failures: if you see one disk\nfail, you are a lot more likely to see a second failure before getting back up if your disks\nare from the same manufacturing batch. Services like Backblaze overcome this by using many\ndifferent models of disks from multiple manufacturers. Hacker news learned this the hard way\nrecently when the primary and backup server went down at the same time. If you are using a hosting provider which rents pre-built servers, it is prudent to rent two\ndifferent types of servers in each of your primary and backup datacenters. This should avoid\nalmost every failure mode present in modern systems. Use the Cloud, but don't be too Cloudy(https://specbranch.com/posts/one-big-server/#use-the-cloud-but-dont-be-too-cloudy) A combination of availability and ease of use is one of the big reasons why I (and most other\nengineers) like cloud computers. Yes, you pay a significant premium to rent the machines, but\nyour cloud provider has so much experience building servers that you don't even see most failures,\nand for the other failures, you can get back up and running really quickly by renting a new\nmachine in their nearly-limitless pool of compute. It is their job to make sure that you don't\nexperience downtime, and while they don't always do it perfectly, they are pretty good at it. Hosting providers who are willing to rent you a server are a cheaper alternative to cloud\nproviders, but these providers can sometimes have poor quality and some of them don't understand\nthings like network provisioning and correlated hardware failures. Also, moving from one rented\nserver to a larger one is a lot more annoying than resizing a cloud VM. Cloud servers have a\nprice premium for a good reason. However, when you deal with clouds, your salespeople will generally push you towards\n\"cloud-native\" architecture. These are things like microservices in auto-scaling VM groups with\nlegions of load balancers between them, and vendor-lock-in-enhancing products like serverless\ncomputing and managed high-availability databases. There is a good reason that cloud\nsalespeople are the ones pushing \"cloud architecture\" - it's better for them! The conventional wisdom is that using cloud architecture is good because it lets you scale up\neffortlessly. There are good reasons to use cloud-native architecture, but serving lots of people\nis not one of them: most services can serve millions of people at a time with one server, and\nwill never give you a surprise five-figure bill. Why Should I Pay for Peak Load?(https://specbranch.com/posts/one-big-server/#why-should-i-pay-for-peak-load) One common criticism of the \"one big server\" approach is that you now have to pay for your peak\nusage instead of paying as you go for what you use. Thus, serverless computing or fleets of\nmicroservice VMs more closely align your costs with your profit. Unfortunately, since all of your services run on servers (whether you like it or not), someone\nin that supply chain is charging you based on their peak load. Part of the \"cloud premium\" for\nload balancers, serverless computing, and small VMs is based on how much extra capacity your\ncloud provider needs to build in order to handle their peak load. You're paying for someone's\npeak load anyway! This means that if your workload is exceptionally bursty - like a simulation that needs\nto run once and then turn off forever - you should prefer to reach for \"cloudy\" solutions, but if\nyour workload is not so bursty, you will often have a cheaper system (and an easier time building\nit) if you go for few large servers. If your cloud provider's usage is more bursty than yours,\nyou are going to pay that premium for no benefit. This premium applies to VMs, too, not just cloud services. However, if you are running a cloud VM\n24/7, you can avoid paying the \"peak load premium\" by using 1-year contracts or negotiating with\na salesperson if you are big enough. Generally, the burstier your workload is, the more cloudy your architecture should be. How Much Does it Cost to be Cloudy?(https://specbranch.com/posts/one-big-server/#how-much-does-it-cost-to-be-cloudy) Being cloudy is expensive. Generally, I would anticipate a 5-30x price premium depending on what\nyou buy from a cloud company, and depending on the baseline. Not 5-30%, a factor of between 5 and\n30. Here is the pricing of AWS lambda: 0.20 p e r 1 M r e q u e s t s + 0.20 per 1M requests + 0.20 p er 1 M re q u es t s + 0.0000166667 per GB-second of RAM. I\nam using pricing for an x86 CPU here to keep parity with the m6a.metal instance we saw above.\nLarge ARM servers and serverless ARM compute are both cheaper. Assuming your server costs $8.2944/hour, and is capable of 1k QPS with 768 GB of RAM: 1k QPS is 60k queries per minute, or 3.6M queries per hour Each query here gets 0.768 GB-seconds of RAM (amortized) Replacing this server would cost about $46/hour using serverless computing The price premium for serverless computing over the instance is a factor of 5.5. If you can keep\nthat server over 20% utilization, using the server will be cheaper than using serverless computing.\nThis is before any form of savings plan you can apply to that server - if you can rent those big\nservers from the spot market or if you compare to the price you can get with a 1-year contract,\nthe price premium is even higher. If you compare to the OVHCloud rental price for the same server, the price premium of buying your\ncompute through AWS lambda is a factor of 25 If you are considering renting a server from a low-cost hosting provider or using AWS lambda, you\nshould prefer the hosting provider if you can keep the server operating at 5% capacity! Also, note that the actual QPS number doesn't matter: if the $8.2944/hour server is capable of 100k\nQPS, the query would use 100x less memory-time, meaning that you would arrive at the same 5.5x\n(or 25x) premium. Of course, you should scale the size of the server to fit your application. Common Objections to One Big Server(https://specbranch.com/posts/one-big-server/#common-objections-to-one-big-server) If you propose using the one big server approach, you will often get pushback from people who are\nmore comfortable with the cloud, prefer to be fashionable, or have legitimate concerns. Use your\njudgment when you think about it, but most people vastly underestimate how much \"cloud\narchitecture\" actually costs compared to the underlying compute. Here are some common objections. But if I use Cloud Architecture, I Don't Have to Hire Sysadmins(https://specbranch.com/posts/one-big-server/#but-if-i-use-cloud-architecture-i-dont-have-to-hire-sysadmins) Yes you do. They are just now called \"Cloud Ops\" and are under a different manager. Also, their\nability to read the arcane documentation that comes from cloud companies and keep up with the\ncorresponding torrents of updates and deprecations makes them 5x more expensive than system\nadministrators. But if I use Cloud Architecture, I Don't Have to Do Security Updates(https://specbranch.com/posts/one-big-server/#but-if-i-use-cloud-architecture-i-dont-have-to-do-security-updates) Yes you do. You may have to do fewer of them, but the ones you don't have to do are the easy ones\nto automate. You are still going to share in the pain of auditing libraries you use, and making\nsure that all of your configurations are secure. But if I use Cloud Architecture, I Don't Have to Worry About it Going Down(https://specbranch.com/posts/one-big-server/#but-if-i-use-cloud-architecture-i-dont-have-to-worry-about-it-going-down) The \"high availability\" architectures you get from using cloudy constructs and microservices just\nabout make up for the fragility they add due to complexity. At this point, if you use two\ndifferent cloud regions or two cloud providers, you can generally assume that is good enough to\navoid your service going down. However, cloud providers have often had global outages in the past,\nand there is no reason to assume that cloud datacenters will be down any less often than your\nindividual servers. Remember that we are trying to prevent correlated failures. Cloud datacenters have a lot of\nparts that can fail in correlated ways. Hosting providers have many fewer of these parts.\nSimilarly, complex cloud services, like managed databases, have more failure modes than simple\nones (VMs). But I can Develop More Quickly if I use Cloud Architecture(https://specbranch.com/posts/one-big-server/#but-i-can-develop-more-quickly-if-i-use-cloud-architecture) Then do it, and just keep an eye on the bill and think about when it's worth it to switch. This\nis probably the strongest argument in favor of using cloudy constructs. However, if you don't\nthink about it as you grow, you will likely end up burning a lot of money on your cloudy\narchitecture long past the time to switch to something more boring. My Workload is Really Bursty(https://specbranch.com/posts/one-big-server/#my-workload-is-really-bursty) Cloud away. That is a great reason to use things like serverless computing. One of the big\nbenefits of cloud architecture constructs is that the scale down really well. If your workload\ngoes through long periods of idleness punctuated with large unpredictable bursts of activity, cloud\narchitecture probably works really well for you. What about CDNs?(https://specbranch.com/posts/one-big-server/#what-about-cdns) It's impossible to get the benefits of a CDN, both in latency improvements and bandwidth savings,\nwith one big server. This is also true of other systems that need to be distributed, like backups.\nThankfully CDNs and backups are competitive markets, and relatively cheap. These are the kind of\nthing to buy rather than build. A Note On Microservices and Monoliths(https://specbranch.com/posts/one-big-server/#a-note-on-microservices-and-monoliths) Thinking about \"one big server\" naturally lines up with thinking about monolithic architectures.\nHowever, you don't need to use a monolith to use one server. You can run many containers on one\nbig server, with one microservice per container. However, microservice architectures in general\nadd a lot of overhead to a system for dubious gain when you are running on one big server. Conclusions(https://specbranch.com/posts/one-big-server/#conclusions) When you experience growing pains, and get close to the limits of your current servers, today's\nconventional wisdom is to go for sharding and horizontal scaling, or to use a cloud architecture\nthat gives you horizontal scaling \"for free.\" It is often easier and more efficient to scale\nvertically instead. Using one big server is comparatively cheap, keeps your overheads at a\nminimum, and actually has a pretty good availability story if you are careful to prevent correlated\nhardware failures. It's not glamorous and it won't help your resume, but one big server will serve\nyou well. Nima Badizadegan Computer systems engineer, performance expert, and applied mathematician in denial. Currently at Anthropic. All views are my own, and do not reflect the opinions of my employer. (https://specbranch.com/about/) (Read More) Read More Recent Posts (https://specbranch.com/posts/fast-exp/) (Exponentials in 3 Instructions) Exponentials in 3 Instructions (https://specbranch.com/posts/fp-rand/) (Perfect Random Floating-Point Numbers) Perfect Random Floating-Point Numbers (https://specbranch.com/posts/cryptographic-santa/) (A Cryptographically Secret Santa) A Cryptographically Secret Santa (https://specbranch.com/posts/time-for-jurors/) (Time Programming for Lawyers and Jurors) Time Programming for Lawyers and Jurors (https://specbranch.com/posts/five-nines/) (Five Nine Problems) Five Nine Problems (https://specbranch.com/posts/ai-infra/) (The Computer Architecture of AI (in 2024)) The Computer Architecture of AI (in 2024) (https://specbranch.com/posts/knight-capital/) (The Knight Capital Disaster) The Knight Capital Disaster (https://specbranch.com/posts/expensive-abstraction/) (Abstraction is Expensive) Abstraction is Expensive Categories (https://specbranch.com/categories/numerical-libraries-and-functions/) (numerical libraries and functions) NUMERICAL LIBRARIES AND FUNCTIONS 5 (https://specbranch.com/categories/random/) (random) RANDOM 4 (https://specbranch.com/categories/engineering-practices/) (engineering practices) ENGINEERING PRACTICES 3 (https://specbranch.com/categories/fundamentals-of-performance/) (fundamentals of performance) FUNDAMENTALS OF PERFORMANCE 3 (https://specbranch.com/categories/breaking-the-coding-interview/) (breaking the coding interview) BREAKING THE CODING INTERVIEW 2 (https://specbranch.com/categories/meta-posts/) (meta posts) META POSTS 2 (https://specbranch.com/categories/micro-optimization/) (micro-optimization) MICRO-OPTIMIZATION 2 (https://specbranch.com/categories/randomness/) (randomness) RANDOMNESS 2 (https://specbranch.com/categories/history/) (history) HISTORY 1 Tags (https://specbranch.com/tags/mathematical-algorithms/) (mathematical algorithms) MATHEMATICAL ALGORITHMS 9 (https://specbranch.com/tags/performance/) (performance) PERFORMANCE 9 (https://specbranch.com/tags/software-engineering/) (software engineering) SOFTWARE ENGINEERING 7 (https://specbranch.com/tags/division/) (division) DIVISION 1 (https://cdn.jsdelivr.net/npm/katex@0.15.1/dist/katex.min.css) Copyright 2025 NIMA BADIZADEGAN. All Rights Reserved to-top " ], "output": "htmltotext.txt", "pwd": "/data/archive/1756879672.881544", "schema": "ArchiveResult", "start_ts": "2025-09-03T06:08:32.226173+00:00", "status": "succeeded" } ], "media": [ { "cmd": [ "/usr/local/bin/yt-dlp", "--restrict-filenames", "--trim-filenames", "128", "--write-description", "--write-info-json", "--write-annotations", "--write-thumbnail", "--no-call-home", "--write-sub", "--write-auto-subs", "--convert-subs=srt", "--yes-playlist", "--continue", "--no-abort-on-error", "--ignore-errors", "--geo-bypass", "--add-metadata", "--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)", "--skip-download", "--cache-dir=/data/yt-dlp-cache/", "--cookies=/data/yt-dlp-cache/cookies.txt", "--proxy=socks5://tor-socks-proxy:9150", "--no-playlist", "https://specbranch.com/posts/one-big-server/" ], "cmd_version": "2024.10.7", "end_ts": "2025-09-03T06:08:38.118522+00:00", "index_texts": [], "output": "media/", "pwd": "/data/archive/1756879672.881544", "schema": "ArchiveResult", "start_ts": "2025-09-03T06:08:33.923725+00:00", "status": "succeeded" } ], "mercury": [ { "cmd": [ "/home/archivebox/.npm/bin/postlight-parser", "https://specbranch.com/posts/one-big-server/" ], "cmd_version": "2.2.3", "end_ts": "2025-09-03T06:08:32.199383+00:00", "index_texts": null, "output": "mercury/", "pwd": "/data/archive/1756879672.881544", "schema": "ArchiveResult", "start_ts": "2025-09-03T06:08:29.136630+00:00", "status": "succeeded" } ], "pdf": [], "readability": [ { "cmd": [ "/home/archivebox/.npm/bin/readability-extractor", "/tmp/tmpnokhdrdx", "https://specbranch.com/posts/one-big-server/" ], "cmd_version": "0.0.11", "end_ts": "2025-09-03T06:08:21.729871+00:00", "index_texts": [ "A lot of ink is spent on the \"monoliths vs. microservices\" debate, but the real issue behind\nthis debate is about whether distributed system architecture is worth the developer time and\ncost overheads. By thinking about the real operational considerations of our systems, we can\nget some insight into whether we actually need distributed systems for most things.\nWe have all gotten so familiar with virtualization and abstractions between our software\nand the servers that run it. These days, \"serverless\" computing is all the rage, and even\n\"bare metal\" is a class of virtual machine. However, every piece of software runs on a\nserver. Since we now live in a world of virtualization, most of these servers are a lot\nbigger and a lot cheaper than we actually think.\nMeet Your Server\n\n\n\nThis is a picture of a server used by Microsoft Azure with AMD CPUs. Starting from the left,\nthe big metal fixture on the left (with the copper tubes) is a heatsink, and the metal boxes\nthat the copper tubes are attached to are heat exchangers on each CPU. The CPUs are AMD's\nthird generation server CPU, each of which has the following specifications:\n\n64 cores\n128 threads\n~2-2.5 GHz clock\nCores capable of 4-6 instructions per clock cycle\n256 MB of L3 cache\n\nIn total, this server has 128 cores with 256 simultaneous threads. With all of the cores working\ntogether, this server is capable of 4 TFLOPs of peak double precision computing performance. This\nserver would sit at the top of the top500 supercomputer list in early 2000. It would take until\n2007 for this server to leave the top500 list. Each CPU core is substantially more powerful than a\nsingle core from 10 years ago, and boasts a much wider computation pipeline.\nAbove and below each CPU is the memory: 16 slots of DDR4-3200 RAM per socket. The largest\ncapacity \"cost effective\" DIMMs today are 64 GB. Populated cost-efficiently, this server can hold\n1 TB of memory. Populated with specialized high-capacity DIMMs (which are generally slower\nthan the smaller DIMMs), this server supports up to 8 TB of memory total. At DDR4-3200, with\na total of 16 memory channels, this server will likely see ~200 Gbps of memory throughput across\nall of its cores.\nIn terms of I/O, each CPU offers 64 PCIe gen 4 lanes. With 128 PCIe lanes total, this server is\ncapable of supporting 30 NVMe SSDs plus a network card. Typical configurations you can buy will\noffer slots for around 16 SSDs or disks. The final thing I wanted to point out in this picture is\nin the top right, the network card. This server is likely equipped with a 50-100 Gbps network\nconnection.\nThe Capabilities of One Server\nOne server today is capable of:\n\nServing video files at 400 Gbps (now 800 Gbps)\n1 million IOPS on a NoSQL database\n70k IOPS in PostgreSQL\n500k requests per second to nginx\nCompiling the linux kernel in 20 seconds\nRendering 4k video with x264 at 75 FPS\n\nAmong other things. There are a lot of public benchmarks these days, and if you know how your\nservice behaves, you can probably find a similar benchmark.\nThe Cost of One Server\nIn a large hosting provider, OVHCloud, you can rent an HGR-HCI-6 server with similar specifications\nto the above, with 128 physical cores (256 threads), 512 GB of memory, and 50 Gbps of bandwidth\nfor $1,318/month.\nMoving to the popular budget option, Hetzner, you can rent a smaller server with 32 physical cores\nand 128 GB of RAM for about \u20ac140.00/month. This is a smaller server than the one from OVHCloud\n(1/4 the size), but it gives you some idea of the price spread between hosting providers.\nIn AWS, one of the largest servers you can rent is the m6a.metal server. It offers 50 Gbps\nof network bandwidth, 192 vCPUs (96 physical cores), and 768 GB of memory, and costs 8.2944/hourintheUSEastregion.Thiscomesoutto8.2944/hour\nin the US East region. This comes out to 6,055/month. The cloud premium is real!\nA similar server, with 128 physical cores and 512 GB of memory (as well as appropriate NICs,\nSSDs, and support contracts), can be purchased from the Dell website for about $40,000. However,\nif you are going to spend this much on a server, you should probably chat with a salesperson to\nmake sure you are getting the best deal you can. You will also need to pay to host this server\nand connect it to a network, though.\nIn comparison, buying servers takes about 8 months to break even compared to using cloud servers,\nand 30 months to break even compared to renting. Of course, buying servers has a lot of drawbacks,\nand so does renting, so going forward, we will think a little bit about the \"cloud premium\" and\nwhether you should be willing to pay it (spoiler alert: the answer is \"yes, but not as much as the\ncloud companies want you to pay\").\nThinking about the Cloud\nThe \"cloud era\" began in earnest around 2010. At the time, the state of the art CPU was an\n8-core Intel Nehalem CPU. Hyperthreading had just begun, so that 8-core CPU offered a\nwhopping 16 threads. Hardware acceleration was about to arrive for AES encryption, and\nvectors were 128 bits wide. The largest CPUs had 24 MB of cache, and your server could fit a\nwhopping 256 GB of DDR3-1066 memory. If you wanted to store data, Seagate had just begun to\noffer a 3 TB hard drive. Each core offered 4 FLOPs per cycle, meaning that your 8-core\nserver running at 2.5 GHz offered a blazing fast 80 GFLOPs.\nThe boom in distributed computing rode on this wave: if you wanted to do anything that\ninvolved retrieval of data, you needed a lot of disks to get the storage throughput you want.\nIf you wanted to do large computations, you generally needed a lot of CPUs. This meant that\nyou needed to coordinate between a lot of CPUs to get most things done.\nSince that time began, the size of servers has increased a lot, and SSDs have increased available\nIOPS by a factor of at least 100, but the size of mainstream VMs and containers hasn't increased\nmuch, and we still use virtualized drives that perform more like hard drives than SSDs (although\nthis gap is closing).\nOne Server (Plus a Backup) is Usually Plenty\nIf you are doing anything short of video streaming, and you have under 10k QPS, one server\nwill generally be fine for most web services. For really simple services, one server could\neven make it to a million QPS or so. Very few web services get this much traffic - if you\nhave one, you know about it. Even if you're serving video, running only one server for your\ncontrol plane is very reasonable. A benchmark can help you determine where you are.\nAlternatively, you can use common benchmarks of similar applications, or\ntables of common performance numbers to estimate how big of a\nmachine you might need.\nTall is Better than Wide\nWhen you need a cluster of computers, if one server is not enough, using fewer larger servers\nwill often be better than using a large fleet of small machines. There is non-zero overhead\nto coordinate a cluster, and that overhead is frequently O(n) on each server. To reduce this\noverhead, you should generally prefer to use a few large servers than to use many small servers.\nIn the case of things like serverless computing, where you allocate tiny short-lived containers,\nthis overhead accounts for a large fraction of the cost of use. On the other extreme end,\ncoordinating a cluster of one computer is trivial.\nBig Servers and Availability\nThe big drawback of using a single big server is availability. Your server is going to need\ndowntime, and it is going to break. Running a primary and a backup server is usually enough,\nkeeping them in different datacenters. A 2x2 configuration should appease the truly paranoid: two\nservers in a primary datacenter (or cloud provider) and two servers in a backup datacenter will\ngive you a lot of redundancy. If you want a third backup deployment, you can often make that\nsmaller than your primary and secondary.\nHowever, you may still have to be concerned about correlated hardware failures. Hard drives\n(and now SSDs) have been known to occasionally have correlated failures: if you see one disk\nfail, you are a lot more likely to see a second failure before getting back up if your disks\nare from the same manufacturing batch. Services like Backblaze overcome this by using many\ndifferent models of disks from multiple manufacturers. Hacker news learned this the hard way\nrecently when the primary and backup server went down at the same time.\nIf you are using a hosting provider which rents pre-built servers, it is prudent to rent two\ndifferent types of servers in each of your primary and backup datacenters. This should avoid\nalmost every failure mode present in modern systems.\nUse the Cloud, but don't be too Cloudy\nA combination of availability and ease of use is one of the big reasons why I (and most other\nengineers) like cloud computers. Yes, you pay a significant premium to rent the machines, but\nyour cloud provider has so much experience building servers that you don't even see most failures,\nand for the other failures, you can get back up and running really quickly by renting a new\nmachine in their nearly-limitless pool of compute. It is their job to make sure that you don't\nexperience downtime, and while they don't always do it perfectly, they are pretty good at it.\nHosting providers who are willing to rent you a server are a cheaper alternative to cloud\nproviders, but these providers can sometimes have poor quality and some of them don't understand\nthings like network provisioning and correlated hardware failures. Also, moving from one rented\nserver to a larger one is a lot more annoying than resizing a cloud VM. Cloud servers have a\nprice premium for a good reason.\nHowever, when you deal with clouds, your salespeople will generally push you towards\n\"cloud-native\" architecture. These are things like microservices in auto-scaling VM groups with\nlegions of load balancers between them, and vendor-lock-in-enhancing products like serverless\ncomputing and managed high-availability databases. There is a good reason that cloud\nsalespeople are the ones pushing \"cloud architecture\" - it's better for them!\nThe conventional wisdom is that using cloud architecture is good because it lets you scale up\neffortlessly. There are good reasons to use cloud-native architecture, but serving lots of people\nis not one of them: most services can serve millions of people at a time with one server, and\nwill never give you a surprise five-figure bill.\nWhy Should I Pay for Peak Load?\nOne common criticism of the \"one big server\" approach is that you now have to pay for your peak\nusage instead of paying as you go for what you use. Thus, serverless computing or fleets of\nmicroservice VMs more closely align your costs with your profit.\nUnfortunately, since all of your services run on servers (whether you like it or not), someone\nin that supply chain is charging you based on their peak load. Part of the \"cloud premium\" for\nload balancers, serverless computing, and small VMs is based on how much extra capacity your\ncloud provider needs to build in order to handle their peak load. You're paying for someone's\npeak load anyway!\nThis means that if your workload is exceptionally bursty - like a simulation that needs\nto run once and then turn off forever - you should prefer to reach for \"cloudy\" solutions, but if\nyour workload is not so bursty, you will often have a cheaper system (and an easier time building\nit) if you go for few large servers. If your cloud provider's usage is more bursty than yours,\nyou are going to pay that premium for no benefit.\nThis premium applies to VMs, too, not just cloud services. However, if you are running a cloud VM\n24/7, you can avoid paying the \"peak load premium\" by using 1-year contracts or negotiating with\na salesperson if you are big enough.\nGenerally, the burstier your workload is, the more cloudy your architecture should be.\nHow Much Does it Cost to be Cloudy?\nBeing cloudy is expensive. Generally, I would anticipate a 5-30x price premium depending on what\nyou buy from a cloud company, and depending on the baseline. Not 5-30%, a factor of between 5 and\n30.\nHere is the pricing of AWS lambda: 0.20per1Mrequests+0.20 per 1M requests + 0.0000166667 per GB-second of RAM. I\nam using pricing for an x86 CPU here to keep parity with the m6a.metal instance we saw above.\nLarge ARM servers and serverless ARM compute are both cheaper.\nAssuming your server costs $8.2944/hour, and is capable of 1k QPS with 768 GB of RAM:\n\n\n1k QPS is 60k queries per minute, or 3.6M queries per hour\n\n\nEach query here gets 0.768 GB-seconds of RAM (amortized)\n\n\nReplacing this server would cost about $46/hour using serverless computing\n\n\nThe price premium for serverless computing over the instance is a factor of 5.5. If you can keep\nthat server over 20% utilization, using the server will be cheaper than using serverless computing.\nThis is before any form of savings plan you can apply to that server - if you can rent those big\nservers from the spot market or if you compare to the price you can get with a 1-year contract,\nthe price premium is even higher.\nIf you compare to the OVHCloud rental price for the same server, the price premium of buying your\ncompute through AWS lambda is a factor of 25\nIf you are considering renting a server from a low-cost hosting provider or using AWS lambda, you\nshould prefer the hosting provider if you can keep the server operating at 5% capacity!\nAlso, note that the actual QPS number doesn't matter: if the $8.2944/hour server is capable of 100k\nQPS, the query would use 100x less memory-time, meaning that you would arrive at the same 5.5x\n(or 25x) premium. Of course, you should scale the size of the server to fit your application.\nCommon Objections to One Big Server\nIf you propose using the one big server approach, you will often get pushback from people who are\nmore comfortable with the cloud, prefer to be fashionable, or have legitimate concerns. Use your\njudgment when you think about it, but most people vastly underestimate how much \"cloud\narchitecture\" actually costs compared to the underlying compute. Here are some common objections.\nBut if I use Cloud Architecture, I Don't Have to Hire Sysadmins\nYes you do. They are just now called \"Cloud Ops\" and are under a different manager. Also, their\nability to read the arcane documentation that comes from cloud companies and keep up with the\ncorresponding torrents of updates and deprecations makes them 5x more expensive than system\nadministrators.\nBut if I use Cloud Architecture, I Don't Have to Do Security Updates\nYes you do. You may have to do fewer of them, but the ones you don't have to do are the easy ones\nto automate. You are still going to share in the pain of auditing libraries you use, and making\nsure that all of your configurations are secure.\nBut if I use Cloud Architecture, I Don't Have to Worry About it Going Down\nThe \"high availability\" architectures you get from using cloudy constructs and microservices just\nabout make up for the fragility they add due to complexity. At this point, if you use two\ndifferent cloud regions or two cloud providers, you can generally assume that is good enough to\navoid your service going down. However, cloud providers have often had global outages in the past,\nand there is no reason to assume that cloud datacenters will be down any less often than your\nindividual servers.\nRemember that we are trying to prevent correlated failures. Cloud datacenters have a lot of\nparts that can fail in correlated ways. Hosting providers have many fewer of these parts.\nSimilarly, complex cloud services, like managed databases, have more failure modes than simple\nones (VMs).\nBut I can Develop More Quickly if I use Cloud Architecture\nThen do it, and just keep an eye on the bill and think about when it's worth it to switch. This\nis probably the strongest argument in favor of using cloudy constructs. However, if you don't\nthink about it as you grow, you will likely end up burning a lot of money on your cloudy\narchitecture long past the time to switch to something more boring.\nMy Workload is Really Bursty\nCloud away. That is a great reason to use things like serverless computing. One of the big\nbenefits of cloud architecture constructs is that the scale down really well. If your workload\ngoes through long periods of idleness punctuated with large unpredictable bursts of activity, cloud\narchitecture probably works really well for you.\nWhat about CDNs?\nIt's impossible to get the benefits of a CDN, both in latency improvements and bandwidth savings,\nwith one big server. This is also true of other systems that need to be distributed, like backups.\nThankfully CDNs and backups are competitive markets, and relatively cheap. These are the kind of\nthing to buy rather than build.\nA Note On Microservices and Monoliths\nThinking about \"one big server\" naturally lines up with thinking about monolithic architectures.\nHowever, you don't need to use a monolith to use one server. You can run many containers on one\nbig server, with one microservice per container. However, microservice architectures in general\nadd a lot of overhead to a system for dubious gain when you are running on one big server.\nConclusions\nWhen you experience growing pains, and get close to the limits of your current servers, today's\nconventional wisdom is to go for sharding and horizontal scaling, or to use a cloud architecture\nthat gives you horizontal scaling \"for free.\" It is often easier and more efficient to scale\nvertically instead. Using one big server is comparatively cheap, keeps your overheads at a\nminimum, and actually has a pretty good availability story if you are careful to prevent correlated\nhardware failures. It's not glamorous and it won't help your resume, but one big server will serve\nyou well." ], "output": "readability/", "pwd": "/data/archive/1756879672.881544", "schema": "ArchiveResult", "start_ts": "2025-09-03T06:08:19.162933+00:00", "status": "succeeded" } ], "screenshot": [], "singlefile": [], "title": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://specbranch.com/posts/one-big-server/" ], "cmd_version": "8.10.1", "end_ts": "2025-09-03T06:08:18.435322+00:00", "index_texts": null, "output": "Use One Big Server | Speculative Branches", "pwd": "/data/archive/1756879672.881544", "schema": "ArchiveResult", "start_ts": "2025-09-03T06:08:18.386622+00:00", "status": "succeeded" } ], "wget": [] }, "icons": null, "is_archived": true, "is_static": false, "latest": { "archive_org": "TimeoutExpired: Command '['/usr/bin/curl', '--silent', '--location', '--compressed', '--proxy', 'socks5://tor-socks-proxy:9150', '--head', '--max-time', '60', '--user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)', 'https://web.archive.org/save/https://specbranch.com/posts/one-big-server/']' timed out after 60 seconds", "dom": "output.html", "favicon": "favicon.ico", "git": null, "media": "media/", "pdf": null, "screenshot": null, "singlefile": null, "title": "Use One Big Server | Speculative Branches", "warc": null, "wget": null }, "link_dir": "/data/archive/1756879672.881544", "newest_archive_date": "2025-09-03T06:08:38.161969+00:00", "num_failures": 1, "num_outputs": 8, "oldest_archive_date": "2025-09-03T06:07:58.369891+00:00", "path": "/posts/one-big-server/", "schema": "Link", "scheme": "https", "snapshot_abid": "snp_01K4730NKJ8AA9619D01JNS93A", "snapshot_id": "971148f4-fe74-4188-8669-4c54255ca46a", "sources": [ "/data/sources/1756879671-import.txt" ], "tags": null, "tags_str": "", "timestamp": "1756879672.881544", "title": "Use One Big Server | Speculative Branches", "url": "https://specbranch.com/posts/one-big-server/" }