{ "archive_path": "archive/1778470165.028955", "base_url": "til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary", "basename": "", "bookmarked_date": "2026-05-11 03:29", "canonical": { "archive_org_path": "https://web.archive.org/web/til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary", "dom_path": "output.html", "favicon_path": "favicon.ico", "git_path": "git/", "google_favicon_path": "https://www.google.com/s2/favicons?domain=til.andrew-quinn.me", "headers_path": "headers.json", "htmltotext_path": "htmltotext.txt", "index_path": "index.html", "media_path": "media/", "mercury_path": "mercury/content.html", "pdf_path": "output.pdf", "readability_path": "readability/content.html", "screenshot_path": "screenshot.png", "singlefile_path": "singlefile.html", "warc_path": "warc/", "wget_path": null }, "domain": "til.andrew-quinn.me", "downloaded_at": "2026-05-11T03:29:43.397636+00:00", "downloaded_datestr": "2026-05-11 03:29", "extension": "", "hash": "VQXP4D4QR8PEWMSQRHCS", "history": { "archive_org": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--head", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://web.archive.org/save/https://til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary/" ], "cmd_version": "8.10.1", "end_ts": "2026-05-11T03:32:51.434348+00:00", "index_texts": null, "output": "TimeoutExpired: Command '['/usr/bin/curl', '--silent', '--location', '--compressed', '--proxy', 'socks5://tor-socks-proxy:9150', '--head', '--max-time', '60', '--user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)', 'https://web.archive.org/save/https://til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary/']' timed out after 60 seconds", "pwd": "/data/archive/1778470165.028955", "schema": "ArchiveResult", "start_ts": "2026-05-11T03:31:51.362917+00:00", "status": "failed" } ], "dom": [ { "cmd": [ "/usr/bin/chromium-browser", "--proxy-server=socks5://tor-socks-proxy:9150", "--disable-features=DarkMode", "--run-all-compositor-stages-before-draw", "--hide-scrollbars", "--autoplay-policy=no-user-gesture-required", "--no-first-run", "--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--simulate-outdated-no-au='Tue, 31 Dec 2099 23:59:59 GMT'", "--headless=new", "--no-sandbox", "--no-zygote", "--disable-dev-shm-usage", "--disable-software-rasterizer", "--disable-sync", "--window-size=1440,2000", "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "--user-data-dir=/data/personas/Default/chrome_profile", "--profile-directory=Default", "--dump-dom", "https://til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary/" ], "cmd_version": "131.0.6778", "end_ts": "2026-05-11T03:31:24.774531+00:00", "index_texts": null, "output": "output.html", "pwd": "/data/archive/1778470165.028955", "schema": "ArchiveResult", "start_ts": "2026-05-11T03:30:34.984264+00:00", "status": "succeeded" } ], "favicon": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--max-time", "60", "--output", "favicon.ico", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://www.google.com/s2/favicons?domain=til.andrew-quinn.me" ], "cmd_version": "8.10.1", "end_ts": "2026-05-11T03:29:53.726659+00:00", "index_texts": null, "output": "favicon.ico", "pwd": "/data/archive/1778470165.028955", "schema": "ArchiveResult", "start_ts": "2026-05-11T03:29:43.751284+00:00", "status": "succeeded" } ], "git": [], "headers": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--head", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary/" ], "cmd_version": "8.10.1", "end_ts": "2026-05-11T03:29:55.999969+00:00", "index_texts": null, "output": "headers.json", "pwd": "/data/archive/1778470165.028955", "schema": "ArchiveResult", "start_ts": "2026-05-11T03:29:54.708992+00:00", "status": "succeeded" } ], "htmltotext": [ { "cmd": [ "(internal) archivebox.extractors.htmltotext", "./{singlefile,dom}.html" ], "cmd_version": "0.8.5rc51", "end_ts": "2026-05-11T03:31:43.324566+00:00", "index_texts": [ "(https://giscus.app/default.css) (https://fonts.googleapis.com) (https://fonts.gstatic.com) (https://fonts.googleapis.com/css2?family=Lexend+Deca:wght@300&display=swap) (https://fonts.googleapis.com) (https://fonts.gstatic.com) (https://fonts.googleapis.com/css2?family=Atkinson+Hyperlegible:wght@700&display=swap) (https://fonts.googleapis.com) (https://fonts.gstatic.com) (https://fonts.googleapis.com/css2?family=Fira+Code&display=swap) Replacing a 3 GB SQLite database with a 10 MB FST (finite state transducer) binaryAndrew Quinn's TILs (https://til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary/) (/assets/css/stylesheet.e9659aa9d3756c7714c0630cae93d9fe20b172d690645945e9956263aa280e9a.css) (https://til.andrew-quinn.me/favicon.ico) (https://til.andrew-quinn.me/favicon-16x16.png) (https://til.andrew-quinn.me/favicon-32x32.png) (https://til.andrew-quinn.me/apple-touch-icon.png) (https://til.andrew-quinn.me/safari-pinned-tab.svg) (https://til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary/) (https://til.andrew-quinn.me/) (Andrew Quinn's TILs (Alt + H)) Andrew Quinn's TILs ((Alt + T)) (https://github.com/hiAndrewQuinn/til) (Archive) Archive (https://github.com/search?q=repo%3AhiAndrewQuinn%2Ftil+&type=code) (Search) Search (https://til.andrew-quinn.me/tags/) (Tags) Tags (https://andrew-quinn.me) (andrew-quinn.me) andrew-quinn.me (https://til.andrew-quinn.me/) Home \u00bb (https://til.andrew-quinn.me/posts/) Posts Replacing a 3 GB SQLite database with a 10 MB FST (finite state transducer) binary (2026-05-10 00:00:00 +0000 UTC) May 10, 2026 (https://x.com/hiAndrewQuinn) Add me on X / Twitter !\nYou can cite this post as a reason if you're shy. Note for (https://www.youtube.com/watch?v=aOJOfh2_4PE) numberphiles :\nall numbers have been rounded to their first significant\ndigit, because I\u2019m a fan of Rob Eastaway\u2019s (https://robeastaway.com/blog/introducing-zequals) \u201czequals\u201d method of getting to the point when it comes to estimation. It\u2019s much\nmore valuable to walk away with the heuristic \u201csome dude got\na 300x memory reduction by swapping out a database he hacked\ntogether for a tiny, static, specialized data structure that\ndoes exactly what he needs it to and no more.\u201d I found myself with an increasingly rare opportunity to work\nthis weekend on (https://taskusanakirja.com/) Taskusanakirja ,\nalso often called tsk ,\na Finnish-English dictionary with incremental search-as-you-type.1 Fundamentally this problem reduces down to (https://en.wikipedia.org/wiki/Trie) prefix search ,\nand the canonical solution for prefix search with autocomplete\nis to implement (https://www.baeldung.com/cs/tries-prefix-trees) a trie . And this worked wonderfully for the first implementation\nof tsk , which was in Go (and which I have written about (/posts/you-don-t-need-cgo-to-use-sqlite-in-your-go-binary/) elsewhere and (/posts/cross-platform-tuis-are-easier-than-cross-platform-guis/) elsewhere and (/posts/the-highest-personal-roi-program-i-have-written-so-far/) elsewhere ).\nWith a few basic optimizations.\nTo prevent matching on some single-digit percentage of the\nmid-six-figures number of words we were baking into the\nbinary (it\u2019s been a design goal from the start to ship the\nentire program as one .exe , one .app , or one statically linked binary), we set some limit of e.g. only the\nfirst 50 or 100 matches or so and then just memoized all\nof the 1-, 2-, and 3-character combinations, after which\nI\u2019ve never noticed a perceptible delay in the\nprogram again after a year of heavy personal use. We could\njust about squeeze a trie with some basic optimizations\nlike that into ~60 MB of space. But Finnish is a heavily (https://en.wikipedia.org/wiki/Agglutinative_language) agglutinative language. It\u2019s not\nimpossible for a single base word in the language to have\nover one hundred possible endings, when all combinations\nare considered. And the combinations are not regular!2 The\nextremely regularized orthography of the Finnish language also means no fibbing when it comes to what speakers actually say on the page, and that means that base words\nstretch and shift and transmute in acoustically-pleasing\nways as you layer on endings, which makes perfect sense after you\u2019ve spent a couple years already immersed in\nthe language. When you\u2019re a beginner, and you see a\nsentence like e.g. \u201cOpiskelijassammekin on leijonan syd\u00e4n\u201d,\nthere is one word you are disproportionately likely to\nget stuck on. Part of what this tool attempts to do is\nhelp the student figure out how to cleave the word at\nthe right edges by embedding all that information as well. The trie fell down at that point. I could keep ~400,000\nitems in the trie sipping ~50 MB of RAM.\nThe same trick does not scale to 40-60 million. Not if you want\nit all to run on the old laptop of a college kid from Jakarta.\nFrustrated and running out of time, I threw up my hands and\nsaid \u201cWe\u2019ll ship the inflections in a separate\nSQLite database with FTS (Full Text Search) and\nlet them search on that if they\u2019re so desperate,\u201d which worked \u2014 still without perceptible delay \u2014 but it required\na one time 3 gigabyte download. Not ideal! That was where the story stopped about 9 months ago. This\nweekend, with 9 more months of intense full time software\nengineering under my belt, I boldly asked: (https://transitiontech.ca/random/RIIR) Had I considered rewriting it in Rust? 3 It turns out there is a very, very smart guy named (https://github.com/BurntSushi) BurntSushi aka Andrew Gallant ,\ninfamous for (https://github.com/BurntSushi/ripgrep) ripgrep, a really really fast grep \u2014\na tool so ubiquitously useful (https://github.com/hiAndrewQuinn/shell-bling-ubuntu#the-holy-trinity) I put it years ago in my \u201cHoly Trinity\u201d of modern shell commands \u2014\nwho also faced a similar problem\nat some point in the past, and wrote a post called (https://burntsushi.net/transducers/) Index 1,600,000,000 Keys with Automata and Rust .\n(Warning: long, extremely interesting.)\nThe opening spoils it: It turns out that finite state machines are useful for things other than expressing computation. Finite state machines can also be used to compactly represent ordered sets or maps of strings that can be [prefix, fuzzy, suffix] searched very quickly. Well, I thought, this seems promising. Let\u2019s write\na minimal Rust program to strip the data out of that\n3 GB database and compact it down into one of these\nFST thingies.4 I mean, it was always obvious that was\na hack, but it was the best hack I could manage with\nthe time and energy at the time. How small could we\nget it? Ten _mega_bytes. A 300x reduction in space.\nEven in the world of (https://docs.rs/fst/) fst crate users,\nthis particular application \u2014 mapping conjugations\nand declensions of a highly agglutinative language\nback to their source definitions \u2014 was extremely\nwell suited to the domain. Unlike tries, FSTs compress both prefixes and suffixes, and in a language like\nFinnish, there are a very small handful of popular\nsuffixes which get repeated extremely often in\nthe dictionary corpus. The data load is static\nat runtime, which gets around fst \u2019s greatest\nweakness. I do wish to point out, of course, that the whole\nreason it was possible to experiment cheaply and come\nacross this serendipity was because 9 months ago,\nfaced with the choice to either do the bad easy thing\nor the good nothing, I chose to do the bad easy thing.5 The SQLite database worked! I understood how it worked, behind the scenes with its B-trees and its (https://www.sqlite.org/fts5.html) Full Text Search extension . I think I even used that same\nFTS extension to power certain lesser used features that\nare not in the alphas of tsk v2.0.0 at the time being\nand are likely to be dropped entirely if it means compromising\nthis now salivatory memory footprint. Because the Pro version of v2 is shaping up to be about 20 megabytes, all batteries\nincluded, which is 3 times less than the free version of v1 ever was. We\u2019ll see what makes it past the cutting room in\ntime. tsk started life as a TUI Go program \u2014 and in fact evolved out of an earlier fzf prototype called finstem , see (/posts/the-highest-personal-roi-program-i-have-written-so-far/) the highest-ROI program I\u2019ve written so far . The \u201cpocket dictionary\u201d framing (taskusanakirja literally means \u201cpocket dictionary\u201d in Finnish) was always load-bearing: if it doesn\u2019t fit on the kind of dusty laptop someone might inherit from an uncle, it isn\u2019t a pocket dictionary, it\u2019s an old Oxford that happens to compile. \u21a9\ufe0e Linguists call the deformations triggered by suffixes (https://en.wikipedia.org/wiki/Consonant_gradation) consonant gradation and (https://en.wikipedia.org/wiki/Vowel_harmony) vowel harmony , and Finnish wields both at once. Take katu (\u201cstreet\u201d), whose genitive is not katun but kadun \u2014 the t softens to d because the syllable closed. Multiply that across 15 cases, then 2 plurals, then 6 possessive suffixes, then some indetermintate amount of possible clitics, and you can see why a na\u00efve trie capitulates. It simply has no way to share the cost of the thousands of words that all end in -ssa-mme-kin (\u201cin- our [X]-, as well\u201d). \u21a9\ufe0e \u201cRewrite It In Rust\u201d is enough of a meme that there is an entire genre of blog posts pushing back on it. One honest version of the meme is something like: If your problem is in the intersection of \u201cneeds to be fast\u201d, \u201cneeds to be portable\u201d, and \u201cthe existing tooling has gnarly memory ergonomics\u201d, Rust might put you in clover. \u21a9\ufe0e The trick that makes FSTs so much more compact than tries on natural-language data is suffix sharing : a trie shares prefixes (so kadun and kaduille share their first three nodes) but stores every distinct suffix path independently, while a (https://en.wikipedia.org/wiki/Deterministic_acyclic_finite_state_automaton) minimal acyclic deterministic finite-state automaton merges any two subtrees that are structurally identical. For a corpus where 100,000 words all end in the same dozen inflectional patterns, this is a license to print memory. \u21a9\ufe0e This is a recurring shape to my notes here that I keep bumping into qua (/posts/it-s-okay-to-solve-a-problem-twice/) \u201cit\u2019s okay to solve a problem twice\u201d . One could say in the first quarter-century of my life, that while I was always fascinated by programming, I could never overcome the guilt of not really knowing whether the tool I am building right now isn\u2019t already superceded by some much better implementation someone else has already written 30 or 40 years ago; I could write a TSV-aware search and replace, or I could find out about awk and solve that entire class of problems in one fell swoop, for example. My central conceit is that this is a trap . You need to reinvent a couple of wheels to get to the edge of what we know about wheel-making, not a thousand wheels, and not zero; probably four or five is sufficient in most domains, maybe closer to twenty or thirty in the most epistemically rigorous and developed fields like mathematics or computer science. Each wheel you reinvent, and every driected question you ask along the way, will propel you faster to the true frontier than that same amount of time spend in idle study, or even five times that amount. This is at heart a (https://www.betonit.ai/p/how_people_gethtml) Caplanian view : \u201cIf schools teach few job skills, transfer of learning is mostly wishful thinking, and the effect of education on intelligence is largely hollow, how on earth do human beings get good at their jobs? The same way you get to Carnegie Hall: practice .\u201d Or if you prefer exhortations, (https://www.betonit.ai/p/do-ten-times-as-much) Do Ten Times as Much is my favorite unpleasant advice that works. \u21a9\ufe0e (https://til.andrew-quinn.me/tags/against-entropy/) Against-Entropy (https://til.andrew-quinn.me/tags/simple-suboptimal-solutions/) Simple-Suboptimal-Solutions (https://til.andrew-quinn.me/tags/worse-is-better/) Worse-Is-Better (https://til.andrew-quinn.me/tags/finnish/) Finnish (https://til.andrew-quinn.me/tags/data-structures-and-algorithms/) Data-Structures-and-Algorithms (https://til.andrew-quinn.me/tags/cost-optimization/) Cost-Optimization (https://til.andrew-quinn.me/tags/rust/) Rust (https://til.andrew-quinn.me/tags/fst/) Fst (https://til.andrew-quinn.me/posts/i-still-like-jenkins/) Next \u00bb I still like Jenkins \u00a9 2026 (https://til.andrew-quinn.me/) Andrew Quinn's TILs Powered by (https://gohugo.io/) Hugo & (https://github.com/adityatelange/hugo-PaperMod/) PaperMod All thoughts are my own and not my employer's. (Go to Top (Alt + G)) (Comments) " ], "output": "htmltotext.txt", "pwd": "/data/archive/1778470165.028955", "schema": "ArchiveResult", "start_ts": "2026-05-11T03:31:43.297046+00:00", "status": "succeeded" } ], "media": [ { "cmd": [ "/usr/local/bin/yt-dlp", "--restrict-filenames", "--trim-filenames", "128", "--write-description", "--write-info-json", "--write-annotations", "--write-thumbnail", "--no-call-home", "--write-sub", "--write-auto-subs", "--convert-subs=srt", "--yes-playlist", "--continue", "--no-abort-on-error", "--ignore-errors", "--geo-bypass", "--add-metadata", "--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)", "--skip-download", "--cache-dir=/data/yt-dlp-cache/", "--cookies=/data/yt-dlp-cache/cookies.txt", "--proxy=socks5://tor-socks-proxy:9150", "--no-playlist", "https://til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary/" ], "cmd_version": "2024.10.7", "end_ts": "2026-05-11T03:31:51.277827+00:00", "index_texts": [], "output": "media/", "pwd": "/data/archive/1778470165.028955", "schema": "ArchiveResult", "start_ts": "2026-05-11T03:31:46.010706+00:00", "status": "succeeded" } ], "mercury": [ { "cmd": [ "/home/archivebox/.npm/bin/postlight-parser", "https://til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary/" ], "cmd_version": "2.2.3", "end_ts": "2026-05-11T03:31:43.243929+00:00", "index_texts": null, "output": "mercury/", "pwd": "/data/archive/1778470165.028955", "schema": "ArchiveResult", "start_ts": "2026-05-11T03:31:40.685904+00:00", "status": "succeeded" } ], "pdf": [], "readability": [ { "cmd": [ "/home/archivebox/.npm/bin/readability-extractor", "/tmp/tmp9msgf6ci", "https://til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary/" ], "cmd_version": "0.0.11", "end_ts": "2026-05-11T03:31:31.014390+00:00", "index_texts": [ "Note for\nnumberphiles:\nall numbers have been rounded to their first significant\ndigit, because I\u2019m a fan of Rob Eastaway\u2019s\n\u201czequals\u201d method\nof getting to the point when it comes to estimation. It\u2019s much\nmore valuable to walk away with the heuristic \u201csome dude got\na 300x memory reduction by swapping out a database he hacked\ntogether for a tiny, static, specialized data structure that\ndoes exactly what he needs it to and no more.\u201dI found myself with an increasingly rare opportunity to work\nthis weekend on\nTaskusanakirja,\nalso often called tsk,\na Finnish-English dictionary with incremental search-as-you-type.1\nFundamentally this problem reduces down to\nprefix search,\nand the canonical solution for prefix search with autocomplete\nis to implement\na trie.And this worked wonderfully for the first implementation\nof tsk, which was in Go (and which I have written about\nelsewhere\nand elsewhere\nand elsewhere).\nWith a few basic optimizations.\nTo prevent matching on some single-digit percentage of the\nmid-six-figures number of words we were baking into the\nbinary (it\u2019s been a design goal from the start to ship the\nentire program as one .exe, one .app, or one\nstatically linked binary), we set some limit of e.g. only the\nfirst 50 or 100 matches or so and then just memoized all\nof the 1-, 2-, and 3-character combinations, after which\nI\u2019ve never noticed a perceptible delay in the\nprogram again after a year of heavy personal use. We could\njust about squeeze a trie with some basic optimizations\nlike that into ~60 MB of space.But Finnish is a heavily\nagglutinative\nlanguage. It\u2019s not\nimpossible for a single base word in the language to have\nover one hundred possible endings, when all combinations\nare considered. And the combinations are not regular!2 The\nextremely regularized orthography of the Finnish language\nalso means no fibbing when it comes to what speakers\nactually say on the page, and that means that base words\nstretch and shift and transmute in acoustically-pleasing\nways as you layer on endings, which makes perfect sense\nafter you\u2019ve spent a couple years already immersed in\nthe language. When you\u2019re a beginner, and you see a\nsentence like e.g. \u201cOpiskelijassammekin on leijonan syd\u00e4n\u201d,\nthere is one word you are disproportionately likely to\nget stuck on. Part of what this tool attempts to do is\nhelp the student figure out how to cleave the word at\nthe right edges by embedding all that information as well.The trie fell down at that point. I could keep ~400,000\nitems in the trie sipping ~50 MB of RAM.\nThe same trick does not scale to 40-60 million. Not if you want\nit all to run on the old laptop of a college kid from Jakarta.\nFrustrated and running out of time, I threw up my hands and\nsaid \u201cWe\u2019ll ship the inflections in a separate\nSQLite database with FTS (Full Text Search) and\nlet them search on that if they\u2019re so desperate,\u201d which\nworked \u2014 still without perceptible delay \u2014 but it required\na one time 3 gigabyte download. Not ideal!That was where the story stopped about 9 months ago. This\nweekend, with 9 more months of intense full time software\nengineering under my belt, I boldly asked:\nHad I considered rewriting it in Rust?3It turns out there is a very, very smart guy named\nBurntSushi aka Andrew Gallant,\ninfamous for\nripgrep, a really really fast grep \u2014\na tool so ubiquitously useful\nI put it years ago in my \u201cHoly Trinity\u201d of modern shell commands \u2014\nwho also faced a similar problem\nat some point in the past, and wrote a post called\nIndex 1,600,000,000 Keys with Automata and Rust.\n(Warning: long, extremely interesting.)\nThe opening spoils it:It turns out that finite state machines are useful for things other than expressing computation. Finite state machines can also be used to compactly represent ordered sets or maps of strings that can be [prefix, fuzzy, suffix] searched very quickly.Well, I thought, this seems promising. Let\u2019s write\na minimal Rust program to strip the data out of that\n3 GB database and compact it down into one of these\nFST thingies.4 I mean, it was always obvious that was\na hack, but it was the best hack I could manage with\nthe time and energy at the time. How small could we\nget it?Ten _mega_bytes. A 300x reduction in space.\nEven in the world of fst crate users,\nthis particular application \u2014 mapping conjugations\nand declensions of a highly agglutinative language\nback to their source definitions \u2014 was extremely\nwell suited to the domain. Unlike tries, FSTs compress\nboth prefixes and suffixes, and in a language like\nFinnish, there are a very small handful of popular\nsuffixes which get repeated extremely often in\nthe dictionary corpus. The data load is static\nat runtime, which gets around fst\u2019s greatest\nweakness.I do wish to point out, of course, that the whole\nreason it was possible to experiment cheaply and come\nacross this serendipity was because 9 months ago,\nfaced with the choice to either do the bad easy thing\nor the good nothing, I chose to do the bad easy thing.5\nThe SQLite database worked! I understood\nhow it worked, behind the scenes with its B-trees and its\nFull Text Search extension. I think I even used that same\nFTS extension to power certain lesser used features that\nare not in the alphas of tsk v2.0.0 at the time being\nand are likely to be dropped entirely if it means compromising\nthis now salivatory memory footprint.Because the Pro version of\nv2 is shaping up to be about 20 megabytes, all batteries\nincluded, which is 3 times less than the free version of v1\never was. We\u2019ll see what makes it past the cutting room in\ntime." ], "output": "readability/", "pwd": "/data/archive/1778470165.028955", "schema": "ArchiveResult", "start_ts": "2026-05-11T03:31:27.450164+00:00", "status": "succeeded" } ], "screenshot": [], "singlefile": [], "title": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary/" ], "cmd_version": "8.10.1", "end_ts": "2026-05-11T03:31:26.062465+00:00", "index_texts": null, "output": "Replacing a 3 GB SQLite database with a 10 MB FST (finite state transducer) binaryAndrew Quinn's TILs", "pwd": "/data/archive/1778470165.028955", "schema": "ArchiveResult", "start_ts": "2026-05-11T03:31:25.938091+00:00", "status": "succeeded" } ], "wget": [] }, "icons": null, "is_archived": true, "is_static": false, "latest": { "archive_org": "TimeoutExpired: Command '['/usr/bin/curl', '--silent', '--location', '--compressed', '--proxy', 'socks5://tor-socks-proxy:9150', '--head', '--max-time', '60', '--user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)', 'https://web.archive.org/save/https://til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary/']' timed out after 60 seconds", "dom": "output.html", "favicon": "favicon.ico", "git": null, "media": "media/", "pdf": null, "screenshot": null, "singlefile": null, "title": "Replacing a 3 GB SQLite database with a 10 MB FST (finite state transducer) binaryAndrew Quinn's TILs", "warc": null, "wget": null }, "link_dir": "/data/archive/1778470165.028955", "newest_archive_date": "2026-05-11T03:31:51.362917+00:00", "num_failures": 1, "num_outputs": 8, "oldest_archive_date": "2026-05-11T03:29:43.751284+00:00", "path": "/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary/", "schema": "Link", "scheme": "https", "snapshot_abid": "snp_01KRAHA6NWF084D4D101STSAST", "snapshot_id": "5f2a42bc-f9d7-47de-a7a2-c07c33acab3a", "sources": [ "/data/sources/1778470163-import.txt" ], "tags": null, "tags_str": "", "timestamp": "1778470165.028955", "title": "Replacing a 3 GB SQLite database with a 10 MB FST (finite state transducer) binaryAndrew Quinn's TILs", "url": "https://til.andrew-quinn.me/posts/replacing-a-3-gb-sqlite-database-with-a-7-mb-fst-finite-state-trandsucer-binary/" }