{ "archive_path": "archive/1740536293.956353", "base_url": "noperator.dev/posts/document-ranking-for-complex-problems", "basename": "", "bookmarked_date": "2025-02-26 02:18", "canonical": { "archive_org_path": "https://web.archive.org/web/noperator.dev/posts/document-ranking-for-complex-problems", "dom_path": "output.html", "favicon_path": "favicon.ico", "git_path": "git/", "google_favicon_path": "https://www.google.com/s2/favicons?domain=noperator.dev", "headers_path": "headers.json", "htmltotext_path": "htmltotext.txt", "index_path": "index.html", "media_path": "media/", "mercury_path": "mercury/content.html", "pdf_path": "output.pdf", "readability_path": "readability/content.html", "screenshot_path": "screenshot.png", "singlefile_path": "singlefile.html", "warc_path": "warc/", "wget_path": null }, "domain": "noperator.dev", "downloaded_at": "2025-02-26T02:18:17.850368+00:00", "downloaded_datestr": "2025-02-26 02:18", "extension": "", "hash": "EB05MX8HT8E40DGJP87Z", "history": { "archive_org": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--head", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://web.archive.org/save/https://noperator.dev/posts/document-ranking-for-complex-problems/" ], "cmd_version": "8.10.1", "end_ts": "2025-02-26T02:20:06.586093+00:00", "index_texts": null, "output": "TimeoutExpired: Command '['/usr/bin/curl', '--silent', '--location', '--compressed', '--proxy', 'socks5://tor-socks-proxy:9150', '--head', '--max-time', '60', '--user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)', 'https://web.archive.org/save/https://noperator.dev/posts/document-ranking-for-complex-problems/']' timed out after 60 seconds", "pwd": "/data/archive/1740536293.956353", "schema": "ArchiveResult", "start_ts": "2025-02-26T02:19:06.527823+00:00", "status": "failed" } ], "dom": [ { "cmd": [ "/usr/bin/chromium-browser", "--proxy-server=socks5://tor-socks-proxy:9150", "--disable-features=DarkMode", "--run-all-compositor-stages-before-draw", "--hide-scrollbars", "--autoplay-policy=no-user-gesture-required", "--no-first-run", "--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--simulate-outdated-no-au='Tue, 31 Dec 2099 23:59:59 GMT'", "--headless=new", "--no-sandbox", "--no-zygote", "--disable-dev-shm-usage", "--disable-software-rasterizer", "--disable-sync", "--window-size=1440,2000", "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "--user-data-dir=/data/personas/Default/chrome_profile", "--profile-directory=Default", "--dump-dom", "https://noperator.dev/posts/document-ranking-for-complex-problems/" ], "cmd_version": "131.0.6778", "end_ts": "2025-02-26T02:18:31.385115+00:00", "index_texts": null, "output": "output.html", "pwd": "/data/archive/1740536293.956353", "schema": "ArchiveResult", "start_ts": "2025-02-26T02:18:25.614295+00:00", "status": "succeeded" } ], "favicon": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--max-time", "60", "--output", "favicon.ico", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://www.google.com/s2/favicons?domain=noperator.dev" ], "cmd_version": "8.10.1", "end_ts": "2025-02-26T02:18:21.523789+00:00", "index_texts": null, "output": "favicon.ico", "pwd": "/data/archive/1740536293.956353", "schema": "ArchiveResult", "start_ts": "2025-02-26T02:18:18.132922+00:00", "status": "succeeded" } ], "git": [], "headers": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--head", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://noperator.dev/posts/document-ranking-for-complex-problems/" ], "cmd_version": "8.10.1", "end_ts": "2025-02-26T02:18:21.700529+00:00", "index_texts": null, "output": "headers.json", "pwd": "/data/archive/1740536293.956353", "schema": "ArchiveResult", "start_ts": "2025-02-26T02:18:21.559302+00:00", "status": "succeeded" } ], "htmltotext": [ { "cmd": [ "(internal) archivebox.extractors.htmltotext", "./{singlefile,dom}.html" ], "cmd_version": "0.8.5rc51", "end_ts": "2025-02-26T02:18:56.315739+00:00", "index_texts": [ "Hard problems that reduce to document ranking | noperator (https://noperator.dev/posts/document-ranking-for-complex-problems/) (/assets/css/stylesheet.5cf4e1218b79c13e3fbba607a60f894e607d5fdd79efe3df3c386c7b2fcd2f34.css) (https://noperator.dev/favicon.ico) (https://noperator.dev/favicon-16x16.png) (https://noperator.dev/favicon-32x32.png) (https://noperator.dev/apple-touch-icon.png) (https://noperator.dev/safari-pinned-tab.svg) (https://noperator.dev/) (noperator (Alt + H)) noperator Hard problems that reduce to document ranking (2025-02-24 00:00:00 +0000 UTC) February 24, 2025 There are two claims I\u2019d like to make: LLMs can be used effectively1 for listwise (https://en.wikipedia.org/wiki/Learning_to_rank#Approaches) document ranking . Some complex problems can (surprisingly) be solved by (https://en.wikipedia.org/wiki/Reduction_(complexity)) transforming them into document ranking problems. I\u2019ve primarily explored both of these claims in the context of using patch diffing to locate N-day vulnerabilities\u2014a sufficiently domain-specific problem that can be solved using general purpose language models as comparators in document ranking algorithms. I demonstrated at (https://youtu.be/IBuL1zY69tY?si=l27sUOaECO-o9QFW&t=1846) RVAsec \u201824 that listwise document ranking can be used to locate the specific function in a patch diff that actually fixes a vulnerability described by a security advisory, and later wrote on the (https://bishopfox.com/blog/raink-llms-document-ranking) Bishop Fox blog in greater defense of listwise ranking by publishing a (https://github.com/noperator/raink) command-line tool implementation (raink ) to prove the idea. The key insight is that instead of treating patch diffing as a complex problem requiring specialized security engineering knowledge, you can reframe it as ranking diffs (documents) by their relevance to a security advisory (query), applying proven document ranking techniques from information retrieval. Using this technique, I proved at (https://www.youtube.com/live/aQyBRlu-cA4?si=3V79VdVmPO9D5WVW&t=260) DistrictCon \u201825 that GPT-4o mini could locate a fixed vulnerability in a haystack of over 1600 changed (and stripped!) functions in a patch\u2014costing only 5 minutes and 30 cents to do so2 . Document ranking can be applied to other offensive security problems, like identifying candidate functions for fuzzing targets (in addition to using them for (https://blog.oss-fuzz.com/posts/introducing-llm-based-harness-synthesis-for-unfuzzed-projects/) auto-generating harnesses ), or prioritizing potential injection points in a web application for deeper testing. A few potentially powerful improvements to this technique: Analyze the top N ranked results, and then apply the same ranking algorithm to the analyses. Make the ranked results verifiable; e.g., for N-day vulnerabilities, use an LLM to generate an automatically testable proof-of-concept exploit3 . Following Thomas Dullien\u2019s FUZZING \u201824 keynote (https://www.youtube.com/watch?v=Jd1hItbf52k&t=95s) \u201cReasons for the Unreasonable Success of Fuzzing\u201d , I\u2019m inclined to give a similar talk\u2014\u201cReasons for the Unreasonable Success of LLMs.\u201d A few others have explored the idea of document ranking using LLMs, but favored the computationally complex pairwise ranking method while noting the challenges of the more efficient but yet-unimplemented listwise ranking method. See (https://blog.reachsumit.com/posts/2023/12/prompting-llm-for-ranking/) Prompting-based Methods for Text Ranking Using Large Language Models (Dec \u201823) and (https://arxiv.org/html/2306.17563v2) Large Language Models are Effective Text Rankers with Pairwise Ranking Prompting (Mar \u201824). \u21a9\ufe0e DistrictCon slides (https://drive.google.com/file/d/1DsIsme23HTjTZYVLul-lWtTErKS9mZUy/view) here . \u21a9\ufe0e See aforementioned DistrictCon talk for an example of o3\u2011mini\u2011high successfully generating an exploit for (https://bishopfox.com/blog/sonicwall-cve-2024-53704-ssl-vpn-session-hijacking) CVE-2024-53704 , an authentication bypass in SonicWall firewalls. \u21a9\ufe0e (Go to Top (Alt + G)) " ], "output": "htmltotext.txt", "pwd": "/data/archive/1740536293.956353", "schema": "ArchiveResult", "start_ts": "2025-02-26T02:18:56.272921+00:00", "status": "succeeded" } ], "media": [ { "cmd": [ "/usr/local/bin/yt-dlp", "--restrict-filenames", "--trim-filenames", "128", "--write-description", "--write-info-json", "--write-annotations", "--write-thumbnail", "--no-call-home", "--write-sub", "--write-auto-subs", "--convert-subs=srt", "--yes-playlist", "--continue", "--no-abort-on-error", "--ignore-errors", "--geo-bypass", "--add-metadata", "--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)", "--skip-download", "--cache-dir=/data/yt-dlp-cache/", "--cookies=/data/yt-dlp-cache/cookies.txt", "--proxy=socks5://tor-socks-proxy:9150", "--no-playlist", "https://noperator.dev/posts/document-ranking-for-complex-problems/" ], "cmd_version": "2024.10.7", "end_ts": "2025-02-26T02:19:06.390482+00:00", "index_texts": [], "output": "media/", "pwd": "/data/archive/1740536293.956353", "schema": "ArchiveResult", "start_ts": "2025-02-26T02:18:59.064850+00:00", "status": "succeeded" } ], "mercury": [ { "cmd": [ "/home/archivebox/.npm/bin/postlight-parser", "https://noperator.dev/posts/document-ranking-for-complex-problems/" ], "cmd_version": "2.2.3", "end_ts": "2025-02-26T02:18:56.212237+00:00", "index_texts": null, "output": "mercury/", "pwd": "/data/archive/1740536293.956353", "schema": "ArchiveResult", "start_ts": "2025-02-26T02:18:51.349769+00:00", "status": "succeeded" } ], "pdf": [], "readability": [ { "cmd": [ "/home/archivebox/.npm/bin/readability-extractor", "/tmp/tmptso1denn", "https://noperator.dev/posts/document-ranking-for-complex-problems/" ], "cmd_version": "0.0.11", "end_ts": "2025-02-26T02:18:37.585152+00:00", "index_texts": [ "There are two claims I\u2019d like to make:\n\nLLMs can be used effectively1 for listwise document ranking.\nSome complex problems can (surprisingly) be solved by transforming them into document ranking problems.\n\nI\u2019ve primarily explored both of these claims in the context of using patch diffing to locate N-day vulnerabilities\u2014a sufficiently domain-specific problem that can be solved using general purpose language models as comparators in document ranking algorithms. I demonstrated at RVAsec \u201824 that listwise document ranking can be used to locate the specific function in a patch diff that actually fixes a vulnerability described by a security advisory, and later wrote on the Bishop Fox blog in greater defense of listwise ranking by publishing a command-line tool implementation (raink) to prove the idea.\nThe key insight is that instead of treating patch diffing as a complex problem requiring specialized security engineering knowledge, you can reframe it as ranking diffs (documents) by their relevance to a security advisory (query), applying proven document ranking techniques from information retrieval.\n\n\n\n\n\n\n\n\nUsing this technique, I proved at DistrictCon \u201825 that GPT-4o mini could locate a fixed vulnerability in a haystack of over 1600 changed (and stripped!) functions in a patch\u2014costing only 5 minutes and 30 cents to do so2.\n\n\nDocument ranking can be applied to other offensive security problems, like identifying candidate functions for fuzzing targets (in addition to using them for auto-generating harnesses), or prioritizing potential injection points in a web application for deeper testing. A few potentially powerful improvements to this technique:\n\nAnalyze the top N ranked results, and then apply the same ranking algorithm to the analyses.\nMake the ranked results verifiable; e.g., for N-day vulnerabilities, use an LLM to generate an automatically testable proof-of-concept exploit3.\n\nFollowing Thomas Dullien\u2019s FUZZING \u201824 keynote \u201cReasons for the Unreasonable Success of Fuzzing\u201d, I\u2019m inclined to give a similar talk\u2014\u201cReasons for the Unreasonable Success of LLMs.\u201d" ], "output": "readability/", "pwd": "/data/archive/1740536293.956353", "schema": "ArchiveResult", "start_ts": "2025-02-26T02:18:33.269542+00:00", "status": "succeeded" } ], "screenshot": [], "singlefile": [], "title": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://noperator.dev/posts/document-ranking-for-complex-problems/" ], "cmd_version": "8.10.1", "end_ts": "2025-02-26T02:18:31.437783+00:00", "index_texts": null, "output": "Hard problems that reduce to document ranking | noperator", "pwd": "/data/archive/1740536293.956353", "schema": "ArchiveResult", "start_ts": "2025-02-26T02:18:31.423611+00:00", "status": "succeeded" } ], "wget": [] }, "icons": null, "is_archived": true, "is_static": false, "latest": { "archive_org": "TimeoutExpired: Command '['/usr/bin/curl', '--silent', '--location', '--compressed', '--proxy', 'socks5://tor-socks-proxy:9150', '--head', '--max-time', '60', '--user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)', 'https://web.archive.org/save/https://noperator.dev/posts/document-ranking-for-complex-problems/']' timed out after 60 seconds", "dom": "output.html", "favicon": "favicon.ico", "git": null, "media": "media/", "pdf": null, "screenshot": null, "singlefile": null, "title": "Hard problems that reduce to document ranking | noperator", "warc": null, "wget": null }, "link_dir": "/data/archive/1740536293.956353", "newest_archive_date": "2025-02-26T02:19:06.527823+00:00", "num_failures": 1, "num_outputs": 8, "oldest_archive_date": "2025-02-26T02:18:18.132922+00:00", "path": "/posts/document-ranking-for-complex-problems/", "schema": "Link", "scheme": "https", "snapshot_abid": "snp_01JN00RAM16F0C4D5F01NBXZBN", "snapshot_id": "23e9da3b-0923-47a8-9db2-f225eabefd75", "sources": [ "/data/sources/1740536293-import.txt" ], "tags": null, "tags_str": "", "timestamp": "1740536293.956353", "title": "Hard problems that reduce to document ranking | noperator", "url": "https://noperator.dev/posts/document-ranking-for-complex-problems/" }