{ "archive_path": "archive/1741121993.301808", "base_url": "allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?m=1", "basename": "the-inspection-paradox-is-everywhere.html", "bookmarked_date": "2025-03-04 20:59", "canonical": { "archive_org_path": "https://web.archive.org/web/allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?m=1", "dom_path": "output.html", "favicon_path": "favicon.ico", "git_path": "git/", "google_favicon_path": "https://www.google.com/s2/favicons?domain=allendowney.blogspot.com", "headers_path": "headers.json", "htmltotext_path": "htmltotext.txt", "index_path": "index.html", "media_path": "media/", "mercury_path": "mercury/content.html", "pdf_path": "output.pdf", "readability_path": "readability/content.html", "screenshot_path": "screenshot.png", "singlefile_path": "singlefile.html", "warc_path": "warc/", "wget_path": null }, "domain": "allendowney.blogspot.com", "downloaded_at": "2025-03-04T20:59:58.353912+00:00", "downloaded_datestr": "2025-03-04 20:59", "extension": "html", "hash": "EBRXSN9XM2EGNT0H159B", "history": { "archive_org": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--head", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://web.archive.org/save/http://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?m=1" ], "cmd_version": "8.10.1", "end_ts": "2025-03-04T21:00:54.821549+00:00", "index_texts": null, "output": "ArchiveError: Failed to find \"content-location\" URL header in Archive.org response.", "pwd": "/data/archive/1741121993.301808", "schema": "ArchiveResult", "start_ts": "2025-03-04T21:00:52.902996+00:00", "status": "failed" } ], "dom": [ { "cmd": [ "/usr/bin/chromium-browser", "--proxy-server=socks5://tor-socks-proxy:9150", "--disable-features=DarkMode", "--run-all-compositor-stages-before-draw", "--hide-scrollbars", "--autoplay-policy=no-user-gesture-required", "--no-first-run", "--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--simulate-outdated-no-au='Tue, 31 Dec 2099 23:59:59 GMT'", "--headless=new", "--no-sandbox", "--no-zygote", "--disable-dev-shm-usage", "--disable-software-rasterizer", "--disable-sync", "--window-size=1440,2000", "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "--user-data-dir=/data/personas/Default/chrome_profile", "--profile-directory=Default", "--dump-dom", "http://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?m=1" ], "cmd_version": "131.0.6778", "end_ts": "2025-03-04T21:00:30.537725+00:00", "index_texts": null, "output": "output.html", "pwd": "/data/archive/1741121993.301808", "schema": "ArchiveResult", "start_ts": "2025-03-04T21:00:09.312129+00:00", "status": "succeeded" } ], "favicon": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--max-time", "60", "--output", "favicon.ico", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "https://www.google.com/s2/favicons?domain=allendowney.blogspot.com" ], "cmd_version": "8.10.1", "end_ts": "2025-03-04T21:00:02.038207+00:00", "index_texts": null, "output": "favicon.ico", "pwd": "/data/archive/1741121993.301808", "schema": "ArchiveResult", "start_ts": "2025-03-04T20:59:58.716973+00:00", "status": "succeeded" } ], "git": [], "headers": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--head", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "http://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?m=1" ], "cmd_version": "8.10.1", "end_ts": "2025-03-04T21:00:02.535040+00:00", "index_texts": null, "output": "headers.json", "pwd": "/data/archive/1741121993.301808", "schema": "ArchiveResult", "start_ts": "2025-03-04T21:00:02.291488+00:00", "status": "succeeded" } ], "htmltotext": [ { "cmd": [ "(internal) archivebox.extractors.htmltotext", "./{singlefile,dom}.html" ], "cmd_version": "0.8.5rc51", "end_ts": "2025-03-04T21:00:47.236926+00:00", "index_texts": [ "(https://www.blogger.com/static/v1/widgets/1539816172-widget_css_mobile_2_bundle.css) (https://allendowney.blogspot.com/favicon.ico) (http://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html) (Probably Overthinking It - Atom) (https://allendowney.blogspot.com/feeds/posts/default) (Probably Overthinking It - RSS) (https://allendowney.blogspot.com/feeds/posts/default?alt=rss) (Probably Overthinking It - Atom) (https://www.blogger.com/feeds/6894866515532737257/posts/default) (Probably Overthinking It - Atom) (https://allendowney.blogspot.com/feeds/8428645127515984892/comments/default) (https://lh3.googleusercontent.com/9fsuQt-pbezfe7mkVDGWtgTg5wqkz4vvshLDYW7-Gv-YSAtl0Y9_LQ3T2kUBIm0VnN8qLtabVrxzswFAfjmFN9gPV6_KANFyjy5GV2oMhrD7niE6kvEzfq43FdxfeKxK8g3OVQ4) Probably Overthinking It: The Inspection Paradox is Everywhere (https://www.blogger.com/dyn-css/authorization.css?targetBlogID=6894866515532737257&zx=3debb293-f8ce-41be-99a4-cc9093ef69e1) (https://www.blogger.com/dyn-css/authorization.css?targetBlogID=6894866515532737257&zx=3debb293-f8ce-41be-99a4-cc9093ef69e1) (Advertisement) (https://allendowney.blogspot.com/?m=1) Probably Overthinking It A blog by Allen Downey. Tuesday, August 18, 2015 The Inspection Paradox is Everywhere I have updated this article with new data, better code, and friendlier data visualization. (https://towardsdatascience.com/the-inspection-paradox-is-everywhere-2ef1c2e9d709?source=friends_link&sk=a38a2925e44f481d27e5b2452128716a) You can read the new version here . The Inspection Paradox is Everywhere The inspection paradox is a common source of confusion, an occasional source of error, and an opportunity for clever experimental design. \u00a0Most people are unaware of it, but like the cue marks that appear in movies to signal reel changes, once you notice it, you can\u2019t stop seeing it. A common example is the apparent paradox of class sizes. \u00a0Suppose you ask college students how big their classes are and average the responses. \u00a0The result might be 56. \u00a0But if you ask the school for the average class size, they might say 31. \u00a0It sounds like someone is lying, but they could both be right. The problem is that when you survey students, you oversample large classes. \u00a0If there are 10 students in a class, you have 10 chances to sample that class. \u00a0If there are 100 students, you have 100 chances. \u00a0In general, if the class size is x , it will be overrepresented in the sample by a factor of x . That\u2019s not necessarily a mistake. \u00a0If you want to quantify student experience, the average across students might be a more meaningful statistic than the average across classes. \u00a0But you have to be clear about what you are measuring and how you report it. By the way, I didn\u2019t make up the numbers in this example. \u00a0They come from class sizes reported by Purdue University for undergraduate classes in the 2013-14 academic year. (https://www.purdue.edu/datadigest/2013-14/InstrStuLIfe/DistUGClasses.html) https://www.purdue.edu/datadigest/2013-14/InstrStuLIfe/DistUGClasses.html From the data in their report, I estimate the actual distribution of class sizes; then I compute the \u201cbiased\u201d distribution you would get by sampling students. \u00a0The CDFs of these distributions are in Figure 1. Going the other way, if you are given the biased distribution, you can invert the process to estimate the actual distribution. \u00a0You could use this strategy if the actual distribution is not available, or if it is easier to run the biased sampling process. (purdue3.png) Figure 1: Undergraduate class sizes at Purdue University, 2013-14 academic year: actual distribution and biased view as seen by students. The same effect applies to passenger planes. \u00a0Airlines complain that they are losing money because so many flights are nearly empty. \u00a0At the same time passengers complain that flying is miserable because planes are too full. \u00a0They could both be right. \u00a0When a flight is nearly empty, only a few passengers enjoy the extra space. \u00a0But when a flight is full, many passengers feel the crunch. Once you notice the inspection paradox, you see it everywhere. \u00a0Does it seem like you can never get a taxi when you need one? \u00a0Part of the problem is that when there is a surplus of taxis, only a few customers enjoy it. \u00a0When there is a shortage, many people feel the pain. Another example happens when you are waiting for public transportation. \u00a0Buses and trains are supposed to arrive at constant intervals, but in practice some intervals are longer than others. \u00a0With your luck, you might think you are more likely to arrive during a long interval. \u00a0It turns out you are right: a random arrival is more likely to fall in a long interval because, well, it\u2019s longer. To quantify this effect, I collected data from the Red Line in Boston. \u00a0Using their real-time data service, I recorded the arrival times for 70 trains between 4pm and 5pm over several days. (redline2.png) Figure 2: Distribution of time between trains on the Red Line in Boston, between 4pm and 5pm. The shortest gap between trains was less than 3 minutes; the longest was more than 15. \u00a0Figure 2 shows the actual distribution of time between trains, and the biased distribution that would be observed by passengers. \u00a0The average time between trains is 7.8 minutes, so we might expect the average wait time to be 3.8 minutes. \u00a0But the average of the biased distribution is 8.8 minutes, and the average wait time for passengers is 4.4 minutes, about 15% longer. In this case the difference between the two distributions is not very big because the variance of the actual distribution is moderate. \u00a0When the actual distribution is long-tailed, the effect of the inspection paradox can be much bigger. An example of a long-tailed distribution comes up in the context of social networks. \u00a0In 1991, Scott Feld presented the \u201cfriendship paradox\u201d: the observation that most people have fewer friends than their friends have. \u00a0He studied real-life friends, but the same effect appears in online networks: if you choose a random Facebook user, and then choose one of their friends at random, the chance is about 80% that the friend has more friends. The friendship paradox is a form of the inspection paradox. \u00a0When you choose a random user, every user is equally likely. \u00a0But when you choose one of their friends, you are more likely to choose someone with a lot of friends. \u00a0Specifically, someone with x friends is overrepresented by a factor of x . To demonstrate the effect, I use data from the Stanford Large Network Dataset Collection ( (http://snap.stanford.edu/data) http://snap.stanford.edu/data ), which includes a sample of about 4000 Facebook users. \u00a0We can compute the number of friends each user has and plot the distribution, shown in Figure 3. \u00a0The distribution is skewed: most users have only a few friends, but some have hundreds. We can also compute the biased distribution we would get by choosing choosing random friends, also shown in Figure 3. \u00a0The difference \u00a0is huge. \u00a0In this dataset, the average user has 42 friends; the average friend has more than twice as many, 91. (social4.png) Figure 3: Number of online friends for Facebook users: actual distribution and biased distribution seen by sampling friends. Some examples of the inspection paradox are more subtle. \u00a0One of them occurred to me when I ran a 209-mile relay race in New Hampshire. \u00a0I ran the sixth leg for my team, so when I started running, I jumped into the middle of the race. \u00a0After a few miles I noticed something unusual: when I overtook slower runners, they were usually much slower; and when faster runners passed me, they were usually much faster. At first I thought the distribution of runners was bimodal, with many slow runners, many fast runners, and few runners like me in the middle. \u00a0Then I realized that I was fooled by the inspection paradox. In many long relay races, teams start at different times, and most teams include a mix of faster and slower runners. \u00a0As a result, runners at different speeds end up spread over the course; if you stand at \u00a0random spot and watch runners go by, you see a nearly representative sample of speeds. \u00a0But if you jump into the race in the middle, the sample you see depends on your speed. Whatever speed you run, you are more likely to pass very slow runners, more likely to be overtaken by fast runners, and unlikely to see anyone running at the same speed as you. \u00a0The chance of seeing another runner is proportional to the difference between your speed and theirs. We can simulate this effect using data from a conventional road race. \u00a0Figure 4 shows the actual distribution of speeds from the James Joyce Ramble, a 10K race in Massachusetts. \u00a0It also shows biased distributions that would be seen by runners at 6.5 and 7.5 mph. \u00a0The observed distributions are bimodal, with fast and slow runners oversampled and fewer runners in the middle. (relay3.png) Figure 4: Distribution of speed for runners in a 10K, and biased distributions as seen by runners at different speeds. A final example of the inspection paradox occurred to me when I was reading Orange is the New Black , a memoir by Piper Kerman, who spent 13 months in a federal prison. \u00a0At several points Kerman expresses surprise at the length of the sentences her fellow prisoners are serving. \u00a0She is right to be surprised, but it turns out that she is the victim of not just an inhumane prison system, but also the inspection paradox. If you arrive at a prison at a random time and choose a random prisoner, you are more likely to choose a prisoner with a long sentence. \u00a0Once again, a prisoner with sentence x is oversampled by a factor of x . Using data from the U.S. Sentencing Commission, I made a rough estimate of the distribution of sentences for federal prisoners, shown in Figure 5. \u00a0I also computed the biased distribution as observed by a random arrival. (orange2.png) Figure 5: Approximate distribution of federal prison sentences, and a biased distribution as seen by a random arrival. As expected, the biased distribution is shifted to the right. \u00a0In the actual distibution the mean is 121 months; in the biased distribution it is 183 months. So what happens if you observe a prison over an interval like 13 months? \u00a0It turns out that if your sentence is y months, the chance of overlapping with a prisoner whose sentence is x months is proportional to x + y . Figure 6 shows biased distributions as seen by hypothetical prisoners serving sentences of 13, 120, and 600 months. (orange6.png) Figure 6: Biased distributions as seen by prisoners with different sentences. Over an interval of 13 months, the observed sample is not much better than the biased sample seen by a random arrival. \u00a0After 120 months, the magnitude of the bias is about halved. \u00a0Only after a very long sentence, 600 months, do we get a more representative sample, and even then it is not entirely unbiased. These examples show that the inspection paradox appears in many domains, sometimes in subtle ways. \u00a0If you are not aware of it, it can cause statistical errors and lead to invalid inferences. \u00a0But in many cases it can be avoided, or even used deliberately as part of an experimental design. Further reading I discuss the class size example in my book, Think Stats , 2nd Edition, O\u2019Reilly Media, 2014, and the Red Line example in Think Bayes , O\u2019Reilly Media, 2013. \u00a0I wrote about relay races, social networks, and Orange Is the New Black in my blog, \u201cProbably Overthinking It\u201d. (http://allendowney.blogspot.com/) http://allendowney.blogspot.com/ John Allen Paulos presents the friendship paradox in Scientific American, \u201cWhy You're Probably Less Popular Than Your Friends\u201d, 18 January 2011. (http://www.scientificamerican.com/article/why-youre-probably-less-popular/) http://www.scientificamerican.com/article/why-youre-probably-less-popular/ The original paper on the topic might be Scott Feld, \u00a0\u201cWhy Your Friends Have More Friends Than You Do\u201d, American Journal of Sociology, Vol. 96, No. 6 (May, 1991), pp. 1464-1477. (http://www.jstor.org/stable/2781907) http://www.jstor.org/stable/2781907 Amir Aczel discusses some of these examples, and a few different ones, in a Discover Magazine blog article, \u201cOn the Persistence of Bad Luck (and Good)\u201d, 4 September 4, 2013. (http://blogs.discovermagazine.com/crux/2013/09/04/on-the-persistence-of-bad-luck-and-good) http://blogs.discovermagazine.com/crux/2013/09/04/on-the-persistence-of-bad-luck-and-good The code I used to generate these examples is in these Jupyter notebooks: (https://github.com/AllenDowney/ProbablyOverthinkingIt/blob/master/inspection.ipynb) https://github.com/AllenDowney/ProbablyOverthinkingIt/blob/master/inspection.ipynb (https://github.com/AllenDowney/ProbablyOverthinkingIt/blob/master/redline.ipynb) https://github.com/AllenDowney/ProbablyOverthinkingIt/blob/master/redline.ipynb (https://github.com/AllenDowney/ProbablyOverthinkingIt/blob/master/social.ipynb) https://github.com/AllenDowney/ProbablyOverthinkingIt/blob/master/social.ipynb (https://github.com/AllenDowney/ProbablyOverthinkingIt/blob/master/relay.ipynb) https://github.com/AllenDowney/ProbablyOverthinkingIt/blob/master/relay.ipynb (https://github.com/AllenDowney/ProbablyOverthinkingIt/blob/master/orange.ipynb) https://github.com/AllenDowney/ProbablyOverthinkingIt/blob/master/orange.ipynb Bio Allen Downey is a Professor of Computer Science at Olin College of Engineering in Massachusetts. \u00a0He is the author of several books, including Think Python , Think Stats , and Think Bayes . \u00a0He is a runner with a maximum 10K speed of 8.7 mph. (https://www.blogger.com/profile/01633071333405221858) (author profile) Allen Downey at (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?m=1) (permanent link) (2015-08-18T11:40:00-07:00) 11:40\u202fAM Share 24 comments: () (https://www.blogger.com/profile/09299746495471300195) Robert Dodier (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1439926542541&m=1#c7495544081961280299) August 18, 2015 at 12:35\u202fPM Thanks a lot, very informative, and enjoyable. A minor quibble: I find it easier to comprehend densities rather than cumulative distribution functions. Perhaps I can sway you to illustrate distributions by their pdfs instead of cdfs. Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=7495544081961280299) Delete Replies () (https://www.blogger.com/profile/01633071333405221858) Allen Downey (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1439931441577&m=1#c6956192266036590124) August 18, 2015 at 1:57\u202fPM I totally understand, and I am sympathetic -- I think most people are more familiar with PDFs than CDFs. But for these examples I have empirical data rather than continuous mathematical models, and in that case using PMFs or estimated PDFs creates a whole bunch of problems that CDFs neatly avoid.I decided it was better to use CDFs throughout. But it means the reader has to work a little harder, which I regret. (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=6956192266036590124) Delete Replies Reply () (https://www.blogger.com/profile/08198557649340374292) Unknown (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440013021555&m=1#c7397550448863801721) August 19, 2015 at 12:37\u202fPM yeah, the expected class size of a random student is not the sum of the class sizes divided by the number of classes, but rather the sum of the squared class sizes divided by the sum of the class sizes.good examples though. another very obvious one is the speed of cars you pass on the highway.be aware of one limitation to your inversion: by looking at the samples, you cannot determine the number of classes with zero students, so you'll never know the actual average class size. (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=7397550448863801721) Delete Replies Reply () (https://www.blogger.com/profile/01633071333405221858) Allen Downey (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440013875188&m=1#c4944536262773523923) August 19, 2015 at 12:51\u202fPM Good point. Similarly with the relay race example, you can't tell how many runners are the same speed as you. (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=4944536262773523923) Delete Replies Reply Reply () (https://www.blogger.com/profile/05396839866731499408) luispedro (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1439970171294&m=1#c8052679278678616634) August 19, 2015 at 12:42\u202fAM My favorite example (true in many European countries): most families have a single child, but most kids have siblings. Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=8052679278678616634) Delete Replies () (https://www.blogger.com/profile/09604463877214596048) Pierre (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440142887297&m=1#c3421591696234941075) August 21, 2015 at 12:41\u202fAM Even after having read the article, it took me 10 minutes to figure it out ;) (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=3421591696234941075) Delete Replies Reply () (https://www.blogger.com/profile/01633071333405221858) Allen Downey (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440363427581&m=1#c8929021211445605460) August 23, 2015 at 1:57\u202fPM Good one. Thanks! (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=8929021211445605460) Delete Replies Reply Reply () (https://www.blogger.com/profile/09089984219014317886) Nu\u00f1o (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440062252081&m=1#c5009238912738524529) August 20, 2015 at 2:17\u202fAM This is a fascinating post, thanks. Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=5009238912738524529) Delete Replies Reply () (https://www.blogger.com/profile/09879280683468964072) Steve (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440098963212&m=1#c151468784020347419) August 20, 2015 at 12:29\u202fPM Nice post. An example from my own field: The inspection paradox causes most students to get the wrong answer by a factor of 2 in a certain physics calculation, related to electron motion in solids - see http://physics.stackexchange.com/questions/88015/definition-of-mean-free-time-in-the-drude-model/88045#88045 Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=151468784020347419) Delete Replies Reply () (https://www.blogger.com/profile/08965300451627112100) Craig (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440124211512&m=1#c1963445459873560535) August 20, 2015 at 7:30\u202fPM Allen, nice article...well-written and easy to understand. One suggestion: use \"interarrival time\" in the train example instead of \"arrival time\" because the former is standard queueing parlance. Thanks. Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=1963445459873560535) Delete Replies Reply () (https://www.blogger.com/profile/08012486411258768407) 4verageJo3 (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440155751682&m=1#c585719875779286897) August 21, 2015 at 4:15\u202fAM Here's something you might want to clarify:\"when there is a surplus of taxis, only a few customers enjoy it. When there is a shortage, many people feel the pain.\" This is true if the number of taxis stays constant and the people vary; but if the number of people stays constant and the taxis vary, it doesn't hold. Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=585719875779286897) Delete Replies () (https://www.blogger.com/profile/01633071333405221858) Allen Downey (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440363506521&m=1#c5565322114734245461) August 23, 2015 at 1:58\u202fPM Good point. I will revise this and talk about changes of demand rather than supply, which I think will be clearer and avoid the issue you raised. (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=5565322114734245461) Delete Replies Reply Reply () (https://www.blogger.com/profile/15958070809506087817) Jim (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440160034784&m=1#c3478249254921846042) August 21, 2015 at 5:27\u202fAM If [there] are 10 students in a class, you have 10 chances to sample that class.- great article! Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=3478249254921846042) Delete Replies Reply () (https://www.blogger.com/profile/06143376466358780225) fengshuileung (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440176549806&m=1#c1961129926378041927) August 21, 2015 at 10:02\u202fAM nice piece, thx. Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=1961129926378041927) Delete Replies Reply () (https://www.blogger.com/profile/13039129656717602178) Roger (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440281030395&m=1#c6415400875842220232) August 22, 2015 at 3:03\u202fPM Worth reading. I recall when the PC and Windows first hit the business desktop, being dumbfound by users tolerance of the rate at which they crashed. At the time I was accustomed to mature IBM AS/400 super mini hardware that had an MTBF of about 5 years, and I realized that when PC crashed, it was just one person who was impacted by it, but when the main computer for the whole company crashed, everyone experienced the pain at the same time. Even though the total man-hours a PC was down was orders of magnitude greater than the man-hours the AS/400 was down, the distribution of the pain a little at a time made is seem like less. \ufeff Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=6415400875842220232) Delete Replies Reply () (https://www.blogger.com/profile/08967761812729609699) Unknown (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440293794396&m=1#c2933391668642365310) August 22, 2015 at 6:36\u202fPM Very nice indeed. I shared this via G+ and got a lot of very positive reader feedback on your article as well. (https://plus.google.com/+YonatanZunger/posts/FjCWr1SySQg)One question: Does it make sense to use PDF's instead of CDF's to illustrate this? With the jogging example (figure 4) in particular, I think a PDF would have highlighted the bimodality far more vividly. (As well as being much more accessible to a non-specialist audience) Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=2933391668642365310) Delete Replies () (https://www.blogger.com/profile/01633071333405221858) Allen Downey (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440363716817&m=1#c2904563190080581200) August 23, 2015 at 2:01\u202fPM Yes, PDFs are definitely easier to read for most people. The problem is that if you have discrete data, you have to smooth the PDFs to see the shape, and if you smooth too much, you obliterate the bimodality. And even if you get it right, it is harder to compare multiple PDFs to see the kind of shifts I want to show.But since several people have asked, I might try again to make PDF-based visualizations for these examples and see if I can make them work. (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=2904563190080581200) Delete Replies Reply Reply () (https://www.blogger.com/profile/18349479103216755952) gwern (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1440434928945&m=1#c2044953260307876459) August 24, 2015 at 9:48\u202fAM > Once you notice the inspection paradox, you see it everywhere. Does it seem like you can never get a taxi when you need one? And of course, besides buses, taxis, and planes, there's cars: why does the highway so often seem to be jammed? Because if it's jammed, there's a lot more people in it than when it's moving smoothly; I saw this version in Nick Bostrom's http://www.anthropic-principle.com/?q=book/table_of_contentsWhich makes me wonder if there is any mode of transportation that the inspection paradox *doesn't* apply to... perhaps when the bucket or lump size equals one and so there is no difference in sampling, like walking or riding your bike or roller skating? Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=2044953260307876459) Delete Replies Reply () (https://www.blogger.com/profile/18248083811361997088) Unknown (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1442000404528&m=1#c3233102041005467952) September 11, 2015 at 12:40\u202fPM maybe not so many people experience what you say about runnersbut if you drive long-distance (above an hour) you'll have the same impression about cars on the road \"nobody is driving like me, either too slow or too fast\" Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=3233102041005467952) Delete Replies () (https://www.blogger.com/profile/01633071333405221858) Allen Downey (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1442001706168&m=1#c3243604165178682281) September 11, 2015 at 1:01\u202fPM Good example! (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=3243604165178682281) Delete Replies Reply Reply () (https://www.blogger.com/profile/12159599444378188669) Anon (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1444064506832&m=1#c5612084898687107968) October 5, 2015 at 10:01\u202fAM I wonder how much of the Heuristics and Biases literature in psychology is attributing real biases (like these) to mistakes in the mind! Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=5612084898687107968) Delete Replies Reply () (https://www.blogger.com/profile/01013886950887317666) agnieszka (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1444508717366&m=1#c694716164179316742) October 10, 2015 at 1:25\u202fPM I mostly get this, but what I'm still confused about is the class sizes example. If you have a class size of, say, 15 (according to school admin records) but asking a student they tell you size of 23 - how on earth can they possibly be both right? It's not like the students sit faster or attend a longer class... surely counting a static number by everyone should reveal the same number?? They're not running in class, after all - it's easy to count occupied desks when everyone is seated. Also you did not explain/say/expand what a \"CDF\" is.... Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=694716164179316742) Delete Replies () (https://www.blogger.com/profile/01633071333405221858) Allen Downey (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1444511873057&m=1#c2149756276521319270) October 10, 2015 at 2:17\u202fPM There has to be some variation in class sizes. Suppose there are only two classes, with sizes 1 and 9. The average class size is 5, but the average student report is 8.2.For more about CDFs, see Chapter 4 of Think Stats: http://greenteapress.com/thinkstats2/html/thinkstats2005.html (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=2149756276521319270) Delete Replies Reply Reply () (https://www.blogger.com/profile/03049412878821578827) Sharda Hinkel (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?showComment=1450691723520&m=1#c6356184392602468927) December 21, 2015 at 1:55\u202fAM I must say that's impressive post. Reply (https://www.blogger.com/delete-comment.g?blogID=6894866515532737257&postID=6356184392602468927) Delete Replies Reply Add comment Load more... (https://www.blogger.com/comment/frame/6894866515532737257?po=8428645127515984892&hl=en&m=1&blogspotRpcToken=9248709) (https://allendowney.blogspot.com/2015/08/bayes-theorem-in-real-life.html?m=1) (Newer Post) \u2039 (https://allendowney.blogspot.com/2015/08/orange-is-new-stat.html?m=1) (Older Post) \u203a (https://allendowney.blogspot.com/?m=1) Home (https://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?m=0) View web version About Me (https://www.blogger.com/profile/01633071333405221858) Allen Downey (https://www.blogger.com/profile/01633071333405221858) View my complete profile Powered by (https://www.blogger.com) Blogger . (Advertisement) " ], "output": "htmltotext.txt", "pwd": "/data/archive/1741121993.301808", "schema": "ArchiveResult", "start_ts": "2025-03-04T21:00:47.126695+00:00", "status": "succeeded" } ], "media": [ { "cmd": [ "/usr/local/bin/yt-dlp", "--restrict-filenames", "--trim-filenames", "128", "--write-description", "--write-info-json", "--write-annotations", "--write-thumbnail", "--no-call-home", "--write-sub", "--write-auto-subs", "--convert-subs=srt", "--yes-playlist", "--continue", "--no-abort-on-error", "--ignore-errors", "--geo-bypass", "--add-metadata", "--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)", "--skip-download", "--cache-dir=/data/yt-dlp-cache/", "--cookies=/data/yt-dlp-cache/cookies.txt", "--proxy=socks5://tor-socks-proxy:9150", "--no-playlist", "http://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?m=1" ], "cmd_version": "2024.10.7", "end_ts": "2025-03-04T21:00:52.876938+00:00", "index_texts": [], "output": "media/", "pwd": "/data/archive/1741121993.301808", "schema": "ArchiveResult", "start_ts": "2025-03-04T21:00:48.594433+00:00", "status": "succeeded" } ], "mercury": [ { "cmd": [ "/home/archivebox/.npm/bin/postlight-parser", "http://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?m=1" ], "cmd_version": "2.2.3", "end_ts": "2025-03-04T21:00:46.976891+00:00", "index_texts": null, "output": "mercury/", "pwd": "/data/archive/1741121993.301808", "schema": "ArchiveResult", "start_ts": "2025-03-04T21:00:44.092191+00:00", "status": "succeeded" } ], "pdf": [], "readability": [ { "cmd": [ "/home/archivebox/.npm/bin/readability-extractor", "/tmp/tmpvqlsix5g", "http://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?m=1" ], "cmd_version": "0.0.11", "end_ts": "2025-03-04T21:00:35.648386+00:00", "index_texts": [ "The inspection paradox is a common source of confusion, an occasional source of error, and an opportunity for clever experimental design. \u00a0Most people are unaware of it, but like the cue marks that appear in movies to signal reel changes, once you notice it, you can\u2019t stop seeing it.\nA common example is the apparent paradox of class sizes. \u00a0Suppose you ask college students how big their classes are and average the responses. \u00a0The result might be 56. \u00a0But if you ask the school for the average class size, they might say 31. \u00a0It sounds like someone is lying, but they could both be right.\nThe problem is that when you survey students, you oversample large classes. \u00a0If there are 10 students in a class, you have 10 chances to sample that class. \u00a0If there are 100 students, you have 100 chances. \u00a0In general, if the class size is x, it will be overrepresented in the sample by a factor of x.\nThat\u2019s not necessarily a mistake. \u00a0If you want to quantify student experience, the average across students might be a more meaningful statistic than the average across classes. \u00a0But you have to be clear about what you are measuring and how you report it.\n\nFrom the data in their report, I estimate the actual distribution of class sizes; then I compute the \u201cbiased\u201d distribution you would get by sampling students. \u00a0The CDFs of these distributions are in Figure 1.\nGoing the other way, if you are given the biased distribution, you can invert the process to estimate the actual distribution. \u00a0You could use this strategy if the actual distribution is not available, or if it is easier to run the biased sampling process.\n\nFigure 1: Undergraduate class sizes at Purdue University, 2013-14 academic year: actual distribution and biased view as seen by students.\nThe same effect applies to passenger planes. \u00a0Airlines complain that they are losing money because so many flights are nearly empty. \u00a0At the same time passengers complain that flying is miserable because planes are too full. \u00a0They could both be right. \u00a0When a flight is nearly empty, only a few passengers enjoy the extra space. \u00a0But when a flight is full, many passengers feel the crunch.\nOnce you notice the inspection paradox, you see it everywhere. \u00a0Does it seem like you can never get a taxi when you need one? \u00a0Part of the problem is that when there is a surplus of taxis, only a few customers enjoy it. \u00a0When there is a shortage, many people feel the pain.\nAnother example happens when you are waiting for public transportation. \u00a0Buses and trains are supposed to arrive at constant intervals, but in practice some intervals are longer than others. \u00a0With your luck, you might think you are more likely to arrive during a long interval. \u00a0It turns out you are right: a random arrival is more likely to fall in a long interval because, well, it\u2019s longer.\nTo quantify this effect, I collected data from the Red Line in Boston. \u00a0Using their real-time data service, I recorded the arrival times for 70 trains between 4pm and 5pm over several days. \n\nFigure 2: Distribution of time between trains on the Red Line in Boston, between 4pm and 5pm.\nThe shortest gap between trains was less than 3 minutes; the longest was more than 15. \u00a0Figure 2 shows the actual distribution of time between trains, and the biased distribution that would be observed by passengers. \u00a0The average time between trains is 7.8 minutes, so we might expect the average wait time to be 3.8 minutes. \u00a0But the average of the biased distribution is 8.8 minutes, and the average wait time for passengers is 4.4 minutes, about 15% longer.\nIn this case the difference between the two distributions is not very big because the variance of the actual distribution is moderate. \u00a0When the actual distribution is long-tailed, the effect of the inspection paradox can be much bigger.\nAn example of a long-tailed distribution comes up in the context of social networks. \u00a0In 1991, Scott Feld presented the \u201cfriendship paradox\u201d: the observation that most people have fewer friends than their friends have. \u00a0He studied real-life friends, but the same effect appears in online networks: if you choose a random Facebook user, and then choose one of their friends at random, the chance is about 80% that the friend has more friends.\nThe friendship paradox is a form of the inspection paradox. \u00a0When you choose a random user, every user is equally likely. \u00a0But when you choose one of their friends, you are more likely to choose someone with a lot of friends. \u00a0Specifically, someone with x friends is overrepresented by a factor of x.\nTo demonstrate the effect, I use data from the Stanford Large Network Dataset Collection (http://snap.stanford.edu/data), which includes a sample of about 4000 Facebook users. \u00a0We can compute the number of friends each user has and plot the distribution, shown in Figure 3. \u00a0The distribution is skewed: most users have only a few friends, but some have hundreds.\nWe can also compute the biased distribution we would get by choosing choosing random friends, also shown in Figure 3. \u00a0The difference \u00a0is huge. \u00a0In this dataset, the average user has 42 friends; the average friend has more than twice as many, 91.\n\nFigure 3: Number of online friends for Facebook users: actual distribution and biased distribution seen by sampling friends.\nSome examples of the inspection paradox are more subtle. \u00a0One of them occurred to me when I ran a 209-mile relay race in New Hampshire. \u00a0I ran the sixth leg for my team, so when I started running, I jumped into the middle of the race. \u00a0After a few miles I noticed something unusual: when I overtook slower runners, they were usually much slower; and when faster runners passed me, they were usually much faster.\nAt first I thought the distribution of runners was bimodal, with many slow runners, many fast runners, and few runners like me in the middle. \u00a0Then I realized that I was fooled by the inspection paradox.\nIn many long relay races, teams start at different times, and most teams include a mix of faster and slower runners. \u00a0As a result, runners at different speeds end up spread over the course; if you stand at \u00a0random spot and watch runners go by, you see a nearly representative sample of speeds. \u00a0But if you jump into the race in the middle, the sample you see depends on your speed.\nWhatever speed you run, you are more likely to pass very slow runners, more likely to be overtaken by fast runners, and unlikely to see anyone running at the same speed as you. \u00a0The chance of seeing another runner is proportional to the difference between your speed and theirs.\nWe can simulate this effect using data from a conventional road race. \u00a0Figure 4 shows the actual distribution of speeds from the James Joyce Ramble, a 10K race in Massachusetts. \u00a0It also shows biased distributions that would be seen by runners at 6.5 and 7.5 mph. \u00a0The observed distributions are bimodal, with fast and slow runners oversampled and fewer runners in the middle.\n\nFigure 4: Distribution of speed for runners in a 10K, and biased distributions as seen by runners at different speeds.\nA final example of the inspection paradox occurred to me when I was reading Orange is the New Black, a memoir by Piper Kerman, who spent 13 months in a federal prison. \u00a0At several points Kerman expresses surprise at the length of the sentences her fellow prisoners are serving. \u00a0She is right to be surprised, but it turns out that she is the victim of not just an inhumane prison system, but also the inspection paradox.\nIf you arrive at a prison at a random time and choose a random prisoner, you are more likely to choose a prisoner with a long sentence. \u00a0Once again, a prisoner with sentence x is oversampled by a factor of x.\nUsing data from the U.S. Sentencing Commission, I made a rough estimate of the distribution of sentences for federal prisoners, shown in Figure 5. \u00a0I also computed the biased distribution as observed by a random arrival.\n\nFigure 5: Approximate distribution of federal prison sentences, and a biased distribution as seen by a random arrival.\nAs expected, the biased distribution is shifted to the right. \u00a0In the actual distibution the mean is 121 months; in the biased distribution it is 183 months.\nSo what happens if you observe a prison over an interval like 13 months? \u00a0It turns out that if your sentence is y months, the chance of overlapping with a prisoner whose sentence is x months is proportional to x + y.\nFigure 6 shows biased distributions as seen by hypothetical prisoners serving sentences of 13, 120, and 600 months.\n\nFigure 6: Biased distributions as seen by prisoners with different sentences.\nOver an interval of 13 months, the observed sample is not much better than the biased sample seen by a random arrival. \u00a0After 120 months, the magnitude of the bias is about halved. \u00a0Only after a very long sentence, 600 months, do we get a more representative sample, and even then it is not entirely unbiased.\nThese examples show that the inspection paradox appears in many domains, sometimes in subtle ways. \u00a0If you are not aware of it, it can cause statistical errors and lead to invalid inferences. \u00a0But in many cases it can be avoided, or even used deliberately as part of an experimental design.\nFurther reading\nI discuss the class size example in my book, Think Stats, 2nd Edition, O\u2019Reilly Media, 2014, and the Red Line example in Think Bayes, O\u2019Reilly Media, 2013. \u00a0I wrote about relay races, social networks, and Orange Is the New Black in my blog, \u201cProbably Overthinking It\u201d. \u00a0http://allendowney.blogspot.com/\n\nThe original paper on the topic might be Scott Feld, \u00a0\u201cWhy Your Friends Have More Friends Than You Do\u201d, American Journal of Sociology, Vol. 96, No. 6 (May, 1991), pp. 1464-1477. http://www.jstor.org/stable/2781907\nAmir Aczel discusses some of these examples, and a few different ones, in a Discover Magazine blog article, \u201cOn the Persistence of Bad Luck (and Good)\u201d, 4 September 4, 2013.\n\nThe code I used to generate these examples is in these Jupyter notebooks:\n\n\n\n\n\nBio\n\nAllen Downey is a Professor of Computer Science at Olin College of Engineering in Massachusetts. \u00a0He is the author of several books, including Think Python, Think Stats, and Think Bayes. \u00a0He is a runner with a maximum 10K speed of 8.7 mph." ], "output": "readability/", "pwd": "/data/archive/1741121993.301808", "schema": "ArchiveResult", "start_ts": "2025-03-04T21:00:31.845842+00:00", "status": "succeeded" } ], "screenshot": [], "singlefile": [], "title": [ { "cmd": [ "/usr/bin/curl", "--silent", "--location", "--compressed", "--proxy", "socks5://tor-socks-proxy:9150", "--max-time", "60", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", "http://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?m=1" ], "cmd_version": "8.10.1", "end_ts": "2025-03-04T21:00:31.291513+00:00", "index_texts": null, "output": "Probably Overthinking It: The Inspection Paradox is Everywhere", "pwd": "/data/archive/1741121993.301808", "schema": "ArchiveResult", "start_ts": "2025-03-04T21:00:31.070501+00:00", "status": "succeeded" } ], "wget": [] }, "icons": null, "is_archived": true, "is_static": false, "latest": { "archive_org": "ArchiveError: Failed to find \"content-location\" URL header in Archive.org response.", "dom": "output.html", "favicon": "favicon.ico", "git": null, "media": "media/", "pdf": null, "screenshot": null, "singlefile": null, "title": "Probably Overthinking It: The Inspection Paradox is Everywhere", "warc": null, "wget": null }, "link_dir": "/data/archive/1741121993.301808", "newest_archive_date": "2025-03-04T21:00:52.902996+00:00", "num_failures": 1, "num_outputs": 8, "oldest_archive_date": "2025-03-04T20:59:58.716973+00:00", "path": "/2015/08/the-inspection-paradox-is-everywhere.html", "schema": "Link", "scheme": "http", "snapshot_abid": "snp_01JNHFAEM452AAB63B018DCWYA", "snapshot_id": "6f305db7-f0b2-44a8-9f17-6b6310d673ca", "sources": [ "/data/sources/1741121992-import.txt" ], "tags": null, "tags_str": "", "timestamp": "1741121993.301808", "title": "Probably Overthinking It: The Inspection Paradox is Everywhere", "url": "http://allendowney.blogspot.com/2015/08/the-inspection-paradox-is-everywhere.html?m=1" }