michael/minify-html
michael
/
minify-html
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add bench

This commit is contained in:
Wilson Lin 2019-12-30 19:58:50 +11:00
parent 9321e2769b
commit 6ed63afda8
12 changed files with 58105 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
bench/bench.js Normal file
View File

@ -0,0 +1,95 @@
"use strict";
const fs = require("fs");
const path = require("path");
const benchmark = require("benchmark");
const htmlMinifier = require("html-minifier");
const minimize = require("minimize");
const hyperbuild = require("hyperbuild");
const tests_dir = path.join(__dirname, "tests");
const tests = fs.readdirSync(tests_dir).map(name => ({
name,
content: fs.readFileSync(path.join(tests_dir, name), "utf8"),
}));
const sizes = {};
const setSize = (program, test, result) => {
console.log(`Received result for ${program} - ${test}`);
if (!sizes[test]) {
sizes[test] = {
original: {
result: tests.find(t => t.name === test).content.length,
},
};
}
const original = sizes[test].original.result;
sizes[test][program] = {
result: result,
difference: `${((result - original) / original * 100).toFixed(2)}%`,
};
};
const htmlMinifierSettings = {
caseSensitive: false,
collapseBooleanAttributes: true,
collapseInlineTagWhitespace: true,
collapseWhitespace: true,
conservativeWhitespace: false,
customEventAttributes: [],
decodeEntities: true,
html5: true,
ignoreCustomComments: [],
ignoreCustomFragments: [],
includeAutoGeneratedTags: true,
keepClosingSlash: false,
minifyCSS: false,
minifyJS: false,
minifyURLs: false,
preserveLineBreaks: false,
preventAttributesEscaping: false,
processConditionalComments: true,
processScripts: [],
removeAttributeQuotes: true,
removeComments: true,
removeEmptyAttributes: false,
removeEmptyElements: false,
removeOptionalTags: true,
removeRedundantAttributes: true,
removeScriptTypeAttributes: true,
removeStyleLinkTypeAttributes: true,
removeTagWhitespace: true,
sortAttributes: true,
sortClassName: true,
trimCustomFragments: false,
useShortDoctype: true,
};
new benchmark.Suite()
.add("hyperbuild", () => {
for (const t of tests) {
setSize("hyperbuild", t.name, hyperbuild.minify(Buffer.from(t.content)));
}
})
.add("html-minifier", () => {
for (const t of tests) {
setSize("html-minifier", t.name, htmlMinifier.minify(t.content, htmlMinifierSettings).length);
}
})
.add("minimize", () => {
for (const t of tests) {
setSize("minimize", t.name, new minimize().parse(t.content).length);
}
})
.on('cycle', event => {
console.info(event.target.toString());
})
.on('complete', function () {
console.info(`Fastest is ${this.filter('fastest').map('name')}`);
Object.entries(sizes).forEach(([test, results]) => {
console.info(test);
console.table(results);
});
})
.run({'async': true});

12
bench/package.json Normal file
View File

@ -0,0 +1,12 @@
{
"private": true,
"dependencies": {
"benchmark": "2.1.4",
"html-minifier": "3.5.19",
"minimize": "2.2.0",
"hyperbuild": "0.0.4"
},
"scripts": {
"start": "node bench.js"
}
}

4566
bench/tests/amazon.html Normal file
View File

File diff suppressed because one or more lines are too long

479
bench/tests/bbc.html Normal file
View File

File diff suppressed because one or more lines are too long

1108
bench/tests/bootstrap.html Normal file
View File

File diff suppressed because one or more lines are too long

671
bench/tests/codinghorror.html Normal file
View File

@ -0,0 +1,671 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title>Coding Horror</title>
<meta name="description" content="programming and human factors" />
<meta name="HandheldFriendly" content="True" />
<meta name="MobileOptimized" content="320" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="google-site-verification" content="sl0m9SU_4V0JcvjWlOX4dUFBR6VS2P4tlxjJMo0gphU" />
<script src="/cdn-cgi/apps/head/SYM5cnEAeGsP9NDCaVwd5sK2KK4.js"></script><link rel="shortcut icon" href="/favicon.ico">
<link rel="apple-touch-icon" href="/assets/images/codinghorror-app-icon.png?v=42aa5dfcd1">
<link rel="stylesheet" type="text/css" href="/assets/css/screen.css?v=42aa5dfcd1" />
<link rel="stylesheet" type="text/css" href="//fonts.googleapis.com/css?family=Open+Sans:400italic,700italic,400,700" />
<link rel="alternate" type="application/rss+xml" title="Coding Horror" href="http://feeds.feedburner.com/codinghorror">
<script src="https://code.jquery.com/jquery-2.2.3.min.js" async></script>
<script src="/assets/js/video-resize.js?v=42aa5dfcd1" async></script>
</head>
<body class="home-template">
<header class="site-head">
<div class="site-head-content">
<a class="blog-logo" href="https://blog.codinghorror.com"><img src="/assets/images/codinghorror-app-icon.png?v=42aa5dfcd1" alt="Coding Horror Logo" width="158" height="158" /></a>
<h1 class="blog-title"><a href="https://blog.codinghorror.com">Coding Horror</a></h1>
<h2 class="blog-description">programming and human factors</h2>
<div class="site-search">
<script>
(function() {
var gcse = document.createElement('script');
gcse.type = 'text/javascript';
gcse.async = true;
gcse.src = 'https://www.google.com/cse/cse.js?cx=016956275695630057531:lqveu9tah7y';
var s = document.getElementsByTagName('script')[0];
s.parentNode.insertBefore(gcse, s);
})();
</script>
<gcse:search></gcse:search>
</div>
</div>
</header>
<div class="wrap clearfix">
<div class="clearfix"></div>
<main class="content" role="main">
<article class="post">
<header class="post-header">
<span class="post-meta"><time datetime="2017-12-31">31 Dec 2017</time> </span>
<h2 class="post-title"><a href="/to-serve-man-with-software/">To Serve Man, with Software</a></h2>
</header>
<section class="post-content">
<div class="kg-card-markdown"><p>I didn't choose to be a programmer. Somehow, it seemed, <a href="https://blog.codinghorror.com/if-loving-computers-is-wrong-i-dont-want-to-be-right/">the computers chose me</a>. For a long time, that was fine, that was enough; that was all I needed. But along the way I never felt that being a programmer was this <a href="https://blog.codinghorror.com/please-dont-learn-to-code/">unambiguously great-for-everyone</a> career field with zero downsides. There are absolutely occupational hazards of being a programmer, and <a href="https://blog.codinghorror.com/your-favorite-programming-quote/">one of my favorite programming quotes</a> is an allusion to one of them:</p>
<blockquote>
<p>It should be noted that no ethically-trained software engineer would ever consent to write a <code>DestroyBaghdad</code> procedure. Basic professional ethics would instead require him to write a <code>DestroyCity</code> procedure, to which Baghdad could be given as a parameter.</p>
</blockquote>
<p>Which reminds me of <a href="https://waxy.org/2015/12/tracking_the_trump_is_a_comment_section_running_for_president_joke/">another joke that people were telling in 2015</a>:</p>
<blockquote>
<p>Donald Trump is basically a comment section running for president</p>
</blockquote>
<p>Which is troubling because technically, <em>technically</em>, I run a company that <a href="https://discourse.org">builds comment sections</a>.</p>
<p>Here at the tail end of 2017, from where I sit neither of these jokes seem particularly funny to me any more. Perhaps I have lost the capacity to feel joy as a human being? <em>Haha just kidding!</em> <em>... kinda.</em></p>
<p>Remember <a href="https://www.wsj.com/articles/SB10001424053111903480904576512250915629460">in 2011</a> when Marc Andreeseen said that &quot;Software is eating the world?&quot;</p>
<p><img src="/content/images/2017/12/software-is-eating-the-world-marc-andreessen.jpg" alt="software is eating the world, Marc Andreessen"></p>
<p>That used to sound all hip and cool and inspirational, like &quot;Wow! We software developers really <em>are</em> making a difference in the world!&quot; and now for the life of me I can't read it as anything other than an ominous warning that we just weren't smart enough to translate properly at the time. But <a href="https://en.wikipedia.org/wiki/To_Serve_Man_(The_Twilight_Zone)">maybe now we are</a>.</p>
<p><a href="https://en.wikipedia.org/wiki/To_Serve_Man_(The_Twilight_Zone)"><img src="/content/images/2017/12/to-serve-man.jpg" alt="to-serve-man"></a></p>
<p>I've said many, many times that the key to becoming an experienced software developer is to understand that you are, at all times, <a href="https://blog.codinghorror.com/on-the-meaning-of-coding-horror/">your own worst enemy</a>. I don't mean this in a negative way – you have to constantly plan for and design around your inevitable human mistakes and fallibility. It's fundamental to good software engineering because, well, we're all human. The good-slash-bad news is that you're only <a href="https://blog.codinghorror.com/the-trap-you-set-for-yourself/"><em>accidentally</em> out to get yourself</a>. But what happens when we're infinitely connected and software is suddenly <em>everywhere</em>, in everyone's pockets every moment of the day, starting to approximate a natural extension of our bodies? All of a sudden those little collective social software accidents become <a href="https://www.wired.com/story/the-other-tech-bubble/">considerably more dangerous</a>:</p>
<blockquote>
<p>The issue is bigger than any single scandal, I told him. As headlines have exposed the troubling inner workings of company after company, startup culture no longer feels like fodder for gentle parodies about ping pong and hoodies. It feels ugly and rotten. Facebook, the greatest startup success story of this era, isn’t a merry band of hackers building cutesy tools that allow you to digitally Poke your friends. It’s a powerful and potentially sinister collector of personal data, a propaganda partner to government censors, and an enabler of discriminatory advertising.</p>
</blockquote>
<p>I'm reminded of a particular Mitchell and Webb skit: <em>&quot;Are we the baddies?&quot;</em></p>
<iframe width="560" height="315" src="https://www.youtube.com/embed/qv2XGQBcvxQ" frameborder="0" gesture="media" allow="encrypted-media" allowfullscreen></iframe>
<p>On the topic of unanticipated downsides to technology, there is no show more essential than <a href="https://www.netflix.com/title/70264888">Black Mirror</a>. If you haven't watched Black Mirror yet, do not pass go, do not collect $200, go immediately to Netflix and watch it. Go on! Go ahead!</p>
<iframe width="560" height="315" src="https://www.youtube.com/embed/jDiYGjp5iFg" frameborder="0" gesture="media" allow="encrypted-media" allowfullscreen></iframe>
<blockquote>
<p>⚠ Fair warning: please DO NOT start with season 1 episode 1 of Black Mirror! Start with season 3, and go forward. If you like those, dip into season 2 and the just-released season 4, then the rest. But humor me and please at least watch the first episode of season 3.</p>
</blockquote>
<p>The technology described in Black Mirror can be fanciful at times, but several episodes portray disturbingly plausible scenarios with <em>today's</em> science and tech, much less what we'll have 20 to 50 years from now. These are very real cautionary tales, and some of this stuff is well on its way toward being realized.</p>
<p>Programmers don't think of themselves as people with the power to change the world. Most programmers I know, including myself, grew up as nerds, geeks, social outcasts. Did I ever tell you about the time I wrote a self-destructing Apple // boot disk program to let a girl in middle school know that I liked her? I was (and still am) a terrible programmer, but oh man did I ever test the heck out of <em>that</em> code before copying on to her school floppy disc. But I digress. What do you do when you wake up one day and software <em>has</em> kind of eaten the world, and <strong>it is no longer clear if software is in fact an unambiguously good thing, like we thought, like everyone told us … like we <em>wanted it to be?</em></strong></p>
<p>Months ago I submitted a brief interview for a <a href="https://www.amazon.com/dp/1465462333/?tag=codihorr-20">children's book about coding</a>.</p>
<p><a href="https://www.amazon.com/dp/1465462333/?tag=codihorr-20"><img src="/content/images/2017/12/9780241285060.jpg" width="400"></a></p>
<p>I recently recieved a complimentary copy of the book in the mail. I paged to <a href="/content/images/2017/12/dk-findout-coding-page-50.jpg">my short interview</a>, alongside the very cool <a href="http://www.prottsman.com/">Kiki Prottsman</a>. I had no real recollection of the interview questions after the months of lead time it takes to print a physical book, but reading the printed page, I suddenly hit myself over the head with the very answer I had been searching my soul for these past 6 months:</p>
<p><a href="/content/images/2017/12/dk-findout-coding-page-50.jpg"><img src="/content/images/2017/12/dk-findout-coding-page-50-quote.jpg" alt="Jeff Atwood quote: what do you love most about coding?"></a></p>
<p>In attempting to simplify my answers for an audience of kids, I had concisely articulated the one thing that keeps me coming back to software: <strong>to serve man</strong>. Not on a platter, for bullshit monetization – but software that <a href="https://blog.codinghorror.com/the-just-in-time-theory/">helps people</a> be the best version of themselves.</p>
<iframe width="560" height="315" src="https://www.youtube.com/embed/cNi_HC839Wo?start=118" frameborder="0" gesture="media" allow="encrypted-media" allowfullscreen></iframe>
<p>And you know why I do it? I need that help, too. I get tired, angry, upset, emotional, cranky, irritable, frustrated and I need to be reminded from time to time to choose to be the better version of myself. I don't always succeed. But <em>I want to</em>. And I believe everyone else – for some reasonable statistical value of everyone else – fundamentally does, too.</p>
<p>That was the not-so-secret design philosophy behind Stack Overflow, that <strong>by helping others become better programmers, you too would become a better programmer</strong>. It's unavoidable. And, even better, if we leave enough helpful breadcrumbs behind for those that follow us, <em>we collectively advance the whole of programming for everyone</em>.</p>
<p>I apologize for not blogging much in 2017. I've certainly been busy with Discourse which is actually going great; we grew to 21 people and <a href="https://blog.discourse.org/2017/12/discourse-gives-back-2017/">gave $55,000 back</a> this year to the open source ecosystem we build on. But that's no excuse. The truth is that it's been hard to write because this has been a deeply troubling year in so many dimensions — for men, for tech, for American democracy. I'm ashamed of much that happened, and I think one of the first and most important steps we can take is to <a href="https://blog.codinghorror.com/the-hugging-will-continue-until-morale-improves/">embrace explicit codes of conduct</a> throughout our industry. I also continue to believe, if we start to think more holistically about what our software can do to <strong>serve all people</strong>, not just ourselves personally (or, even worse, the company we work for) — that software can and should be part of the solution.</p>
<p>I tried to amplify on these thoughts in recent podcasts:</p>
<table>
<tr>
<td><img src="/content/images/2017/12/kim-crayton.jpg" width="100" height="100"></td>
<td>&nbsp;<a href="http://www.kimcrayton.com/commengreport-jeff-atwood/">Community Engineering Report</a> with Kim Crayton</td>
</tr>
<tr>
<td><img src="/content/images/2017/12/dave-rael.jpg" width="100" height="100"></td>
<td>&nbsp;<a href="http://developeronfire.com/podcast/episode-258-jeff-atwood-sharing-the-house">Developer on Fire</a> with Dave Rael</td>
</tr>
<tr>
<td><img src="/content/images/2017/12/william-channer.jpg" width="100" height="100"></td>
<td>&nbsp;<a href="https://drt.fm/jeff-atwood">Dorm Room Tycoon</a> with William Channer</td>
</tr>
</table>
<p>Software is easy to change, but people ... aren't. So in the new year, as software developers, let's make a resolution to focus on the part we <em>can</em> change, and keep asking ourselves one very important question: <strong>how can our software help people become the best version of themselves?</strong></p>
</div>
<b><a href="//blog.codinghorror.com/to-serve-man-with-software/#discourse-comments">Discussion</a></b>
</section>
</article>
<article class="post">
<header class="post-header">
<span class="post-meta"><time datetime="2017-11-05">5 Nov 2017</time> </span>
<h2 class="post-title"><a href="/the-existential-terror-of-battle-royale/">The Existential Terror of Battle Royale</a></h2>
</header>
<section class="post-content">
<div class="kg-card-markdown"><p>It's been a while since I wrote a blog post, I guess in general, but also a blog post about video games. Video games are probably <a href="https://www.google.com/search?q=site%3Ablog.codinghorror.com+%22video+games%22">the single thing most attributable to my career as a programmer</a>, and everything else I've done professionally after that. I still feel video games are one of the best ways to learn and teach programming, <a href="https://blog.codinghorror.com/heres-the-programming-game-you-never-asked-for/">if properly scoped</a>, and furthermore I take <a href="https://blog.codinghorror.com/level-one-the-intro-stage/">many cues from video games in building software</a>.</p>
<p>I would characterize my state of mind for the last six to eight months as … <em>poor</em>. Not only because of <a href="https://blog.codinghorror.com/im-loyal-to-nothing-except-the-dream/">current events in the United States</a>, though the neverending barrage of bad news weighs heavily on me, and I continue to be profoundly disturbed by the erosion of core values that I thought most of us stood for as Americans. Didn't we used to look out for each other, care about each other, and fight to protect those that can't protect themselves?</p>
<p>In times like these, I sometimes turn to video games for escapist entertainment. One game in particular caught my attention because of its meteoric rise in player count over the last year.</p>
<p><a href="http://steamcharts.com/app/578080"><img src="/content/images/2017/11/pubg-steam-stats-nov-2017.png" alt="pubg-steam-stats-nov-2017"></a></p>
<p>That game is <a href="http://store.steampowered.com/app/578080/PLAYERUNKNOWNS_BATTLEGROUNDS/">Player Unknown's Battlegrounds</a>. I was increasingly curious why it was so popular, and kept getting more popular every month. Calling it a mere phenomenon seems like underselling it; something truly unprecedented is happening here. I finally broke down and bought a copy for $30 in September.</p>
<p><img src="/content/images/2017/11/player-unknown-battleground.jpg" alt="player-unknown-battleground"></p>
<p>After a few hours in, I had major flashbacks to <a href="https://blog.codinghorror.com/the-gamification/">the first time I played Counter-Strike in 1998</a>. I realized that <strong>we are witnessing the birth of an entirely new genre of game: the Battle Royale</strong>. I absolutely believe that huge numbers of people will still be playing some form of this game 20 years from now, too.</p>
<p><a href="http://store.steampowered.com/stats/"><img src="/content/images/2017/11/steam-top-games-by-player-count-nov-2017.png" alt="steam-top-games-by-player-count-nov-2017"></a></p>
<p>I've seen <a href="https://en.wikipedia.org/wiki/Battle_Royale_(film)">the Japanese movie</a>, and it's true that there were <a href="https://en.wikipedia.org/wiki/Battle_royale_game">a few Battle Royale games</a> before PUBG, but this is clearly the defining moment and game for the genre, the one that sets a precedent for everyone else to follow.</p>
<p>It's hard to explain why Battlegrounds is so compelling, but let's start with the loneliness.</p>
<p>Although you can play in squads (and I recommend it), the purest original form of the game is 100 players, last man standing. You begin with nothing but the clothes on your back, in a cargo aircraft, flying over an unknown island in a random trajectory.</p>
<p><img src="/content/images/2017/11/battlegrounds-cargo-plane.jpg" alt="battlegrounds-cargo-plane"></p>
<p>It's up to you to decide when to drop, and where to land on this huge island, full of incredibly detailed cities, buildings and houses – but strangely devoid of all life.</p>
<p><img src="/content/images/2017/11/playerunknown-battleground-drop.jpg" alt="playerunknown-battleground-drop"></p>
<p>What happened to everyone? Where did they go? The sense of apocalypse is overwhelming. It's you versus the world, but where did the rest of the world go? You'll wander this vast deserted island, scavenging for weapons and armor in near complete silence. You'll hear nothing but the wind blowing and the occasional buzzing of flies. But then, suddenly the jarring pak-pak-pak of gunfire off in the distance, reminding you that other people are here. And they aren't your friends.</p>
<p><img src="/content/images/2017/11/battle-royale-vista.jpg" alt="battle-royale-vista"></p>
<p>the dread of never knowing when another of the 100 players on this enormous island is going to suddenly appear around a corner or over a hill is <em>intense</em>. You'll find yourself wearing headphones, cranking the volume, constantly on edge listening for the implied threat of footfalls. Wait, did I hear someone just now, or was that me? You clench, and wait. I've had so many visceral panic moments playing this game, to the point that I had to stop playing just to calm down.</p>
<p><img src="/content/images/2017/11/pubg-combat.jpg" alt="pubg-combat"></p>
<p>PUBG is, in its way, the scariest zombie movie I've ever seen, though it lacks a single zombie. It dispenses with the pretense of a story, so you can realize much sooner that the zombies, as terrible as they may be, are nowhere as dangerous to you as your fellow man.</p>
<p>Meanwile, that huge cargo airplane still roars overhead every so often, impassive, indifferent, occasionally dropping supply crates with high powered items to fight over. Airstrikes randomly target areas circled in red on the map, masking footfalls, and forcing movement while raining arbitrary death and terror.</p>
<p><img src="/content/images/2017/11/pubg-map.jpg" alt="pubg-map"></p>
<p>Although the island is huge and you can land anywhere, after a few minutes a random circle is overlaid on the map, and a slowly moving wall of deadly energy starts closing in on that circle. Stay outside that circle at your peril; if you find yourself far on the opposite side of the map from a circle, you better start hunting for a vehicle or boat (they're present, but rare) quickly. These terrordome areas are always shrinking, always impending, in an ever narrowing cone, forcing the remaining survivors closer and closer together. The circles get tighter and deadlier and quicker as the game progresses, ratcheting up the tension and conflict.</p>
<p>Eventually the circle becomes so small that it's impossible for the handful of remaining survivors to avoid contact, and one person, <em>one</em> out of the hundred that originally dropped out of the cargo plane, emerges as the winner. I've never won solo, but I have won squad, and even finishing first out of 25 squads is an unreal, euphoric experience. The odds are so incredibly against you from the outset, plus you quickly discover that 85% of the game is straight up chance: someone happens to roll up behind you, a sniper gets the drop on you, or you get caught in the open with few options. Wrong place, wrong time, game over. Sucks to be you.</p>
<p><img src="/content/images/2017/11/pubg-vehicle-shooting.jpg" alt="pubg-vehicle-shooting"></p>
<p>You definitely learn to be careful, but there's only so careful you can be. Death comes quickly, without warning, and often at random. What else can you expect from a game mode where there are 100 players but only 1 eventual winner?</p>
<p>There haven't been many Battle Royale games, so this game mode is a relatively new phenomenon. If you'd like to give it a try for free, <strong>I highly recommend <a href="https://www.epicgames.com/fortnite/">Fortnite's Battle Royale mode</a> which is 100% free, a near-clone of PUBG, and quite good in its own right.</strong> They added their Battle Royale mode well after the fact; the core single player &quot;save the world&quot; gameplay of building stuff and fighting zombie hordes is quite fun too, though a bit shallow. It also has what is, in my opinion, some of the most outstanding visual style I've ever seen in a game – a cool, hyperbolic cartoon mix of Chuck Jones, Sam &amp; Max, and Cloudy with a Chance of Meatballs. It's also delightfully diverse in its character models.</p>
<p><a href="https://www.epicgames.com/fortnite/"><img src="/content/images/2017/11/fortnite-battle-royale.jpg" alt="fortnite-battle-royale"></a></p>
<p>(The only things you'll give up over PUBG are the realistic art style, vehicles, and going prone. But the superb structure building system in Fortnite <em>almost</em> makes up for that. If nothing else it is a demonstration of how incredibly compelling the Battle Royale game mode is, because that part of the game is <a href="https://www.pcgamesn.com/fortnite/fortnite-battle-royale-player-numbers">wildly successful</a> in a a way that the core game, uh, wasn't. Also it's free!)</p>
<p>I didn't intend for this to happen, but to me, <strong>the Battle Royale game mode perfectly captures the zeitgeist of the current moment</strong>, and matches my current state of mind to a disturbing degree. It's an absolutely terrifying experience of every human for themselves, winner takes all, with impossible odds. There are moments it can be thrilling, even inspiring, but mostly it's brutal and unforgiving. To succeed you need to be exceedingly cautious, highly skilled, and just plain <em>lucky</em>. Roll the dice again, but know that everyone will run towards the sound of gunfire in hopes of picking off survivors and looting their corpses. Including you.</p>
<p>Battle Royale is not the game mode we wanted, it's not the game mode we needed, it's the game mode we all <em>deserve</em>. And the best part is, when we're done playing, we can turn it off.</p>
</div>
<b><a href="//blog.codinghorror.com/the-existential-terror-of-battle-royale/#discourse-comments">Discussion</a></b>
</section>
</article>
<article class="post">
<header class="post-header">
<span class="post-meta"><time datetime="2017-06-02">2 Jun 2017</time> </span>
<h2 class="post-title"><a href="/hacker-hack-thyself/">Hacker, Hack Thyself</a></h2>
</header>
<section class="post-content">
<div class="kg-card-markdown"><p>We've read so many sad stories about communities that were fatally compromised or destroyed due to security exploits. We took that lesson to heart when we founded the <a href="https://discourse.org">Discourse</a> project; we endeavor to build open source software that is secure and safe for communities by default, even if there are thousands, or millions, of them out there.</p>
<p>However, we also value <em>portability</em>, the ability to get your data into and out of Discourse at will. This is why Discourse, unlike other forum software, defaults to a Creative Commons license. As a basic user on any Discourse you can easily export and download all your posts right from your user page.</p>
<p><img src="/content/images/2017/06/discourse-download-all.png" alt="Discourse Download All Posts"></p>
<p>As a site owner, you can easily back up and restore your entire site database from the admin panel, right in your web browser. Automated weekly backups are set up for you out of the box, too. I'm not <a href="https://blog.codinghorror.com/international-backup-awareness-day/">the world's foremost expert on backups</a> for nothing, man!</p>
<p><img src="/content/images/2017/06/discourse-backups.png" alt="Discourse database backup download"></p>
<p>Over the years, we've learned that balancing security and data portability can be tricky. You bet your sweet ASCII a <strong>full database download</strong> is what hackers start working toward the minute they gain any kind of foothold in your system. It's the ultimate prize.</p>
<p>To mitigate this threat, we've slowly tightened restrictions around Discourse backups in various ways:</p>
<ul>
<li>
<p>Administrators have a minimum password length of 15 characters.</p>
</li>
<li>
<p>Both backup creation and backup download administrator actions are formally logged.</p>
</li>
<li>
<p>Backup download tokens are single use and emailed to the address of the administrator, to confirm that user has full control over the email address.</p>
</li>
</ul>
<p>The name of the security game is defense in depth, so all these hardening steps help … but we still need to <strong>assume that Internet Bad Guys will somehow get a copy of your database</strong>. And then what? Well, what's in the database?</p>
<ul>
<li>
<p>Identity cookies</p>
<p>Cookies are, of course, how the browser can tell who you are. Cookies are usually stored as hashes, rather than the actual cookie value, so having the hash doesn't let you impersonate the target user. Furthermore, most modern web frameworks rapidly cycle cookies, so they are only valid for a brief 10 to 15 minute window anyway.</p>
</li>
<li>
<p>Email addresses</p>
<p>Although users have reason to be concerned about their emails being exposed, very few people treat their email address as anything particularly precious these days.</p>
</li>
<li>
<p>All posts and topic content</p>
<p>Let's assume for the sake of argument that this is a fully public site and nobody was posting anything particularly sensitive there. So we're not worried, at least for now, about trade secrets or other privileged information being revealed, since they were all public posts anyway. If we were, that's a whole other blog post I can write at a later date.</p>
</li>
<li>
<p>Password hashes</p>
<p>What's left is <strong>the password hashes</strong>. And that's … <a href="https://blog.codinghorror.com/speed-hashing/">a serious problem indeed</a>.</p>
</li>
</ul>
<p>Now that the attacker has your database, they can crack your password hashes with <a href="https://blog.codinghorror.com/your-password-is-too-damn-short/">large scale offline attacks</a>, using the full resources of any cloud they can afford. And once they've cracked a particular password hash, <strong>they can log in as that user … forever</strong>. Or at least until that user changes their password.</p>
<blockquote>
<p>⚠️ That's why, if you know (or even suspect!) your database was exposed, the very first thing you should do is reset everyone's password.</p>
</blockquote>
<p><img src="/content/images/2017/06/discourse-db-password-hashes.png" alt="Discourse database password hashes"></p>
<p>But what if you <em>don't</em> know? Should you preemptively reset everyone's password every 30 days, like the world's worst bigco IT departments? That's downright user hostile, and leads to serious pathologies of its own. The reality is that you probably <em>won't</em> know when your database has been exposed, at least not until it's too late to do anything about it. So it's crucial to slow the attackers down, to give yourself time to deal with it and respond.</p>
<p>Thus, the only real protection you can offer your users is just how resistant to attack your stored password hashes are. There are two factors that go into password hash strength:</p>
<ol>
<li>
<p><strong>The hashing algorithm</strong>. As slow as possible, and ideally designed to be <em>especially</em> slow on GPUs for reasons that will become painfully obvious about 5 paragraphs from now.</p>
</li>
<li>
<p><strong>The work factor</strong> or <strong>number of iterations</strong>. Set this as high as possible, without opening yourself up to a possible denial of service attack.</p>
</li>
</ol>
<p>I've seen guidance that said you should set the overall work factor high enough that hashing a password takes at least 8ms on the target platform. It turns out <a href="https://samsaffron.com/">Sam Saffron</a>, one of my Discourse co-founders, made a good call back in 2013 when he selected the NIST recommendation of <strong>PBKDF2-HMAC-SHA256</strong> and <strong>64k iterations</strong>. We measured, and that indeed takes roughly 8ms using our existing Ruby login code on our current (fairly high end, Skylake 4.0 Ghz) servers.</p>
<p>But that was 4 years ago. Exactly how secure are our password hashes in the database today? Or 4 years from now, or 10 years from now? We're building open source software for the long haul, and we need to be sure we are making reasonable decisions that protect everyone. So in the spirit of <a href="https://blog.codinghorror.com/designing-for-evil/">designing for evil</a>, it's time to put on our Darth Helmet and play the bad guy – <strong>let's crack our own hashes!</strong></p>
<p><img src="/content/images/2017/06/dark-helmet.jpg" alt=""></p>
<p>We're gonna use the biggest, baddest single GPU out there at the moment, <a href="https://gist.github.com/epixoip/a83d38f412b4737e99bbef804a270c40#gistcomment-2060753">the GTX 1080 Ti</a>. As a point of reference, for PBKDF2-HMAC-SHA256 the 1080 achieves 1180 kH/s, whereas the 1080 Ti achieves 1640 kH/s. In a <em>single</em> video card generation the attack hash rate has increased nearly 40 percent. Ponder that.</p>
<p>First, a tiny hello world test to see if things are working. I downloaded <a href="https://hashcat.net/hashcat/">hashcat</a>. I logged into our demo at <a href="http://try.discourse.org">try.discourse.org</a> and created a new account with the password <code>0234567890</code>; I checked the database, and this generated the following values in the hash and salt database columns for that new user:</p>
<blockquote>
<p>hash<br>
<code>93LlpbKZKficWfV9jjQNOSp39MT0pDPtYx7/gBLl5jw=</code><br>
salt<br>
<code>ZWVhZWQ4YjZmODU4Mzc0M2E2ZDRlNjBkNjY3YzE2ODA=</code></p>
</blockquote>
<p>Hashcat requires the following input file format: one line per hash, with the hash type, number of iterations, salt and hash (base64 encoded) separated by colons:</p>
<pre><code>type iter salt hash
sha256:64000:ZWVhZWQ4YjZmODU4Mzc0M2E2ZDRlNjBkNjY3YzE2ODA=:93LlpbKZKficWfV9jjQNOSp39MT0pDPtYx7/gBLl5jw=
</code></pre>
<p>Let's hashcat it up and see if it works:</p>
<p><code>./h64 -a 3 -m 10900 .\one-hash.txt 0234567?d?d?d</code></p>
<p>Note that this is an intentionally tiny amount of work, it's only guessing three digits. And sure enough, we cracked it fast! See the password there on the end? We got it.</p>
<p><code>sha256:64000:ZWVhZWQ4YjZmODU4Mzc0M2E2ZDRlNjBkNjY3YzE2ODA=:93LlpbKZKficWfV9jjQNOSp39MT0pDPtYx7/gBLl5jw=:0234567890</code></p>
<p>Now that we know it works, let's get down to business. But we'll start easy. How long does it take to brute force attack <strong>the easiest possible Discourse password, 8 numbers</strong> – that's &quot;only&quot; 10<sup>8</sup> combinations, a little over one hundred million.</p>
<pre><code>Hash.Type........: PBKDF2-HMAC-SHA256
Time.Estimated...: Fri Jun 02 00:15:37 2017 (1 hour, 0 mins)
Guess.Mask.......: ?d?d?d?d?d?d?d?d [8]
</code></pre>
<p>Even with a top of the line GPU that's … OK, I guess. Remember this is just one hash we're testing against, so you'd need one hour per row (user) in the table. And I have more bad news for you: Discourse hasn't allowed 8 character passwords for <a href="https://blog.codinghorror.com/your-password-is-too-damn-short/">quite some time now</a>. How long does it take if we try longer numeric passwords?</p>
<pre><code>?d?d?d?d?d?d?d?d?d [9]
Fri Jun 02 10:34:42 2017 (11 hours, 18 mins)
?d?d?d?d?d?d?d?d?d?d [10]
Tue Jun 06 17:25:19 2017 (4 days, 18 hours)
?d?d?d?d?d?d?d?d?d?d?d [11]
Mon Jul 17 23:26:06 2017 (46 days, 0 hours)
?d?d?d?d?d?d?d?d?d?d?d?d [12]
Tue Jul 31 23:58:30 2018 (1 year, 60 days)
</code></pre>
<p>But all digit passwords are easy mode, for babies! How about some <em>real</em> passwords that use at least lowercase letters, or lowercase + uppercase + digits?</p>
<pre><code>Guess.Mask.......: ?l?l?l?l?l?l?l?l [8]
Time.Estimated...: Mon Sep 04 10:06:00 2017 (94 days, 10 hours)
Guess.Mask.......: ?1?1?1?1?1?1?1?1 [8] (-1 = ?l?u?d)
Time.Estimated...: Sun Aug 02 09:29:48 2020 (3 years, 61 days)
</code></pre>
<p>A brute force try-every-single-letter-and-number attack is not looking so hot for us at this point, even with a high end GPU. But what if we divided the number by <strong>eight</strong> … <a href="https://gist.github.com/epixoip/a83d38f412b4737e99bbef804a270c40">by putting eight video cards in a single machine?</a> That's well within the reach of a small business budget or a wealthy individual. Unfortunately, dividing 38 months by 8 isn't such a dramatic reduction in the time to attack. Instead, let's talk about nation state attacks where they have the budget to throw <em>thousands</em> of these GPUs at the problem (1.1 days), maybe even <em>tens of thousands</em> (2.7 hours), then … yes. Even allowing for 10 character password minimums, you are in serious trouble at that point.</p>
<p><img src="/content/images/2017/06/8-gpu-cracking-rig.jpg" alt=""></p>
<p>If we want Discourse to be nation state attack resistant, clearly we'll need to do better. Hashcat has a handy benchmark mode, and <a href="https://docs.google.com/spreadsheets/d/1iwoMR5TBYAZ5eiSphkIQfIEfbrVvWW_tKwS4L1cYlaI/pubhtml?gid=0&amp;single=true">here's a sorted list of the strongest (slowest) hashes that Hashcat knows about</a> benchmarked on a rig with 8 Nvidia GTX 1080 GPUs. Of the things I recognize on that list, <strong>bcrypt</strong>, <strong>scrypt</strong> and <strong>PBKDF2-HMAC-SHA512</strong> stand out.</p>
<p>My quick hashcat results gave me some confidence that we weren't doing anything terribly wrong with the Discourse password hashes stored in the database. But I wanted to be <em>completely sure</em>, so I hired someone with a background in security and penetration testing to, under a signed NDA, try cracking the password hashes of two live and very popular Discourse sites <a href="https://discourse.org/customers">we currently host</a>.</p>
<blockquote>
<p>I was provided two sets of password hashes from two different Discourse communities, containing 5,909 and 6,088 hashes respectively. Both used the PBKDF2-HMAC-SHA256 algorithm with a work factor of 64k. Using hashcat, my Nvidia GTX 1080 Ti GPU generated these hashes at a rate of ~27,000/sec.</p>
<p>Common to all discourse communities are various password requirements:</p>
<ul>
<li>All users must have a minimum password length of 10 characters.</li>
<li>All administrators must have a minimum password length of 15 characters.</li>
<li>Users cannot use any password matching a blacklist of the 10,000 most commonly used passwords.</li>
<li>Users can choose to create a username and password or use various third party authentication mechanisms (Google, Facebook, Twitter, etc). If this option is selected, a secure random 32 character password is autogenerated. It is not possible to know whether any given password is human entered, or autogenerated.</li>
</ul>
<p>Using common password lists and masks, I cracked 39 of the 11,997 hashes in about three weeks, 25 from the ████████ community and 14 from the ████████ community.</p>
</blockquote>
<p>This is a security researcher who commonly runs these kinds of audits, so all of the attacks used <strong>wordlists</strong>, along with known effective patterns and <a href="https://hashcat.net/wiki/doku.php?id=mask_attack">masks</a> derived from the researcher's previous password cracking experience, instead of raw brute force. That recovered the following passwords (and one duplicate):</p>
<table>
<tr>
<td>
<code>007007bond</code><br>
<code>123password</code><br>
<code>1qaz2wsx3e</code><br>
<code>A3eilm2s2y</code><br>
<code>Alexander12</code><br>
<code>alexander18</code><br>
<code>belladonna2</code><br>
<code>Charlie123</code><br>
<code>Chocolate1</code><br>
<code>christopher8</code><br>
<code>Elizabeth1</code><br>
<code>Enterprise01</code><br>
<code>Freedom123</code><br>
<code>greengrass123</code><br>
<code>hellothere01</code><br>
<code>I123456789</code><br>
<code>Iamawesome</code><br>
<code>khristopher</code><br>
<code>l1ghthouse</code><br>
</td>
<td>
<code>l3tm3innow</code><br>
<code>Neversaynever</code><br>
<code>password1235</code><br>
<code>pittsburgh1</code><br>
<code>Playstation2</code><br>
<code>Playstation3</code><br>
<code>Qwerty1234</code><br>
<code>Qwertyuiop1</code><br>
<code>qwertyuiop1234567890</code><br>
<code>Spartan117</code><br>
<code>springfield0</code><br>
<code>Starcraft2</code><br>
<code>strawberry1</code><br>
<code>Summertime</code><br>
<code>Testing123</code><br>
<code>testing1234</code><br>
<code>thecakeisalie02</code><br>
<code>Thirteen13</code><br>
<code>Welcome123</code><br>
</td>
</tr>
</table>
<p>If we multiply this effort by 8, and double the amount of time allowed, it's conceivable that a <em>very</em> motivated attacker, or <a href="https://arstechnica.com/security/2013/10/how-the-bible-and-youtube-are-fueling-the-next-frontier-of-password-cracking/">one with a sophisticated set of wordlists and masks</a>, could eventually recover 39 × 16 = 624 passwords, or about <strong>five percent</strong> of the total users. That's reasonable, but higher than I would like. We absolutely plan to add a hash type table in future versions of Discourse, so we can switch to an even more secure (read: <a href="http://www.pxdojo.net/2015/08/what-i-learned-from-cracking-4000.html">much slower</a>) password hashing scheme in the next year or two.</p>
<pre><code>bcrypt $2*$, Blowfish (Unix)
20273 H/s
scrypt
886.5 kH/s
PBKDF2-HMAC-SHA512
542.6 kH/s
PBKDF2-HMAC-SHA256
1646.7 kH/s
</code></pre>
<p>After this exercise, I now have a much deeper understanding of our worst case security scenario, a database compromise combined with a professional offline password hashing attack. I can also more confidently recommend and stand behind our engineering work in making Discourse secure for everyone. So if, like me, you're not entirely sure you are doing things securely, it's time to put those assumptions to the test. Don't wait around for hackers to attack you — <strong>hacker, hack thyself!</strong></p>
<table>
<tr><td class="welovecodinghorror">[advertisement] At Stack Overflow, we put developers first. We already help you find answers to your tough coding questions; now let us help you <a href="http://careers.stackoverflow.com" rel="nofollow">find your next job</a>.</td></tr>
</table> </div>
<b><a href="//blog.codinghorror.com/hacker-hack-thyself/#discourse-comments">Discussion</a></b>
</section>
</article>
<article class="post">
<header class="post-header">
<span class="post-meta"><time datetime="2017-03-24">24 Mar 2017</time> </span>
<h2 class="post-title"><a href="/thunderbolting-your-video-card/">Thunderbolting Your Video Card</a></h2>
</header>
<section class="post-content">
<div class="kg-card-markdown"><p>When I wrote about <a href="https://blog.codinghorror.com/the-golden-age-of-x86-gaming/">The Golden Age of x86 Gaming</a>, I <em>implied</em> that, in the future, it might be an interesting, albeit expensive, idea to upgrade your video card via an external Thunderbolt 3 enclosure.</p>
<p><img src="/content/images/2017/03/skull-canyon-nuc-with-razer-core.jpg" alt=""></p>
<p>I'm here to report that <strong>the future is now</strong>.</p>
<p>Yes, that's right, I paid $500 for <a href="https://www.razerzone.com/store/razer-core">an external Thunderbolt 3 enclosure</a> to fit a $600 video card, all to enable a plug-in upgrade of a GPU on a <a href="https://blog.codinghorror.com/the-golden-age-of-x86-gaming/">Skull Canyon NUC</a> that itself cost around $1000 fully built. I know, it sounds crazy, and … OK fine, I won't argue with you. It's crazy.</p>
<p>This matters mostly because of 4k, aka 2160p, aka 3840 × 2160, aka <strong>Ultra HD</strong>.</p>
<p><img src="https://blog.codinghorror.com/content/images/2015/08/common-hd-resolutions-compared.png" alt="4k compared to 1080p"></p>
<p>Plain old regular HD, aka 1080p, aka 1920 × 1080, is one quarter the size of 4k, and ¼ the work. By today's GPU standards HD is pretty much <em>easy mode</em> these days. It's not even interesting. No offense to console fans, or anything.</p>
<p>Late in 2016, I got a <a href="https://www.amazon.com/gp/product/B01CDD4J58/?tag=codihorr-20">4k OLED display</a> and it … kind of blew my mind. I have never seen blacks so black, colors so vivid, on a display so thin. It made my previous 2008 era Panasonic plasma set look lame. It's so good that I'm now a little angry that every display that my eyes touch isn't OLED already. I even got into nerd fights over it, and to be honest, I'd still throw down for OLED. It is legitimately <em>that good</em>. Come at me, bro.</p>
<p>Don't believe me? Well, guess which display in the below picture is OLED? Go on, guess:</p>
<p><a href="http://www.consumerreports.org/lcd-led-oled-tvs/2016-LG-4K-oled-tvs/"><img src="/content/images/2017/03/CptX7RCVYAAKNOP.jpg" alt="Guess which screen is OLED?"></a></p>
<blockquote class="twitter-tweet" data-lang="en"><p lang="en" dir="ltr"><a href="https://twitter.com/andrewbstiles">@andrewbstiles</a> if it was physically possible to have sex with this TV I.. uh.. I&#39;d take it on long, romantic walks</p>&mdash; Jeff Atwood (@codinghorror) <a href="https://twitter.com/codinghorror/status/764304493483663361">August 13, 2016</a></blockquote>
<script async src="//platform.twitter.com/widgets.js" charset="utf-8"></script>
<p>There's a reason every site that reviews TVs had to recalibrate their results when <a href="http://thewirecutter.com/reviews/best-tv/">they reviewed the 2016 OLED sets</a>.</p>
<blockquote>
<p>In my extended review at Reference Home Theater, I call it “the best looking TV I’ve ever reviewed.” But we aren’t alone in loving the E6. Vincent Teoh at HDTVtest writes, “We’re not even going to qualify the following endorsement: if you can afford it, this is the TV to buy.” <a href="http://Rtings.com">Rtings.com</a> gave <a href="https://www.amazon.com/gp/product/B01CDD4J58/?tag=codihorr-20">the E6 OLED</a> the highest score of any TV the site has ever tested. <a href="http://Reviewed.com">Reviewed.com</a> awarded it a 9.9 out of 10, with only the LG G6 OLED (which offers the same image but better styling and sound for $2,000 more) coming out ahead.</p>
</blockquote>
<p>But I digress.</p>
<p>Playing games at 1080p in my living room was already possible. But now that I have an incredible 4k display in the living room, it's a whole other level of difficulty. Not just twice as hard – and remember current consoles <em>barely</em> manage to eke out 1080p at 30fps in most games – but <strong>four times as hard</strong>. That's where external GPU power comes in.</p>
<p><img src="/content/images/2017/03/razer-core-with-gpu.jpg" alt=""></p>
<p>The cool technology underpinning all of this is <strong>Thunderbolt 3</strong>. The thunderbolt cable bundled with the Razer Core is rather … diminutive. There's <a href="https://blog.startech.com/post/thunderbolt-3-the-basics/">a reason for this</a>.</p>
<blockquote>
<p><strong>Is there a maximum cable length for Thunderbolt 3 technology?</strong></p>
<p>Thunderbolt 3 passive cables have maximum lengths.</p>
<ul>
<li>0.5m TB 3 (40Gbps)</li>
<li>1.0m TB 3 (20Gbps)</li>
<li>2.0m TB 3 (20Gbps)</li>
</ul>
<p>In the future we will offer active cables which will provide 40Gbps of bandwidth at longer lengths.</p>
</blockquote>
<p>40Gbps is, for the record, an <em>insane</em> amount of bandwidth. Let's use our rule of thumb based on ultra common gigabit ethernet, that 1 gigabit = 120 megabytes/second, and we arrive at <strong>4.8 gigabytes/second</strong>. Zow.</p>
<p>That's more than enough bandwidth to run even the highest of high end video cards, but it is not without overhead. There's <a href="http://www.ultrabookreview.com/10761-razer-core-review/">a mild performance hit</a> for running the card externally, on the order of <strong>15%</strong>. There's also a further performance hit of 10% if you are in &quot;loopback&quot; mode on a laptop where you don't <em>have</em> an external display, so the video frames have to be shuttled back from the GPU to the internal laptop display.</p>
<p>This may look like a gamer-only thing, but surprisingly, it isn't. What you get is the general purpose ability to attach <strong>any PCI express card</strong> to any computer with a <strong>Thunderbolt 3</strong> port and, for the most part, it just works!</p>
<p>Linus breaks it down and answers all your most difficult questions:</p>
<iframe width="560" height="315" src="https://www.youtube.com/embed/2D79GsrEqe4" frameborder="0" allowfullscreen></iframe>
<p>Please watch the above video closely if you're actually interested in this stuff; it is essential. I'll add some caveats of my own after working with the Razer Core for a while:</p>
<ul>
<li>
<p>Make sure the video card you plan to put into the Razer Core is not too tall, or too wide. You can tell if a card is going to be too tall by looking at pictures of the mounting rear bracket. If the card extends significantly above the standard rear mounting bracket, it won't fit. If the card takes more than 2 slots in width, it also won't fit, but this is more rare. Depth (length) is rarely an issue.</p>
</li>
<li>
<p>There are four fans in the Razer Core and although it is <em>reasonably</em> quiet, it's not super silent or anything. You may want to <a href="http://forum.notebookreview.com/threads/razer-core-disassembly-fan-location-guide.802000/">mod the fans</a>. The Razer Core is a remarkably simple device, internally, it's really just a power supply, some Thunderbolt 3 bridge logic, and a PCI express slot. I agree with Linus that the #1 area Razer could improve in the future, beyond generally getting the price down, is to use fewer and larger fans that run quieter.</p>
</li>
<li>
<p>If you're putting a heavy hitter GPU in the Razer Core, I'd try to avoid blower style cards (the ones that exhaust heat from the rear) in favor of those that cool with large fans blowing down and around the card. Dissipating 150w+ is no mean feat and you'll definitely need to keep the enclosure in open air … and of course within 0.5 meters of the computer it's connected to.</p>
</li>
<li>
<p>There is no visible external power switch on the Razer Core. It doesn't power on until you connect a TB3 cable to it. I was totally not expecting that. But once connected, it powers up and the Windows 10 Thunderbolt 3 drivers kick in and ask you to authorize the device, which I did (always authorize). Then it spun a bit, detected the new GPU, and suddenly I had multiple graphics card active on the same computer. I also installed the latest Nvidia drivers just to make sure everything was ship shape.</p>
</li>
<li>
<p>It's kinda ... <em>weird</em> having multiple GPUs simultaneously active. I wanted to make the Razer Core display the only display, but you can't really turn off the built in GPU – you can select &quot;only use display 2&quot;, that's all. I got into several weird states where windows were opening on the other display and I had to mess around a fair bit to get things locked down to just one display. You may want to consider whether you have both &quot;displays&quot; connected for troubleshooting, or not.</p>
</li>
</ul>
<p>And then, there I am, playing Lego Marvel in splitscreen co-op at glorious 3840 × 2160 UltraHD resolution on an amazing OLED display with my son. It is <em>incredible</em>.</p>
<p><img src="/content/images/2017/03/lego-marvel-4k.jpg" alt=""></p>
<p>Beyond the technical &quot;because I could&quot;, I am <strong>wildly optimistic about the future of external Thunderbolt 3 expansion boxes</strong>, and here's why:</p>
<ul>
<li>
<p>The main expense and bottleneck in any stonking gaming rig is, by <em>far</em>, the GPU. It's also the item you are most likely to need to replace a year or two from now.</p>
</li>
<li>
<p>The CPU and memory speeds available today are so comically fast that any device with a low-end i3-7100 for $120 will make zero difference in real world gaming at 1080p or higher … if you're OK with 30fps minimum. If you bump up to $200, you can get a quad-core i5-7500 that guarantees you 60fps minimum everywhere.</p>
</li>
<li>
<p>If you prefer a small system or a laptop, an external GPU makes it so much more flexible. Because CPU and memory speeds are already so fast, 99.9% of the time your bottleneck is the GPU, and almost <strong>any small device you can buy with a Thunderbolt 3 port can now magically transform into a potent gaming rig with a single plug</strong>. Thunderbolt 3 may be a bit cutting edge today, but more and more devices are shipping with Thunderbolt 3. Within a few years, I predict TB3 ports will be as common as USB3 ports.</p>
</li>
<li>
<p>A general purpose external PCI express enclosure will be usable for a very long time. My last <em>seven</em> video card upgrades were plug and play PCI Express cards that would have worked fine in any computer I've built in the last ten years.</p>
</li>
<li>
<p>External GPUs are not meaningfully bottlenecked by Thunderbolt 3 bandwidth; the impact is 15% to 25%, and perhaps even less over time as drivers and implementations mature. While Thunderbolt 3 has &quot;only&quot; PCI Express x4 bandwidth, many benchmarkers have noted that GPUs moving from PCI Express x16 to x8 has <a href="https://www.pugetsystems.com/labs/articles/Impact-of-PCI-E-Speed-on-Gaming-Performance-518/">almost no effect on performance</a>. And there's always Thunderbolt 4 on the horizon.</p>
</li>
</ul>
<p>The future, as they say, is already here – it's just not evenly distributed.</p>
<p>I am painfully aware that <strong>costs need to come down</strong>. Way, <em>way</em> down. The <a href="https://www.razerzone.com/store/razer-core">$499 Razer Core</a> is well made, on the vanguard of what's possible, a harbinger of the future, and fantastically enough, it does <em>even more</em> than what it says on the tin. But it's not exactly <em>affordable</em>.</p>
<p>I would absolutely love to see a modest, dedicated $200 external Thunderbolt 3 box that included an inexpensive current-gen GPU. This would <em>clobber</em> any onboard GPU on the planet. Let's compare my Skull Canyon NUC, which has Intel's <a href="http://www.notebookcheck.net/Intel-Iris-Pro-Graphics-580.160664.0.html">fastest ever, PS4 class embedded GPU</a>, with the modest $150 <a href="http://www.notebookcheck.com/NVIDIA-GeForce-GTX-1050-Ti-Desktop.181030.0.html">GeForce GTX 1050 Ti</a>:</p>
<table width="300px">
<tr>
<td colspan=2><b>1920 &times; 1080 high detail</b></td>
</tr>
<tr>
<td>Bioshock Infinite</td><td>15 → 79 fps</td>
</tr>
<tr>
<td>Rise of the Tomb Raider</td><td>12 → 49 fps</td>
</tr>
<tr>
<td>Overwatch</td><td>43 → 114 fps</td>
</tr>
</table>
<p>As predicted, that's a 3x-5x stompdown. Mac users lamenting their general lack of upgradeability, hear me: <em>this sort of box is exactly what you want and need</em>. Imagine if Apple was to embrace upgrading their laptops and all-in-one systems via Thunderbolt 3.</p>
<p><img src="/content/images/2017/03/razer-core-and-razer-laptop.jpg" alt=""></p>
<p>I know, I know. It's a stretch. But a man can dream … of externally upgradeable GPUs. That are too expensive, sure, but they are here, right now, today. They'll only get cheaper over time.</p>
<table>
<tr><td class="welovecodinghorror">
[advertisement] <a href="http://careers.stackoverflow.com" rel="nofollow">Find a better job the Stack Overflow way</a> - what you need when you need it, no spam, and no scams.
</td></tr>
</table> </div>
<b><a href="//blog.codinghorror.com/thunderbolting-your-video-card/#discourse-comments">Discussion</a></b>
</section>
</article>
<article class="post">
<header class="post-header">
<span class="post-meta"><time datetime="2017-03-10">10 Mar 2017</time> </span>
<h2 class="post-title"><a href="/password-rules-are-bullshit/">Password Rules Are Bullshit</a></h2>
</header>
<section class="post-content">
<div class="kg-card-markdown"><p>Of the many, many, <em>many</em> <a href="https://blog.codinghorror.com/the-dirty-truth-about-web-passwords/">bad things about passwords</a>, you know what the worst is? Password rules.</p>
<blockquote class="twitter-tweet" data-lang="en"><p lang="en" dir="ltr">If we don&#39;t solve the password problem for users in my lifetime I am gonna haunt you from beyond the grave as a ghost <a href="http://t.co/Tf9EnwgoZv">pic.twitter.com/Tf9EnwgoZv</a></p>&mdash; Jeff Atwood (@codinghorror) <a href="https://twitter.com/codinghorror/status/631238409269309440">August 11, 2015</a></blockquote>
<script async src="//platform.twitter.com/widgets.js" charset="utf-8"></script>
<p>Let this pledge be duly noted on the permanent record of the Internet. I don't know if there's an afterlife, but I'll be finding out soon enough, and I plan to go out <em>mad as hell</em>.</p>
<p>The world is absolutely awash in terrible password rules:</p>
<ul>
<li><a href="https://github.com/duffn/dumb-password-rules">Dumb Password Rules</a></li>
<li><a href="http://badpasswordpolicies.tumblr.com/">Bad Password Policies</a></li>
<li><a href="http://password-shaming.tumblr.com/">Password Requirements Shaming</a></li>
</ul>
<p>But I don't need to tell you this. The more likely you are to use a truly random password generation tool, like us über-geeks are supposed to, the more likely you have suffered mightily – and daily – under this regime.</p>
<p>Have you seen the classic XKCD <a href="https://xkcd.com/936/">about passwords</a>?</p>
<p><img src="/content/images/2017/03/password_strength.png" alt="To anyone who understands information theory and security and is in an infuriating argument with someone who does not (possibly involving mixed case), I sincerely apologize."></p>
<p>We <a href="https://security.stackexchange.com/questions/6095/xkcd-936-short-complex-password-or-long-dictionary-passphrase">can certainly debate</a> whether &quot;correct horse battery staple&quot; is a viable password strategy or not, but the argument here is mostly that <em>length matters</em>.</p>
<p><img src="/content/images/2017/03/twss.jpg" alt="That's What She Said"></p>
<p>No, seriously, it does. I'll go so far as to say <a href="https://blog.codinghorror.com/your-password-is-too-damn-short/">your password is too damn short</a>. These days, given the state of cloud computing and GPU password hash cracking, any password of 8 characters or less is perilously close to <em>no password at all</em>.</p>
<p>So then perhaps we have one rule, that <strong>passwords must not be short</strong>. A long password is much more likely to be secure than a short one … right?</p>
<p>What about this four character password?</p>
<h1 id="">✅🐎🔋🖇️</h1>
<p>What about this eight character password?</p>
<h1 id="">正确马电池订书钉</h1>
<p>Or this (hypothetical, but all too real) seven character password?</p>
<h1>ش导พิ한<img src="/content/images/2017/06/klingon-char.png" width="35" height="42" alt="" style="display:inline; vertical-align:middle">✌︎🚖</h1>
<blockquote class="twitter-tweet" data-conversation="none" data-lang="en"><p lang="en" dir="ltr"><a href="https://twitter.com/codinghorror">@codinghorror</a> I&#39;m sorry but your password must contain 1 char each from: Arabic, Chinese, Thai, Korean, Klingon, Wingdings and an emoji</p>&mdash; Finley Creative (@FinleyCreative) <a href="https://twitter.com/FinleyCreative/status/705349059217784833">March 3, 2016</a></blockquote>
<script async src="//platform.twitter.com/widgets.js" charset="utf-8"></script>
<p>You may also be surprised, if you paste the above four Unicode emojis into your favorite login dialog (go ahead – try it), to discover that it … <em>isn't</em> in fact four characters.</p>
<p><img src="/content/images/2017/03/discourse-login-emoji-password.png" alt=""></p>
<p>Oh dear.</p>
<pre><code>&quot;💩&quot;.length === 2
</code></pre>
<p>Our old pal Unicode <a href="http://blog.jonnew.com/posts/poo-dot-length-equals-two">strikes again</a>.</p>
<p>As it turns out, even the simple rule that &quot;your password must be of reasonable length&quot; … ain't necessarily so. Particularly if we stop thinking like <a href="https://blog.codinghorror.com/the-ugly-american-programmer/">Ugly ASCII Americans</a>.</p>
<p>And what of those nice, long passwords? Are they <em>always</em> secure?</p>
<pre><code>aaaaaaaaaaaaaaaaaaa
0123456789012345689
passwordpassword
usernamepassword
</code></pre>
<p>Of course not, because <em>have you met any users lately?</em></p>
<p><img src="/content/images/2017/03/incorrect-password.jpg" alt="I changed all my passwords to &quot;incorrect&quot;"></p>
<p>They consistently ruin every piece of software I've ever written. Yes, yes, I know you, Mr. or Ms. über-geek, know <em>all</em> about the concept of entropy. But expressing your love of entropy as terrible, idiosyncratic password rules …</p>
<ul>
<li>must contain uppercase</li>
<li>must contain lowercase</li>
<li>must contain a number</li>
<li>must contain a special character</li>
</ul>
<p>… is a spectacular failure of imagination in a world of Unicode and Emoji.</p>
<p>As we built <a href="https://discourse.org">Discourse</a>, I discovered that <a href="https://blog.codinghorror.com/the-god-login/">the login dialog was a remarkably complex piece of software</a>, despite its surface simplicity. The primary password rule we used was also the simplest one: <strong>length</strong>. Since I wrote that, we've already increased our minimum password default length from 8 to 10 characters. And if you happen to be an admin or moderator, we decided the minimum has to be even more, <strong>15</strong> characters.</p>
<p>I also advocated <strong>checking passwords against the 100,000 most common passwords</strong>. If you look at <a href="https://blog.keepersecurity.com/2017/01/13/most-common-passwords-of-2016-research-study/">10 million passwords from data breaches in 2016</a>, you'll find the top 25 most used passwords are:</p>
<table width="320px">
<tr>
<td style="vertical-align:top">
<code>123456</code><br>
<code>123456789</code><br>
<code>qwerty</code><br>
<code>12345678</code><br>
<code>111111</code><br>
<code>1234567890</code><br>
<code>1234567</code><br>
<code>password</code><br>
<code>123123</code><br>
<code>987654321</code><br>
<code>qwertyuiop</code><br>
<code>mynoob</code><br>
</td>
<td style="vertical-align:top">
<code>123321</code><br>
<code>666666</code><br>
<code>18atcskd2w</code><br>
<code>7777777</code><br>
<code>1q2w3e4r</code><br>
<code>654321</code><br>
<code>555555</code><br>
<code>3rjs1la7qe</code><br>
<code>google</code><br>
<code>1q2w3e4r5t</code><br>
<code>123qwe</code><br>
<code>zxcvbnm</code><br>
<code>1q2w3e</code><br>
</td>
</tr>
</table>
<p>Even this data betrays some ASCII-centrism. The numbers are the same in any culture I suppose, but I find it hard to believe the average Chinese person will ever choose the passwords &quot;password&quot;, &quot;quertyuiop&quot;, or &quot;mynoob&quot;. So this list <em>has</em> to be customizable, localizable.</p>
<p>(One interesting idea is to search for common shorter password matches inside longer passwords, but I think this would cause too many false positives.)</p>
<p>If you examine the data, this also turns into an argument in favor of password length. Note that only 5 of the top 25 passwords are 10 characters, so if we require 10 character passwords, we've already reduced our exposure to the most common passwords by 80%. I saw this originally when I <a href="https://github.com/danielmiessler/SecLists/tree/master/Passwords">gathered millions and millions of leaked passwords for Discourse research</a>, then filtered the list down to just those passwords reflecting our new minimum requirement of 10 characters or more.</p>
<p><img src="/content/images/2017/03/top-million-common-passwords-by-length.png" alt=""></p>
<p>It suddenly became a <em>tiny</em> list. (If you've done similar common password research, please do share your results in the comments.)</p>
<p>I'd like to offer the following common sense advice to my fellow developers:</p>
<h4 id="1passwordrulesarebullshit">1. Password rules are bullshit</h4>
<ul>
<li>They don't work.</li>
<li>They heavily penalize your ideal audience, people that use real random password generators. Hey guess what, that password randomly <em>didn't</em> have a number or symbol in it. I just double checked my math textbook, and yep, it's possible. I'm pretty sure.</li>
<li>They frustrate average users, who then become uncooperative and use &quot;creative&quot; workarounds that make their passwords <em>less</em> secure.</li>
<li>They are often wrong, in the sense that the rules chosen are grossly incomplete and/or insane, per the many shaming links I've shared above.</li>
<li>Seriously, for the <em>love of God</em>, stop with this arbitrary password rule nonsense already. If you won't take my word for it, read <a href="https://nakedsecurity.sophos.com/2016/08/18/nists-new-password-rules-what-you-need-to-know/">this 2016 NIST password rules recommendation</a>. It's right there, &quot;no composition rules&quot;. However, I do see one error, it should have said &quot;no <em>bullshit</em> composition rules&quot;.</li>
</ul>
<h4 id="2enforceaminimumunicodepasswordlength">2. Enforce a minimum <em>Unicode</em> password length</h4>
<p>One rule is at least easy to remember, understand, and enforce. This is the proverbial one rule to bring them all, and in the darkness bind them.</p>
<p><img src="/content/images/2017/03/one-donut-to-bind-them-all.jpg" alt=""></p>
<ul>
<li>It's simple. Users can count. Most of them, anyway.</li>
<li>It works. The data <em>shows us</em> it works; just download any common password list of your choice and group by password length.</li>
<li>The math doesn't lie. All other things being equal, a longer password <em>will</em> be more random – and thus more secure – than a short password.</li>
<li>Accept that even this one rule isn't inviolate. A minimum password length of 6 on a Chinese site <em>might</em> be perfectly reasonable. A 20 character password <em>can</em> be ridiculously insecure.</li>
<li>If you don't allow (almost) every single unicode character in the password input field, you are probably doing it wrong.</li>
<li>It's a bit of an implementation detail, but make sure <em>maximum</em> password length is reasonable as well.</li>
</ul>
<h4 id="3checkforcommonpasswords">3. Check for common passwords</h4>
<p>As I've already noted, the definition of &quot;common&quot; depends on your audience, and language, but it is a terrible disservice to users when you let them choose passwords that exist in the list of 10k, 100k, or million most common known passwords from data breaches. There's <em>no question</em> that a hacker will submit these common passwords in a hack attempt – and it's shocking how far you can get, even with aggressive password attempt rate limiting, using <a href="https://xato.net/10-000-top-passwords-6d6380716fe0">just the 1,000 most common passwords</a>.</p>
<ul>
<li>1.6% have a password from the top 10 passwords</li>
<li>4.4% have a password from the top 100 passwords</li>
<li>9.7% have a password from the top 500 passwords</li>
<li>13.2% have a password from the top 1,000 passwords</li>
<li>30% have a password from the top 10,000 passwords</li>
</ul>
<p>Lucky you, there are millions and millions of real breached password lists out there to sift through. It is sort of fun to do data forensics, because these aren't hypothetical synthetic Jack the Ripper password rules some bored programmer dreamed up, these are <em>real</em> passwords used by <em>real</em> users.</p>
<p>Do the research. Collect the data. Protect your users from themselves.</p>
<h4 id="4checkforbasicentropy">4. Check for basic entropy</h4>
<p>No need to get fancy here; pick the measure of entropy that satisfies you deep in the truthiness of your gut. But remember you have to be able to <em>explain</em> it to users when they fail the check, too.</p>
<p><a href="http://www.digifail.com/software/spectra.shtml"><img src="/content/images/2017/03/entropy2.png" alt="entropy visualized"></a></p>
<p>I had a bit of a sad when I realized that we were perfectly fine with users selecting a 10 character password that was literally &quot;aaaaaaaaaa&quot;. In my opinion, the simplest way to do this is to ensure that there are at least (x) unique characters out of (y) total characters. And that's what we do as of the current beta version of Discourse. But I'd love your ideas in the comments, too. The simpler and clearer the better!</p>
<h4 id="5checkforspecialcasepasswords">5. Check for special case passwords</h4>
<p>I'm embarrassed to admit that when building the Discourse login, <a href="https://blog.codinghorror.com/the-god-login/">as I discussed in The God Login</a>, we missed two common cases that you really <em>have</em> to block:</p>
<ul>
<li>password equal to username</li>
<li>password equal to email address</li>
</ul>
<p>🤦 If you are using Discourse versions earlier than 1.4, I'm so sorry and <em>please upgrade immediately</em>.</p>
<p>Similarly, you might also want to block other special cases like</p>
<ul>
<li>password equal to URL or domain of website</li>
<li>password equal to app name</li>
</ul>
<p>In short, try to think outside the password input box, like a user would.</p>
<blockquote>
<p>🔔 <strong>Clarification</strong></p>
<p>A few people have interpreted this post as &quot;all the <em>other</em> password rules are bullshit, except these four I will now list.&quot; That's not what I'm trying to say here.</p>
<p>The idea is to focus on the one understandable, simple, practical, works-in-real-life-in-every-situation rule: <strong>length</strong>. Users can enter (almost) anything, in proper Unicode, <em>provided it's long enough</em>. That's the <strong>one rule to bind them all</strong> that we need to teach users: length!</p>
<p>Items #3 through #5 are more like genie-special-exception checks, a <a href="https://www.youtube.com/watch?v=Bwic3hJ4q1A">you can't wish for infinite wishes</a> kind of thing. It doesn't need to be discussed up front because it <em>should</em> be really rare. Yes, you must stop users from having comically bad passwords that equal their username, or <code>aaaaaaaaaaa</code> or <code>0123456789</code>, but only as post-entry checks, not as rules that need to be explained in advance.</p>
<p>So TL;DR: one rule. Length. Enter whatever you want, just make sure it's long enough to be a reasonable password.</p>
</blockquote>
<table>
<tr><td class="welovecodinghorror">
[advertisement] Building out your tech team? <a href="http://careers.stackoverflow.com/products" rel="nofollow">Stack Overflow Careers</a> helps you hire from the largest community for programmers on the planet. We built our site with developers like you in mind.
</td></tr>
</table>
</div>
<b><a href="//blog.codinghorror.com/password-rules-are-bullshit/#discourse-comments">Discussion</a></b>
</section>
</article>
<nav class="pagination" role="navigation">
<span class="page-number">Page 1 of 284</span>
<a class="older-posts" href="/page/2/">Older Posts <span aria-hidden="true">&rarr;</span></a>
</nav>
</main>
<script type="text/javascript">
var discourseUrl = "https://discourse.codinghorror.com/";
(function() {
var d = document.createElement('script'); d.type = 'text/javascript'; d.async = true;
d.src = discourseUrl + 'javascripts/count.js';
(document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(d);
})();
</script>
<aside class="sidebar">
<div id="welovecodinghorror-block">
<span>
<span class="block-wrap">
<a href="https://www.jetbrains.com/rider/?utm_source=codinghorror&utm_medium=cpc&utm_campaign=rider" class="block-img" target="_blank"><img src="/content/images/2017/12/jetbrains-rider.png" width="140" height="140" alt="" border="0" max-width="140px"></a>
<a href="https://www.jetbrains.com/rider/?utm_source=codinghorror&utm_medium=cpc&utm_campaign=rider" class="block-text" target="_blank">JetBrains Rider: new cross-platform .NET IDE. Develop .NET, ASP.NET, .NET Core, Unity on Windows, Mac, Linux</a>
</span>
</span>
</div>
<script async type="text/javascript" src="//cdn.carbonads.com/carbon.js?serve=CKYIK23L&placement=blogcodinghorrorcom" id="_carbonads_js"></script>
<div id="hireme" class="hireme codinghorror" style="min-height: 220px; margin-bottom: 15px;"></div>
<script>
setTimeout(function () {
var a = document.createElement("script");
var b = document.getElementsByTagName('script')[0];
a.src = "//clc.stackoverflow.com/j/p?d=hireme";
a.async = true;
a.type = "text/javascript";
b.parentNode.insertBefore(a, b);
}, 5);
</script>
<div class="welovecodinghorror" style="margin-bottom:15px">
[ad] Enjoy the blog? Read <b><a href="https://www.hyperink.com/Effective-Programming-More-Than-Writing-Code-b1559">Effective Programming: More than Writing Code</a></b> and <b><a href="https://www.hyperink.com/How-To-Stop-Sucking-And-Be-Awesome-Instead-b9A74B5CBA6">How to Stop Sucking and Be Awesome Instead</a></b> on your Kindle, iPad, Nook, or as a PDF.
</div>
<h3>Resources</h3>
<ul>
<li><a href="/about-me/">About Me</a></li>
<li><a href="https://www.discourse.org/">discourse.org</a></li>
<li><a href="https://stackexchange.com/">stackexchange.com</a></li>
<li><a href="http://commonmark.org/help/">Learn Markdown</a></li>
<li><a href="/recommended-reading-for-developers/">Recommended Reading</a></li>
</ul>
<ul>
<li><a href="https://feeds.feedburner.com/codinghorror" class="icon-feed">&nbsp;Subscribe in a reader</a></li>
<li><a href="https://feedburner.google.com/fb/a/mailverify?uri=codinghorror&amp;loc=en_US" class="icon-email">&nbsp;Subscribe via email</a></li>
</ul>
<p>Coding Horror has been continuously published since 2004</p>
<footer class="site-footer">
<section class="copyright">Copyright <a rel="author" href="https://en.wikipedia.org/wiki/Jeff_Atwood">Jeff Atwood</a> &copy; 2018<br />
Logo image &copy; 1993 Steven C. McConnell <br />
Proudly published with <a class="icon-ghost" href="http://ghost.org">Ghost</a></section>
</footer></aside>
</div>
<script>
document.write(unescape("%3Cscript src='https://sb.scorecardresearch.com/beacon.js'%3E%3C/script%3E"));
</script>
<script>
COMSCORE.beacon({
c1: 2,
c2: "6035669",
c3: "",
c4: "http://www.codinghorror.com/blog/",
c5: "",
c6: "",
c15: ""
});
</script>
<noscript>
<img src="//b.scorecardresearch.com/b?c1=2&amp;c2=6035669&amp;c3=&amp;c4=http%3A%2F%2Fwww.codinghorror.com%2Fblog%2F&amp;c5=&amp;c6=&amp;c15=&amp;cv=1.3&amp;cj=1" style="display:none" width="0" height="0" alt="" />
</noscript>
</body>
</html>

38135
bench/tests/es2019-draft.html Normal file
View File

File diff suppressed because one or more lines are too long

6
bench/tests/google.html Normal file
View File

File diff suppressed because one or more lines are too long

4676
bench/tests/nytimes.html Normal file
View File

File diff suppressed because it is too large Load Diff

3891
bench/tests/stackoverflow.html Normal file
View File

File diff suppressed because it is too large Load Diff

1893
bench/tests/wiki.html Normal file
View File

File diff suppressed because one or more lines are too long

2573
bench/tests/yahoo.html Normal file
View File

File diff suppressed because one or more lines are too long