import 'dart:async'; import 'dart:convert'; import 'dart:io'; import 'dart:cli'; // +---------+------------------+------+-----+---------+----------------+ // | Field | Type | Null | Key | Default | Extra | // +---------+------------------+------+-----+---------+----------------+ // | postID | int(10) unsigned | NO | PRI | NULL | auto_increment | // | title | text | NO | | NULL | | // | content | text | NO | | NULL | | // | stamp | int(10) unsigned | NO | | 0 | | // +---------+------------------+------+-----+---------+----------------+ const Map titles = { 'https://plus.google.com/+IanHickson/posts/2S1ETVb5SKL': 'Improving on HTML', 'https://plus.google.com/+IanHickson/posts/NZBJe6Jjt1f': 'Living Standards', 'https://plus.google.com/+IanHickson/posts/SiLdNL9MsFw': 'Replacing HTML', 'https://plus.google.com/+IanHickson/posts/4MrPNJzE8vV': 'Consensus', 'https://plus.google.com/+IanHickson/posts/iPmatxBYuj2': 'DRM', 'https://plus.google.com/+IanHickson/posts/aXEBL5BwXYw': 'HTML 8000', 'https://plus.google.com/+IanHickson/posts/iAozXGH7Eck': 'Looping constructs', 'https://plus.google.com/+IanHickson/posts/RaHnqNJYGoB': 'Flutter: Accessibility', 'https://plus.google.com/+IanHickson/posts/fJBZkA5Tu4V': 'Flutter: Developer experience', 'https://plus.google.com/+IanHickson/posts/GJBrEN9ukRo': 'Flutter: Performance and testing', 'https://plus.google.com/+IanHickson/posts/KyDSnQXomUH': 'Flutter\'s mahogany staircase', 'https://plus.google.com/+IanHickson/posts/2ZMcXwkKLDG': 'Flutter at Google I/O', 'https://plus.google.com/+IanHickson/posts/hmUd1Q7FSD2': 'Flutter: Embrace the yak shave', 'https://plus.google.com/+IanHickson/posts/K8HVNrLCFvc': 'Flutter: Hot reload', 'https://plus.google.com/+IanHickson/posts/a1bdr27bDK8': 'Tolerance', 'https://plus.google.com/+IanHickson/posts/ihQjXCRvmEK': 'Flutter: So what\'d I miss?', 'https://plus.google.com/+IanHickson/posts/P3SnMrKCpoq': 'Flutter: Negative margins', 'https://plus.google.com/+IanHickson/posts/DQ114jUKaXo': 'Indexing into a string', }; class FailedMigration implements Exception { FailedMigration(this.message); final String message; String toString() => message; } String escapeSql(String raw) { if (raw == null) return ''; return raw .replaceAll(r"\", r"\\") .replaceAll("\b", r"\b") .replaceAll("\n", r"\n") .replaceAll("\r", r"\r") .replaceAll("\t", r"\t") .replaceAll(r"%", r"\%") .replaceAll(r"'", r"\'") .replaceAll(r"_", r"\_"); } String escapeHtml(String raw) { if (raw == null) return ''; return raw .replaceAll('&', '&') .replaceAll('<', '<') .replaceAll('>', '>') .replaceAll('"', '"'); } class Post { Post(this.id, this.title, this.content, this.stamp); final String id; final String title; final String content; final DateTime stamp; String toSql() => "INSERT INTO posts (title, content, stamp) VALUES ('${escapeSql(title)}', '${escapeSql(content)}', ${stamp.millisecondsSinceEpoch ~/ 1000});"; String toHtml() => "\n"; } HttpClient client = new HttpClient() ..userAgent = 'Google+ to ln.hixie.ch migration agent, contact ian@hixie.ch'; Map attempts = {}; String substituteLinks(String oldUrl) { assert(oldUrl != null); if (oldUrl == 'http://www.koncision.com/using-reasonable-and-reasonably-in-contracts/') return 'https://web.archive.org/web/20130211130051/$oldUrl'; try { if (!attempts.containsKey(oldUrl)) { attempts[oldUrl] = waitFor(client.getUrl(Uri.parse(oldUrl)).then((request) => request.close()), timeout: const Duration(seconds: 5)) ..drain(); } HttpClientResponse response = attempts[oldUrl]; if (response.statusCode != 200) throw FailedMigration('${response.statusCode} when fetching $oldUrl'); } on TimeoutException { throw FailedMigration('Timeout when fetching $oldUrl'); } return oldUrl; } final RegExp _htmlBrRegexp = RegExp(r'(
)+'); final RegExp _htmlTrailingSpaceInTag = RegExp(' +>'); final RegExp _htmlMultispace = RegExp(' +'); final RegExp _htmlLinebreaks = RegExp('\n+'); String processHtml(String html) { return '

' + html .replaceAll(_htmlBrRegexp, '\n

') .replaceAll('class="ot-anchor bidi_isolate"', '') .replaceAll('dir="ltr"', '') .replaceAll('jslog="10929; track:click"', '') .replaceAll('rel="nofollow"', '') .replaceAll('target="_blank"', '') .replaceAll(_htmlTrailingSpaceInTag, '>') .replaceAll(_htmlMultispace, ' ') .replaceAll(_htmlLinebreaks, '\n'); } String processMedia(var media) { switch (media['contentType']) { case 'image/*': case 'image/png': case 'image/jpeg': return '\n

${escapeHtml(media['description'])}
'; case 'video/*': String url = media['url']; const String kYouTube = '//www.youtube.com/'; if (url.contains(kYouTube)) { url = url.replaceAll('http://', 'https://'); String videoId; const String kYouTubeWatch = 'https://www.youtube.com/watch?v='; if (url.startsWith(kYouTubeWatch)) videoId = url.substring(kYouTubeWatch.length); const String kYouTubeAttribution = 'https://www.youtube.com/attribution_link?a='; if (url.startsWith(kYouTubeAttribution)) videoId = url.substring(kYouTubeAttribution.length, url.indexOf('&')); return '\n

${escapeHtml(media['description'])}
'; } return '\n

${escapeHtml(media['description'])}
'; default: throw 'no code to handle ${media['contentType']}'; } } String processLink(var link) { if (link['imageUrl'] != null) { try { return '\n

${escapeHtml(link['title'])}
'; } on FailedMigration { // try again below } } return '\n

${escapeHtml(link['title'])}'; } String replaceBareLinks(String html, String url, String replacement) { String pattern = '\n

${escapeHtml(url)}'; if (html.contains(pattern)) return html.replaceAll(pattern, replacement); return html += replacement; } void main() { List posts = []; for (File file in Directory('Posts').listSync().whereType()) { try { String rawData = file.readAsStringSync(); var data = json.decode(rawData); if (data['postAcl']['isPublic'] != true) { throw FailedMigration('not public'); } if (!titles.containsKey(data['url'])) { throw FailedMigration('excluded'); } if (data.containsKey('resharedPost')) { throw FailedMigration('reshared post'); } if (data.containsKey('poll')) { throw FailedMigration('poll'); } String html = ''; bool hasContent; if (data.containsKey('content')) { //if (data['content'].contains('comments')) // throw FailedMigration('contains mention of comments'); html += processHtml(data['content']); hasContent = true; } else { hasContent = false; } if (data.containsKey('media')) { assert(!data.containsKey('album')); assert(!data.containsKey('link')); if (!hasContent || !data['content'].contains(data['media']['url'])) html = replaceBareLinks(html, data['media']['url'], processMedia(data['media'])); } if (data.containsKey('album')) { assert(!data.containsKey('media')); assert(!data.containsKey('link')); for (var media in data['album']['media']) { html = replaceBareLinks(html, media['url'], processMedia(media)); } } if (data.containsKey('link')) { assert(!data.containsKey('media')); assert(!data.containsKey('album')); if (!hasContent || !data['content'].contains(data['link']['url'])) html = replaceBareLinks(html, data['link']['url'], processLink(data['link'])); } if (!hasContent) throw FailedMigration('no content except for $html'); String title = ''; if (titles.containsKey(data['url'])) { title = titles[data['url']]; } else if (data['postAcl'].containsKey('collectionAcl')) { title = data['postAcl']['collectionAcl']['collection']['displayName'] + ': '; } else { title = 'TODO(ianh): Add title to migrated blog post'; } stderr.writeln('+ migrated ${file.path} ("$title")'); posts.add(Post(data['url'], title, html, DateTime.parse(data['creationTime']))); } on FailedMigration catch (error) { stderr.writeln('- not migrating because $error: ${file.path}'); } } print(''' Hixie's Natural Log

loge.hixie.ch

Hixie's Natural Log

'''); print(posts.map((Post post) => post.toHtml()).join('\n')); print(''); }