ftw

.gitignore

@@ -1,4 +1,3 @@
 *.swp
 *.pyc
-db.shelve
 env

README.md

@@ -0,0 +1,20 @@
+# Google(Beijing) hosts generator
+
+这个工具做了三件事：
+
+1. 用ping暴力扫描Google北京IP段的存活主机
+2. 遍历检测所有存活主机443端口SSL证书的subject common name(CN) 得到`IP-域名`的映射
+3. 对`google_domains.txt`中每个每个域名进行解析
+
+其中步骤1和步骤2的结果默认会被缓存在`db.shelve`中
+
+## 安装依赖
+
+    pip install -r requirements.txt
+
+## 使用
+
+    cd src
+    python main.py # 可选 -f 会强制执行步骤1和步骤2
+    # 运行完毕后 main.py 同目录会生成hosts文件。其中有些域名可能不能被正常解析，这时你可能要手动打个patch
+    patch hosts postfix.patch

requirements.txt

@@ -0,0 +1 @@
+M2Crypto >= 0.21.1

google_domains.txt → src/google_domains.txt

@@ -1,3 +1,7 @@
+#
+# 所有需要解析的Google主机地址
+#
+
 groups-china.l.google.com
 khm1.google.com
 khm0.google.com

src/hosts

@@ -0,0 +1,111 @@
+203.208.36.1   groups-china.l.google.com
+203.208.36.1   khm1.google.com
+203.208.36.1   khm0.google.com
+203.208.36.1   images.google.com
+203.208.36.1   encrypted-tbn0.google.com
+203.208.36.1   encrypted.google.com
+203.208.36.1   drive.google.com
+203.208.36.1   0.docs.google.com
+203.208.36.1   books.google.com
+203.208.36.1   docs.google.com
+203.208.36.1   dl.google.com
+203.208.36.1   code.google.com
+203.208.36.1   clients7.google.com
+203.208.36.1   clients6.google.com
+203.208.36.1   clients5.google.com
+203.208.36.1   clients4.google.com
+203.208.36.1   clients3.google.com
+203.208.36.1   clients2.google.com
+203.208.36.1   clients1.google.com
+203.208.36.1   chrome.google.com
+203.208.36.1   appengine.google.com
+203.208.36.1   apis.google.com
+203.208.36.23   google.com
+203.208.36.1   imap.gmail.com gmail-imap.l.google.com
+203.208.36.1   0.drive.google.com
+203.208.36.1   lh6.google.com
+203.208.36.1   lh5.google.com
+203.208.36.1   lh4.google.com
+203.208.36.1   lh3.google.com
+203.208.36.1   lh2.google.com
+203.208.36.1   lh6.ggpht.com
+203.208.36.1   lh5.ggpht.com
+203.208.36.1   lh4.ggpht.com
+203.208.36.1   lh3.ggpht.com
+203.208.36.1   www.blogger.com
+203.208.36.1   www.appspot.com
+203.208.36.1   appspot.com
+203.208.36.15   ssl.gstatic.com
+203.208.36.10   www-opensocial.googleusercontent.com
+203.208.36.10   oauth.googleusercontent.com
+203.208.36.1   sandbox.google.com
+203.208.36.1   safebrowsing-cache.google.com
+203.208.36.1   upload.docs.google.com
+203.208.36.1   hostedtalkgadget.google.com
+203.208.36.1   gg.google.com
+203.208.36.30   ssl.google-analytics.com
+203.208.36.1   toolbarqueries.clients.google.com
+203.208.36.1   hostedtalkgadget.google.com
+203.208.36.1   browsersync.google.com
+203.208.36.1   groups.google.com
+203.208.36.1   encrypted-tbn3.google.com
+203.208.36.1   encrypted-tbn2.google.com
+203.208.36.1   encrypted-tbn1.google.com
+203.208.36.1   talkgadget.l.google.com
+203.208.36.1   writely.l.google.com
+203.208.36.1   writely-china.l.google.com
+203.208.36.21   mail.google.com
+203.208.36.29   chatenabled.mail.google.com
+203.208.36.1   maps.google.com
+203.208.36.1   maps-api-ssl.google.com
+203.208.36.1   mt0.google.com
+203.208.36.1   mt1.google.com
+203.208.36.1   mts0.google.com
+203.208.36.1   mts1.google.com
+203.208.36.1   mts2.google.com
+203.208.36.1   mts3.google.com
+203.208.36.1   news.google.com
+203.208.36.1   picasaweb.google.com
+203.208.36.1   plus.google.com
+203.208.36.1   plusone.google.com
+203.208.36.1   profiles.google.com
+203.208.36.1   scholar.google.com
+203.208.36.1   sites.google.com
+203.208.36.1   spreadsheet.google.com
+203.208.36.1   spreadsheets.google.com
+203.208.36.1   spreadsheets.google.com
+203.208.36.1   talkgadget.google.com
+203.208.36.1   translate.google.com
+203.208.36.1   translate.google.cn
+203.208.36.1   plus.url.google.com
+203.208.36.16   www.google.com
+74.125.31.95   ajax.googleapis.com
+74.125.31.95   www.googleapis.com
+203.208.36.1   kelvin-mirex-svn.googlecode.com
+203.208.36.1   kelvin-agent.appspot.com
+203.208.36.10   clients1.googleusercontent.com
+203.208.36.10   clients2.googleusercontent.com
+203.208.36.10   clients3.googleusercontent.com
+203.208.36.10   clients4.googleusercontent.com
+203.208.36.10   clients5.googleusercontent.com
+203.208.36.10   clients6.googleusercontent.com
+203.208.36.10   clients7.googleusercontent.com
+203.208.36.10   images1-focus-opensocial.googleusercontent.com
+203.208.36.10   images2-focus-opensocial.googleusercontent.com
+203.208.36.10   images3-focus-opensocial.googleusercontent.com
+203.208.36.10   lh1.googleusercontent.com
+203.208.36.10   lh2.googleusercontent.com
+203.208.36.10   lh3.googleusercontent.com
+203.208.36.10   lh4.googleusercontent.com
+203.208.36.10   lh5.googleusercontent.com
+203.208.36.10   lh6.googleusercontent.com
+203.208.36.10   mail-attachment.googleusercontent.com
+203.208.36.10   doc-0g-0g-docsviewer.googleusercontent.com
+203.208.36.10   s2.googleusercontent.com
+203.208.36.10   static.googleusercontent.com
+203.208.36.10   themes.googleusercontent.com
+203.208.36.10   translate.googleusercontent.com
+203.208.36.10   webcache.googleusercontent.com
+203.208.36.1   www.googlevideo.com
+203.208.36.15   gstatic.com
+203.208.36.15   maps.gstatic.com

gen_google_hosts.py → src/main.py

@@ -1,5 +1,8 @@
 # coding: UTF-8
 #
+# 自动解析部分Google北京hosts
+# author: mckelvin
+# 
 import os
 import sys
 import subprocess
@@ -12,12 +15,14 @@
 SHELVE_FILE = 'db.shelve'
 GOOGLE_DOMAINS_FILE = 'google_domains.txt'
 K_LATEST_MAPPING_RESULT = 'latest_mapping_result'
-K_LATEST_PING_RESULT = 'latest_ping_ressult'
+K_LATEST_PING_RESULT = 'latest_ping_result'
 
 def one_ping(ip):
-    res = subprocess.Popen(['ping', '-c', '1', '-n', '-W', '1', ip],
+    ping_count = '1'
+    ping_timeout = '1'
+    res = subprocess.Popen(['ping', '-c', ping_count, '-n', '-W', ping_timeout, ip],
             stdout=subprocess.PIPE, stderr=subprocess.PIPE).wait()
-    print 'ping %s\r' % ip
+    print 'ping %s : %s' % (ip , 'active' if res == 0 else 'inactive')
     return ip, res
 
 def get_candidate_google_ips():
@@ -55,7 +60,7 @@ def resolve_host_ip_mapping():
     for each_ip in active_ips:
         each_ip = each_ip.strip()
         c_CN = get_ssl_subject_CN(each_ip)
-        print each_ip, c_CN
+        print 'find mapping: %s in %s' % (each_ip, c_CN)
         host_ips.setdefault(c_CN, []).append(each_ip)
     host_ips.pop(None)
 
@@ -64,21 +69,23 @@ def resolve_host_ip_mapping():
     s.close()
 
 # STEP 3
-def generate_hosts_file():
+def generate_hosts():
     '''通过已知host-ip隐射生成hosts条目'''
     s = shelve.open(SHELVE_FILE)
     host_ips = s[K_LATEST_MAPPING_RESULT]
+    s.close()
     hosts = []
     with open(GOOGLE_DOMAINS_FILE) as fh:
         for domain in fh:
             domain = domain.strip()
+            if domain.startswith('#') or domain == '':
+                continue
             ip = None
             for domain_pattern in host_ips.keys():
                 if domain_pattern is not None and fnmatch(domain, domain_pattern):
                     ip = host_ips[domain_pattern][0]
                     break
             hosts.append((domain, ip))
-    s.close()
     return hosts
 
 def main():
@@ -88,13 +95,17 @@ def main():
     (opts, args) = parser.parse_args()
 
     s = shelve.open(SHELVE_FILE)
-    if not s.get(K_LATEST_PING_RESULT) or opts.force:
+    latest_ping_result = s.get(K_LATEST_PING_RESULT)
+    latest_mapping_result = s.get(K_LATEST_MAPPING_RESULT)
+    s.close()
+    if not latest_ping_result or opts.force:
         find_active_ips() #step 1
-    if not s.get('resolve_host_ip_mapping') or opts.force:
+    if not latest_mapping_result or opts.force:
         resolve_host_ip_mapping() #step2
     hosts = generate_hosts() # test3
-    for i in hosts:
-        print '%s   %s' % (i[1] or '#unknowip', i[0])
+    with open('hosts','w') as fh:
+        fh.writelines(('%s   %s\n' % (i[1] or '#unknown', i[0]) for i in hosts))
+    print 'done! checkout `hosts` in current directory'
 
 if __name__ == '__main__':
     main()

src/postfix.patch

@@ -0,0 +1,36 @@
+31,37c31,37
+< #unknown   lh6.ggpht.com
+< #unknown   lh5.ggpht.com
+< #unknown   lh4.ggpht.com
+< #unknown   lh3.ggpht.com
+< #unknown   www.blogger.com
+< #unknown   www.appspot.com
+< #unknown   appspot.com
+---
+> 203.208.36.1   lh6.ggpht.com
+> 203.208.36.1   lh5.ggpht.com
+> 203.208.36.1   lh4.ggpht.com
+> 203.208.36.1   lh3.ggpht.com
+> 203.208.36.1   www.blogger.com
+> 203.208.36.1   www.appspot.com
+> 203.208.36.1   appspot.com
+79c79
+< #unknown   translate.google.cn
+---
+> 203.208.36.1   translate.google.cn
+82,85c82,85
+< #unknown   ajax.googleapis.com
+< #unknown   www.googleapis.com
+< #unknown   kelvin-mirex-svn.googlecode.com
+< #unknown   kelvin-agent.appspot.com
+---
+> 74.125.31.95   ajax.googleapis.com
+> 74.125.31.95   www.googleapis.com
+> 203.208.36.1   kelvin-mirex-svn.googlecode.com
+> 203.208.36.1   kelvin-agent.appspot.com
+109,110c109,110
+< #unknown   www.googlevideo.com
+< #unknown   gstatic.com
+---
+> 203.208.36.1   www.googlevideo.com
+> 203.208.36.15   gstatic.com