#!/usr/bin/env python2 # Copyright Xyne, 2010 except where otherwise noted. from json import dumps, loads from optparse import OptionParser from sys import argv, exit from urllib2 import Request, URLError, build_opener, install_opener, quote, urlopen from urlparse import urlsplit # API url = 'https://ajax.googleapis.com/ajax/services/search/'; types = 'web local video blogs news books images patent'.split() # API Documentation api_doc_url = 'http://code.google.com/apis/ajaxsearch/documentation/reference.html#_intro_fonje' # Default privoxy url privoxy_url = '127.0.0.1:8118' # Options name = 'sigo' parser = OptionParser(description= name.capitalize() + ' - a simple command-line tool for querying Google\'s secure (HTTPS) AJAX Search API, with implicit and explicit proxy support for e.g. Tor via Privoxy. ' + name.capitalize() + ' uses Python\'s urllib2 module and should auto-detect your proxy settings. If you have not set them or if it fails to detect them, you can use the options below to explicitly set your proxy server. Sigo will pretty-print the JSON response using Python\'s json module. Other output formats may be added later. By passing HTTPS requests via an anonymous proxy such as Tor, it should prevent others from inspecting your searches and maintain your anonymity to Google. However, Sigo is provided WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. It is left to the concerned user to inspect the code, determine its fitness for the intended purpose, and modify it as needed. The author assumes absolutely no responsibility for any consequences incurred through use of Sigo.', usage=name + ' [options] ') parser.add_option("-l", "--hl", dest="hl", action="store", metavar='', help='The host language. Defaults to "en"') parser.add_option("-p", "--proxy", dest="proxy", action="store_true",help='Explicitly use proxy. See "--proxy-url".') parser.add_option("--proxy-url", dest="proxy_url", default=privoxy_url, action="store", help='The explicit proxy URL. Default: ' + privoxy_url + ' (Privoxy)') parser.add_option("-q", dest="q", action="store", metavar='', help='The query. If this option is not supplied, non-option arguments will be used instead.') parser.add_option("-r", "--rsz", dest="rsz", default=8, type="int", action="store", metavar='n', help='The number of results to request. This may be between 1 and 8. The limit is imposed by Google\s API.') parser.add_option("-s", "--start", dest="start", type="int", action="store", metavar='n', help='Start index of the search result.') parser.add_option("-t", "--type", dest="type", default=types[0], action="store", metavar='', help='The type of search. This may be one of the following: "' + '", "'.join(types) + '". Default: ' + types[0] + '. See the documentation at ' + api_doc_url + ' for other recognized types.') parser.add_option("-u", "--url", dest="url", action="store_true",help='Print the request URL instead of sending the request.') parser.add_option("--userip", dest="userip", action="store", metavar='', help='IP address of end user. This is not required and it only provided for completeness') parser.add_option("-v", dest="version", default="1.0", action="store", metavar='', help='Protocol version. Default is "1.0". Check the documentation at ' + api_doc_url + ' for other valid values.') ################################################################################ # Recipe 456195: urrlib2 opener for SSL proxy (CONNECT method) (Python) # Taken from http://code.activestate.com/recipes/456195/ # Created by Alessandro Budai # Modifications: # *) added "timeout" keywork argument to ProxyHTTPSConnection.__init__ # *) updated ProxyHTTPSConnection.connect to use ssl module ################################################################################ import urllib2 import urllib import httplib import socket import ssl class ProxyHTTPConnection(httplib.HTTPConnection): _ports = {'http' : 80, 'https' : 443} def request(self, method, url, body=None, headers={}): #request is called before connect, so can interpret url and get #real host/port to be used to make CONNECT request to proxy proto, rest = urllib.splittype(url) if proto is None: raise ValueError, "unknown URL type: %s" % url #get host host, rest = urllib.splithost(rest) #try to get port host, port = urllib.splitport(host) #if port is not defined try to get from proto if port is None: try: port = self._ports[proto] except KeyError: raise ValueError, "unknown protocol for: %s" % url self._real_host = host self._real_port = port httplib.HTTPConnection.request(self, method, url, body, headers) def connect(self): httplib.HTTPConnection.connect(self) #send proxy CONNECT request self.send("CONNECT %s:%d HTTP/1.0\r\n\r\n" % (self._real_host, self._real_port)) #expect a HTTP/1.0 200 Connection established response = self.response_class(self.sock, strict=self.strict, method=self._method) (version, code, message) = response._read_status() #probably here we can handle auth requests... if code != 200: #proxy returned and error, abort connection, and raise exception self.close() raise socket.error, "Proxy connection failed: %d %s" % (code, message.strip()) #eat up header block from proxy.... while True: #should not use directly fp probablu line = response.fp.readline() if line == '\r\n': break class ProxyHTTPSConnection(ProxyHTTPConnection): default_port = 443 def __init__(self, host, port = None, key_file = None, cert_file = None, strict = None, timeout=60): ProxyHTTPConnection.__init__(self, host, port) self.key_file = key_file self.cert_file = cert_file def connect(self): ProxyHTTPConnection.connect(self) #make the sock ssl-aware ##### [edit] ##### #ssl = socket.ssl(self.sock, self.key_file, self.cert_file) #self.sock = httplib.FakeSocket(self.sock, ssl) self.sock = ssl.wrap_socket(self.sock, self.key_file, self.cert_file) ##### [/edit] ##### class ConnectHTTPHandler(urllib2.HTTPHandler): def do_open(self, http_class, req): return urllib2.HTTPHandler.do_open(self, ProxyHTTPConnection, req) class ConnectHTTPSHandler(urllib2.HTTPSHandler): def do_open(self, http_class, req): return urllib2.HTTPSHandler.do_open(self, ProxyHTTPSConnection, req) ################################################################################ # End of Recipe 456195 ################################################################################ def main(url): (options, args) = parser.parse_args() if not options.q: if not args: print "error: no query" exit(1) else: options.q = ' '.join(args) url += quote(options.type) + '?v=' + quote(options.version) + '&rsz=' + quote(str(options.rsz)) + '&q=' + quote(options.q) if options.hl: url += '&hl=' + quote(options.hl) if options.start: url += '&start=' + quote(str(options.start)) if options.userip: url += '&userip=' + quote(options.userip) if options.url: print url exit() request = Request(url, None, {'Referer': 'http://localhost/'}) if options.proxy: opener = build_opener(ConnectHTTPSHandler) install_opener(opener) url_parts = urlsplit(url) request.set_proxy(options.proxy_url, url_parts[0]) try: f = urlopen(request) except URLError as e: print "URLError: " + str(e) #print "Did you forget to set up your proxy?" exit(1) reply = f.read() f.close() try: j = loads( reply ) except ValueError as e: print "ValueError: " + str(e) print "Reply:\n" + reply exit(1) print dumps(j, sort_keys=False, indent=2) if __name__ == '__main__': try: main(url) except KeyboardInterrupt: print "KeyboardInterrupt: exiting..." exit()