From 328a720d72a4014b77bbb7fe5782be0125c09778 Mon Sep 17 00:00:00 2001 From: Nick Daly Date: Mon, 14 May 2012 22:21:12 -0500 Subject: [PATCH] Almost done. --- ugly_hacks/santiago/protocols/https.py | 20 +- ugly_hacks/santiago/simplesantiago.py | 163 +++++++++------ ugly_hacks/santiago/test_santiago.py | 267 +++++++++++++++++-------- 3 files changed, 301 insertions(+), 149 deletions(-) diff --git a/ugly_hacks/santiago/protocols/https.py b/ugly_hacks/santiago/protocols/https.py index e05f00536..5bd52ec58 100644 --- a/ugly_hacks/santiago/protocols/https.py +++ b/ugly_hacks/santiago/protocols/https.py @@ -3,9 +3,9 @@ from simplesantiago import SantiagoListener, SantiagoSender import cherrypy -import httplib, urllib +import httplib, urllib, urlparse import sys - +import logging class Listener(SantiagoListener): @@ -31,10 +31,11 @@ class Listener(SantiagoListener): def index(self, **kwargs): """Receive an incoming Santiago request from another Santiago client.""" + logging.debug("protocols.https.index: Received request {0}".format(str(kwargs))) try: self.incoming_request(kwargs["request"]) - except: - pass + except Exception as e: + logging.exception(e) @cherrypy.expose def query(self, host, service): @@ -44,6 +45,7 @@ class Listener(SantiagoListener): """ if not cherrypy.request.remote.ip.startswith("127.0.0"): + logging.debug("protocols.https.query: Request from non-local IP") return self.santiago.query(host, service) @@ -51,6 +53,7 @@ class Listener(SantiagoListener): @cherrypy.expose def save_server(self): if not cherrypy.request.remote.ip.startswith("127.0.0"): + logging.debug("protocols.https.save_server: Request from non-local IP") return self.santiago.save_server() @@ -70,8 +73,15 @@ class Sender(SantiagoSender): It's both simple and as reliable as possible. + ``request`` is literally the request's text. It needs to be wrapped for + transport across the protocol. + """ - params = urllib.urlencode(request) + logging.debug("protocols.https.Sender.outgoing_request: request {0}".format(str(request))) + to_send = { "request": request } + + params = urllib.urlencode(to_send) + logging.debug("protocols.https.Sender.outgoing_request: params {0}".format(str(params))) # TODO: Does HTTPSConnection require the cert and key? # Is the fact that the server has it sufficient? I think so. diff --git a/ugly_hacks/santiago/simplesantiago.py b/ugly_hacks/santiago/simplesantiago.py index 015e8a29d..7627f8dc1 100644 --- a/ugly_hacks/santiago/simplesantiago.py +++ b/ugly_hacks/santiago/simplesantiago.py @@ -26,12 +26,10 @@ We also don't: - Use a reasonable data-store. - Have a decent control mechanism. -:FIXME: add that whole pgp thing. :TODO: add doctests -:TODO: Create startup script that adds all necessary things to the PYTHONPATH. +:TODO: move to santiago.py, merge the documentation. :FIXME: allow multiple listeners and senders per protocol (with different proxies) -:TODO: move to santiago.py, merge the documentation. This dead-drop is what came of my trying to learn from bug 4185. @@ -66,12 +64,12 @@ class Santiago(object): """ SUPPORTED_PROTOCOLS = set([1]) - ALL_KEYS = ("host", "client", "service", "locations", "reply_to", - "request_version", "reply_versions") - REQUIRED_KEYS = ("client", "host", "service", - "request_version", "reply_versions") - OPTIONAL_KEYS = ("locations", "reply_to") - LIST_KEYS = ("reply_to", "locations", "reply_versions") + ALL_KEYS = set(("host", "client", "service", "locations", "reply_to", + "request_version", "reply_versions")) + REQUIRED_KEYS = set(("client", "host", "service", + "request_version", "reply_versions")) + OPTIONAL_KEYS = ALL_KEYS ^ REQUIRED_KEYS + LIST_KEYS = set(("reply_to", "locations", "reply_versions")) def __init__(self, listeners = None, senders = None, hosting = None, consuming = None, me = 0): @@ -121,7 +119,7 @@ class Santiago(object): connectors = dict() for protocol in settings.iterkeys(): - module = SimpleSantiago._get_protocol_module(protocol) + module = Santiago._get_protocol_module(protocol) try: connectors[protocol] = \ @@ -154,8 +152,8 @@ class Santiago(object): When this has finished, the Santiago will be ready to go. """ - for connector in list(self.listeners.itervalues()) + \ - list(self.senders.itervalues()): + for connector in (list(self.listeners.itervalues()) + + list(self.senders.itervalues())): connector.start() logging.debug("Santiago started!") @@ -197,6 +195,7 @@ class Santiago(object): except KeyError as e: logging.exception(e) + def query(self, host, service): """Request a service from another Santiago. @@ -204,8 +203,6 @@ class Santiago(object): """ try: - self.requests[host].add(service) - self.outgoing_request( host, self.me, host, self.me, service, None, self.get_client_locations(host, "santiago")) @@ -218,15 +215,25 @@ class Santiago(object): This tag is used when sending queries or replies to other Santiagi. - """ - # FIXME sign the encrypted payload. - payload = self.gpg.encrypt( - {"host": host, "client": client, - "service": service, "locations": locations or "", - "reply_to": reply_to}, to, sign=self.me) - request = self.gpg.sign({"request": payload, "to": to}) + Each incoming item must be a single item or a list. - for destination in self.get_client_locations(to, "santiago"): + The outgoing ``request`` is literally the request's text. It needs to + be wrapped for transport across the protocol. + + """ + self.requests[host].add(service) + + request = self.gpg.encrypt( + str({ "host": host, "client": client, + "service": service, "locations": list(locations or ""), + "reply_to": list(reply_to), + "request_version": 1, + "reply_versions": list(Santiago.SUPPORTED_PROTOCOLS),}), + host, + sign=self.me) + + # FIXME use urlparse.urlparse instead! + for destination in self.get_client_locations(host, "santiago"): protocol = destination.split(":")[0] self.senders[protocol].outgoing_request(request, destination) @@ -250,14 +257,16 @@ class Santiago(object): """ # no matter what happens, the sender will never hear about it. try: - if not verify_message(request): - return + logging.debug("santiago.Santiago.incoming_request: request: {0}".format(str(request))) unpacked = self.unpack_request(request) if not unpacked: + logging.debug("santiago.Santiago.incoming_request: opaque request.") return + logging.debug("santiago.Santiago.incoming_request: unpacked {0}".format(str(unpacked))) + if unpacked["locations"]: self.handle_reply( unpacked["from"], unpacked["to"], @@ -277,27 +286,6 @@ class Santiago(object): except Exception as e: logging.exception("Error: ", str(e)) - def verify_request(self, request): - """Make sure the request meets minimum criteria before we process it. - - - The request must contain required keys. - - The request and client must be of and support protocol versions I - understand. - - """ - if False in map(request.__contains__, - ("request", "request_version", "reply_versions")): - return False - - if not (Santiago.SUPPORTED_PROTOCOLS & set(request["reply_versions"])): - return False - - if not (Santiago.SUPPORTED_PROTOCOLS & - set([request["request_version"]])): - return False - - return True - def unpack_request(self, request): """Decrypt and verify the request. @@ -307,11 +295,20 @@ class Santiago(object): Some lists are changed to sets here. This allows for set-operations (union, intersection, etc) later, making things much more intuitive. + The request and client must be of and support protocol versions I + understand. + """ request = self.gpg.decrypt(request) # skip badly signed messages or ones for other folks. if not (str(request) and request.fingerprint): + logging.debug( + "santiago.Santiago.unpack_request: fail request {0}".format( + str(request))) + logging.debug( + "santiago.Santiago.unpack_request: fail fingerprint {0}".format( + str(request.fingerprint))) return # copy out only required keys from request, throwing away cruft @@ -321,15 +318,31 @@ class Santiago(object): for key in Santiago.ALL_KEYS: request_body[key] = source[key] except KeyError: + logging.debug( + "santiago.Santiago.unpack_request: missing key {0}".format( + str(source))) return # required keys are non-null if None in [request_body[x] for x in Santiago.REQUIRED_KEYS]: + logging.debug( + "santiago.Santiago.unpack_request: blank key {0}: {1}".format( + key, str(request_body))) return # move lists to sets request_body = self.setify_lists(request_body) if not request_body: + logging.debug( + "santiago.Santiago.unpack_request: not sets {0}".format( + str(request_body))) + return + + # versions must overlap. + if not (Santiago.SUPPORTED_PROTOCOLS & request_body["reply_versions"]): + return + if not (Santiago.SUPPORTED_PROTOCOLS & + set([request_body["request_version"]])): return # set implied keys @@ -342,7 +355,7 @@ class Santiago(object): """Convert list nodes to sets.""" try: - for key in ("locations", "reply_to"): + for key in Santiago.LIST_KEYS: if request_body[key] is not None: request_body[key] = set(request_body[key]) except TypeError: @@ -368,8 +381,13 @@ class Santiago(object): - Reply to the client on the appropriate protocol. """ - # return if we won't host for the proxy or the client. - if False in map(self.hosting.__contains__, (from_, client)): + # give up if we won't host the service for the client. + try: + self.hosting[client][service] + except KeyError: + logging.debug( + "santiago.Santiago.handle_request: no host for you".format( + self.hosting)) return # if we don't proxy, learn new reply locations and send the request. @@ -404,24 +422,36 @@ class Santiago(object): locations, if we've requested locations for that service. """ + logging.debug("santiago.Santiago.handle_reply: local {0}".format(str(locals()))) + + # give up if we won't consume the service from the proxy or the client. try: - self.consuming[service][from_] - self.consuming[service][host] - except KeyError as e: + if service not in self.requests[host]: + logging.debug( + "santiago.Santiago.handle_reply: unrequested service {0}: ".format( + service, self.requests)) + return + except KeyError: + logging.debug( + "santiago.Santiago.handle_reply: unrequested host {0}: ".format( + host, self.requests)) return + # give up or proxy if the message isn't for me. if not self.i_am(to): + logging.debug( + "santiago.Santiago.handle_reply: not to {0}".format(to)) return - if not self.i_am(client): + logging.debug( + "santiago.Santiago.handle_reply: not client {0}".format(client)) self.proxy() return self.learn_service(host, "santiago", reply_to) + self.learn_service(host, service, locations) - if service in self.requests[host]: - self.learn_service(host, service, locations) - self.requests[host].remove(service) + self.requests[host].remove(service) def save_server(self): """Save all operational data to files. @@ -429,14 +459,14 @@ class Santiago(object): Save all files with the ``self.me`` prefix. """ - for datum in ("hosting", "consuming"): - name = "%s_%s" % (self.me, datum) + for key in ("hosting", "consuming"): + name = "%s_%s" % (self.me, key) try: with open(name, "w") as output: - output.write(str(getattr(self, datum))) + output.write(str(getattr(self, key))) except Exception as e: - logging.exception("Could not save %s as %s", datum, name) + logging.exception("Could not save %s as %s", key, name) class SantiagoConnector(object): """Generic Santiago connector superclass. @@ -463,8 +493,8 @@ class SantiagoListener(SantiagoConnector): method passes the request along to the Santiago host. """ - def incoming_request(self, **kwargs): - self.santiago.incoming_request(**kwargs) + def incoming_request(self, request): + self.santiago.incoming_request(request) class SantiagoSender(SantiagoConnector): """Generic Santiago Sender superclass. @@ -479,8 +509,8 @@ class SantiagoSender(SantiagoConnector): if __name__ == "__main__": - # FIXME: convert this to the withsqlite setup. - + logging.getLogger().setLevel(logging.DEBUG) + logging.raiseExceptions = False cert = "santiago.crt" listeners = { "https": { "socket_port": 8080, "ssl_certificate": cert, @@ -488,23 +518,24 @@ if __name__ == "__main__": senders = { "https": { "proxy_host": "localhost", "proxy_port": 8118} } mykey = "D95C32042EE54FFDB25EC3489F2733F40928D23A" - # mykey = "0928D23A" # my short key # load hosting try: hosting = load_data(mykey, "hosting") except IOError: hosting = { "a": { "santiago": set( ["https://localhost:8080"] )}, + "b": { "santiago": set( ["https://localhost:8080"] )}, mykey: { "santiago": set( ["https://localhost:8080"] )}} # load consuming try: consuming = load_data(mykey, "consuming") except IOError: consuming = { "santiago": { mykey: set( ["https://localhost:8080"] ), + "b": set( ["https://localhost:8080"] ), "a": set( ["someAddress.onion"] )}} # load the Santiago - santiago_b = SimpleSantiago(listeners, senders, - hosting, consuming, mykey) + santiago_b = Santiago(listeners, senders, + hosting, consuming, mykey) santiago_b.start() diff --git a/ugly_hacks/santiago/test_santiago.py b/ugly_hacks/santiago/test_santiago.py index f0b51696a..b02d4fdb8 100644 --- a/ugly_hacks/santiago/test_santiago.py +++ b/ugly_hacks/santiago/test_santiago.py @@ -43,6 +43,7 @@ import os import sys import unittest +import ast import gnupg import logging import simplesantiago as santiago @@ -452,65 +453,6 @@ import utilities # # pass - -class VerifyRequest(unittest.TestCase): - - """Are incoming requests handled correctly? - - - Messages come with a request. - - Each message identifies the Santiago protocol version it uses. - - Messages come with a range of Santiago protocol versions I can reply with. - - Messages that don't share any of my versions are ignored (either the - client or I won't be able to understand the message). - - Test this in a fairly hacky way. - - """ - def setUp(self): - self.santiago = santiago.Santiago() - self.request = { "request_version": 1, - "reply_versions": [1], - "request": None } - - def test_valid_message(self): - """A known good request passes.""" - - self.assertTrue(self.santiago.verify_request(self.request)) - - def test_required_keys_are_required(self): - """Messages without required keys fail. - - The following keys are required in the un-encrypted part of the message: - - - request - - request_version - - reply_versions - - """ - for key in ("request", "request_version", "reply_versions"): - del self.request[key] - - self.assertFalse(self.santiago.verify_request(self.request)) - - def test_require_protocol_version_overlap(self): - """Clients that can't accept protocols I can send are ignored.""" - - santiago.Santiago.SUPPORTED_PROTOCOLS, unsupported = \ - set(["e"]), santiago.Santiago.SUPPORTED_PROTOCOLS - - self.assertFalse(self.santiago.verify_request(self.request)) - - santiago.Santiago.SUPPORTED_PROTOCOLS, unsupported = \ - unsupported, santiago.Santiago.SUPPORTED_PROTOCOLS - - def test_require_protocol_version_understanding(self): - """I must ignore any protocol versions I can't understand.""" - - self.request["request_version"] = "e" - - self.assertFalse(self.santiago.verify_request(self.request)) - - class UnpackRequest(unittest.TestCase): """Are requests unpacked as expected? @@ -518,6 +460,10 @@ class UnpackRequest(unittest.TestCase): - Messages that aren't for me (that I can't decrypt) are ignored. - Messages with invalid signatures are rejected. - Only passing messages return the dictionary. + - Each message identifies the Santiago protocol version it uses. + - Messages come with a range of Santiago protocol versions I can reply with. + - Messages that don't share any of my versions are ignored (either the + client or I won't be able to understand the message). - The message is unpacked correctly. This is a bit difficult because of the number of overlapping data types. @@ -574,12 +520,13 @@ class UnpackRequest(unittest.TestCase): "locations": [1], "request_version": 1, "reply_versions": [1], } - self.ALL_KEYS = ("host", "client", "service", "locations", "reply_to", - "request_version", "reply_versions") - self.REQUIRED_KEYS = ("client", "host", "service", - "request_version", "reply_versions") - self.OPTIONAL_KEYS = ("locations", "reply_to") - self.LIST_KEYS = ("reply_to", "locations", "reply_versions") + self.ALL_KEYS = set(("host", "client", "service", + "locations", "reply_to", + "request_version", "reply_versions")) + self.REQUIRED_KEYS = set(("client", "host", "service", + "request_version", "reply_versions")) + self.OPTIONAL_KEYS = set(("locations", "reply_to")) + self.LIST_KEYS = set(("reply_to", "locations", "reply_versions")) def test_valid_message(self): """A message that should pass does pass normally.""" @@ -704,6 +651,28 @@ class UnpackRequest(unittest.TestCase): for attribute in ("union", "intersection"): self.assertTrue(hasattr(unpacked[key], attribute)) + def test_require_protocol_version_overlap(self): + """Clients that can't accept protocols I can send are ignored.""" + + santiago.Santiago.SUPPORTED_PROTOCOLS, unsupported = \ + set(["e"]), santiago.Santiago.SUPPORTED_PROTOCOLS + + self.request = self.wrap_message(str(self.request)) + + self.assertFalse(self.santiago.unpack_request(self.request)) + + santiago.Santiago.SUPPORTED_PROTOCOLS, unsupported = \ + unsupported, santiago.Santiago.SUPPORTED_PROTOCOLS + + def test_require_protocol_version_understanding(self): + """The service must ignore any protocol versions it can't understand.""" + + self.request["request_version"] = "e" + + self.request = self.wrap_message(str(self.request)) + + self.assertFalse(self.santiago.unpack_request(self.request)) + class HandleRequest(unittest.TestCase): """Process an incoming request, from a client, for to host services. @@ -762,23 +731,20 @@ class HandleRequest(unittest.TestCase): self.assertTrue(self.santiago.requested) - def test_unwilling_client(self): - """Don't handle the request if the cilent isn't trusted.""" + def test_unwilling_source(self): + """Don't handle the request if the cilent or proxy isn't trusted. - self.client = 0 + Ok, so, "isn't trusted" is the wrong turn of phrase here. Technically, + it's "this Santiago isn't willing to host services for", but the + former's much easier to type. - self.test_call() + """ + for key in ("client", ): + setattr(self, key, 0) - self.assertFalse(self.santiago.requested) + self.test_call() - def test_unwilling_proxy(self): - """Don't handle the request if the proxy isn't trusted.""" - - self.from_ = 0 - - self.test_call() - - self.assertFalse(self.santiago.requested) + self.assertFalse(self.santiago.requested) def test_learn_services(self): """New reply_to locations are learned.""" @@ -791,7 +757,152 @@ class HandleRequest(unittest.TestCase): self.assertEqual(self.santiago.consuming["santiago"][self.keyid], set([1, 2])) +# class HandleReply(unittest.TestCase): +# """ +# def handle_reply(self, from_, to, host, client, +# service, locations, reply_to): +# "Process a reply from a Santiago service. + +# The last call in the chain that makes up the Santiago system, we now +# take the reply from the other Santiago server and learn any new service +# locations, if we've requested locations for that service." + +# """ +# def test_valid_message(self): +# """A valid message should teach new service locations.""" + +# self.fail() + +# def test_no_request_to_host(self): +# """If I haven't asked the host for any services, ignore the reply.""" + +# self.fail() + +# def test_no_request_for_service(self): +# """If I haven't asked the host for this service, ignore the reply.""" + +# self.fail() + +# def test_not_to_me(self): +# """Ignore messages to another Santiago service. + +# if not self.i_am(to): + +# """ + +# self.fail() + +# def test_for_other_client(self): +# """Ignore messages that another Santiago is the client for. + +# if not self.i_am(client): + +# """ + +# self.fail() + +# def test_learn_santiago_locations(self): +# """New Santiago locations are learned.""" + +# self.fail() + +# def test_learn_service_locations(self): +# """New service locations are learned.""" + +# self.fail() + +# def test_dequeue_service_request(self): +# """Don't accept further service requests after the request is handled. + +# Of course, this has its limits. Multiple requests to the same host +# would create multiple outstanding requests. Should they? Think on that. + +# """ +# self.fail() + +class OutgoingRequest(unittest.TestCase): + """Are outgoing requests properly formed? + + Here, we'll use a faux Santiago Sender that merely records and decodes the + request when it goes out. + + """ + class TestRequestSender(object): + """A barebones sender that records details about the request.""" + + def __init__(self): + self.gpg = gnupg.GPG(use_agent = True) + + def outgoing_request(self, request, destination): + """Decrypt and record the pertinent details about the request.""" + + self.destination = destination + self.crypt = request + self.request = ast.literal_eval(str(self.gpg.decrypt(str(request)))) + + def setUp(self): + """Create an encryptable request.""" + + self.keyid = utilities.load_config().get("pgpprocessor", "keyid") + + self.santiago = santiago.Santiago( + me = self.keyid, + consuming = { "santiago": { self.keyid: ( "https://1", )}}) + + self.request_sender = OutgoingRequest.TestRequestSender() + self.santiago.senders = { "https": self.request_sender } + + self.host = self.keyid + self.client = self.keyid + self.service = "santiago" + self.reply_to = [ "https://1" ] + self.locations = [1] + self.request_version = 1 + self.reply_versions = [1] + + self.request = { + "host": self.host, "client": self.client, + "service": self.service, + "reply_to": self.reply_to, "locations": self.locations, + "request_version": self.request_version, + "reply_versions": self.reply_versions } + + def outgoing_call(self): + """A short-hand for calling outgoing_request with all 8 arguments.""" + + self.santiago.outgoing_request( + None, None, self.host, self.client, + self.service, self.locations, self.reply_to) + + def test_valid_message(self): + """Are valid messages properly encrypted and delivered?""" + + self.outgoing_call() + + self.assertEqual(self.request_sender.request, + self.request) + self.assertEqual(self.request_sender.destination, self.reply_to[0]) + + def test_queue_service_request(self): + """Add the host's service to the request queue.""" + + self.outgoing_call() + + self.assertTrue(self.service in self.santiago.requests[self.host]) + + def test_transparent_unwrapping(self): + """Is the unwrapping process transparent?""" + + import urlparse, urllib + + self.outgoing_call() + + request = {"request": str(self.request_sender.crypt) } + + self.assertEqual(request["request"], + urlparse.parse_qs(urllib.urlencode(request))["request"][0]) + if __name__ == "__main__": logging.disable(logging.CRITICAL) unittest.main()