nik/FreedomBox
1
0
Fork 0
mirror of https://github.com/freedombox/FreedomBox.git synced 2026-06-10 11:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Almost done.

Browse Source
This commit is contained in:
Nick Daly 2012-05-14 22:21:12 -05:00
parent 29a1b41996
commit 328a720d72
3 changed files with 301 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
ugly_hacks/santiago/protocols/https.py
View File

@ -3,9 +3,9 @@
from simplesantiago import SantiagoListener, SantiagoSender
import cherrypy
import httplib, urllib
import httplib, urllib, urlparse
import sys
import logging
class Listener(SantiagoListener):
@ -31,10 +31,11 @@ class Listener(SantiagoListener):
def index(self, **kwargs):
"""Receive an incoming Santiago request from another Santiago client."""
logging.debug("protocols.https.index: Received request {0}".format(str(kwargs)))
try:
self.incoming_request(kwargs["request"])
except:
pass
except Exception as e:
logging.exception(e)
@cherrypy.expose
def query(self, host, service):
@ -44,6 +45,7 @@ class Listener(SantiagoListener):
"""
if not cherrypy.request.remote.ip.startswith("127.0.0"):
logging.debug("protocols.https.query: Request from non-local IP")
return
self.santiago.query(host, service)
@ -51,6 +53,7 @@ class Listener(SantiagoListener):
@cherrypy.expose
def save_server(self):
if not cherrypy.request.remote.ip.startswith("127.0.0"):
logging.debug("protocols.https.save_server: Request from non-local IP")
return
self.santiago.save_server()
@ -70,8 +73,15 @@ class Sender(SantiagoSender):
It's both simple and as reliable as possible.
``request`` is literally the request's text. It needs to be wrapped for
transport across the protocol.
"""
params = urllib.urlencode(request)
logging.debug("protocols.https.Sender.outgoing_request: request {0}".format(str(request)))
to_send = { "request": request }
params = urllib.urlencode(to_send)
logging.debug("protocols.https.Sender.outgoing_request: params {0}".format(str(params)))
# TODO: Does HTTPSConnection require the cert and key?
# Is the fact that the server has it sufficient? I think so.

163
ugly_hacks/santiago/simplesantiago.py
View File

@ -26,12 +26,10 @@ We also don't:
- Use a reasonable data-store.
- Have a decent control mechanism.
:FIXME: add that whole pgp thing.
:TODO: add doctests
:TODO: Create startup script that adds all necessary things to the PYTHONPATH.
:TODO: move to santiago.py, merge the documentation.
:FIXME: allow multiple listeners and senders per protocol (with different
proxies)
:TODO: move to santiago.py, merge the documentation.
This dead-drop is what came of my trying to learn from bug 4185.
@ -66,12 +64,12 @@ class Santiago(object):
"""
SUPPORTED_PROTOCOLS = set([1])
ALL_KEYS = ("host", "client", "service", "locations", "reply_to",
"request_version", "reply_versions")
REQUIRED_KEYS = ("client", "host", "service",
"request_version", "reply_versions")
OPTIONAL_KEYS = ("locations", "reply_to")
LIST_KEYS = ("reply_to", "locations", "reply_versions")
ALL_KEYS = set(("host", "client", "service", "locations", "reply_to",
"request_version", "reply_versions"))
REQUIRED_KEYS = set(("client", "host", "service",
"request_version", "reply_versions"))
OPTIONAL_KEYS = ALL_KEYS ^ REQUIRED_KEYS
LIST_KEYS = set(("reply_to", "locations", "reply_versions"))
def __init__(self, listeners = None, senders = None,
hosting = None, consuming = None, me = 0):
@ -121,7 +119,7 @@ class Santiago(object):
connectors = dict()
for protocol in settings.iterkeys():
module = SimpleSantiago._get_protocol_module(protocol)
module = Santiago._get_protocol_module(protocol)
try:
connectors[protocol] = \
@ -154,8 +152,8 @@ class Santiago(object):
When this has finished, the Santiago will be ready to go.
"""
for connector in list(self.listeners.itervalues()) + \
list(self.senders.itervalues()):
for connector in (list(self.listeners.itervalues()) +
list(self.senders.itervalues())):
connector.start()
logging.debug("Santiago started!")
@ -197,6 +195,7 @@ class Santiago(object):
except KeyError as e:
logging.exception(e)
def query(self, host, service):
"""Request a service from another Santiago.
@ -204,8 +203,6 @@ class Santiago(object):
"""
try:
self.requests[host].add(service)
self.outgoing_request(
host, self.me, host, self.me,
service, None, self.get_client_locations(host, "santiago"))
@ -218,15 +215,25 @@ class Santiago(object):
This tag is used when sending queries or replies to other Santiagi.
"""
# FIXME sign the encrypted payload.
payload = self.gpg.encrypt(
{"host": host, "client": client,
"service": service, "locations": locations or "",
"reply_to": reply_to}, to, sign=self.me)
request = self.gpg.sign({"request": payload, "to": to})
Each incoming item must be a single item or a list.
for destination in self.get_client_locations(to, "santiago"):
The outgoing ``request`` is literally the request's text. It needs to
be wrapped for transport across the protocol.
"""
self.requests[host].add(service)
request = self.gpg.encrypt(
str({ "host": host, "client": client,
"service": service, "locations": list(locations or ""),
"reply_to": list(reply_to),
"request_version": 1,
"reply_versions": list(Santiago.SUPPORTED_PROTOCOLS),}),
host,
sign=self.me)
# FIXME use urlparse.urlparse instead!
for destination in self.get_client_locations(host, "santiago"):
protocol = destination.split(":")[0]
self.senders[protocol].outgoing_request(request, destination)
@ -250,14 +257,16 @@ class Santiago(object):
"""
# no matter what happens, the sender will never hear about it.
try:
if not verify_message(request):
return
logging.debug("santiago.Santiago.incoming_request: request: {0}".format(str(request)))
unpacked = self.unpack_request(request)
if not unpacked:
logging.debug("santiago.Santiago.incoming_request: opaque request.")
return
logging.debug("santiago.Santiago.incoming_request: unpacked {0}".format(str(unpacked)))
if unpacked["locations"]:
self.handle_reply(
unpacked["from"], unpacked["to"],
@ -277,27 +286,6 @@ class Santiago(object):
except Exception as e:
logging.exception("Error: ", str(e))
def verify_request(self, request):
"""Make sure the request meets minimum criteria before we process it.
- The request must contain required keys.
- The request and client must be of and support protocol versions I
understand.
"""
if False in map(request.__contains__,
("request", "request_version", "reply_versions")):
return False
if not (Santiago.SUPPORTED_PROTOCOLS & set(request["reply_versions"])):
return False
if not (Santiago.SUPPORTED_PROTOCOLS &
set([request["request_version"]])):
return False
return True
def unpack_request(self, request):
"""Decrypt and verify the request.
@ -307,11 +295,20 @@ class Santiago(object):
Some lists are changed to sets here. This allows for set-operations
(union, intersection, etc) later, making things much more intuitive.
The request and client must be of and support protocol versions I
understand.
"""
request = self.gpg.decrypt(request)
# skip badly signed messages or ones for other folks.
if not (str(request) and request.fingerprint):
logging.debug(
"santiago.Santiago.unpack_request: fail request {0}".format(
str(request)))
logging.debug(
"santiago.Santiago.unpack_request: fail fingerprint {0}".format(
str(request.fingerprint)))
return
# copy out only required keys from request, throwing away cruft
@ -321,15 +318,31 @@ class Santiago(object):
for key in Santiago.ALL_KEYS:
request_body[key] = source[key]
except KeyError:
logging.debug(
"santiago.Santiago.unpack_request: missing key {0}".format(
str(source)))
return
# required keys are non-null
if None in [request_body[x] for x in Santiago.REQUIRED_KEYS]:
logging.debug(
"santiago.Santiago.unpack_request: blank key {0}: {1}".format(
key, str(request_body)))
return
# move lists to sets
request_body = self.setify_lists(request_body)
if not request_body:
logging.debug(
"santiago.Santiago.unpack_request: not sets {0}".format(
str(request_body)))
return
# versions must overlap.
if not (Santiago.SUPPORTED_PROTOCOLS & request_body["reply_versions"]):
return
if not (Santiago.SUPPORTED_PROTOCOLS &
set([request_body["request_version"]])):
return
# set implied keys
@ -342,7 +355,7 @@ class Santiago(object):
"""Convert list nodes to sets."""
try:
for key in ("locations", "reply_to"):
for key in Santiago.LIST_KEYS:
if request_body[key] is not None:
request_body[key] = set(request_body[key])
except TypeError:
@ -368,8 +381,13 @@ class Santiago(object):
- Reply to the client on the appropriate protocol.
"""
# return if we won't host for the proxy or the client.
if False in map(self.hosting.__contains__, (from_, client)):
# give up if we won't host the service for the client.
try:
self.hosting[client][service]
except KeyError:
logging.debug(
"santiago.Santiago.handle_request: no host for you".format(
self.hosting))
return
# if we don't proxy, learn new reply locations and send the request.
@ -404,24 +422,36 @@ class Santiago(object):
locations, if we've requested locations for that service.
"""
logging.debug("santiago.Santiago.handle_reply: local {0}".format(str(locals())))
# give up if we won't consume the service from the proxy or the client.
try:
self.consuming[service][from_]
self.consuming[service][host]
except KeyError as e:
if service not in self.requests[host]:
logging.debug(
"santiago.Santiago.handle_reply: unrequested service {0}: ".format(
service, self.requests))
return
except KeyError:
logging.debug(
"santiago.Santiago.handle_reply: unrequested host {0}: ".format(
host, self.requests))
return
# give up or proxy if the message isn't for me.
if not self.i_am(to):
logging.debug(
"santiago.Santiago.handle_reply: not to {0}".format(to))
return
if not self.i_am(client):
logging.debug(
"santiago.Santiago.handle_reply: not client {0}".format(client))
self.proxy()
return
self.learn_service(host, "santiago", reply_to)
self.learn_service(host, service, locations)
if service in self.requests[host]:
self.learn_service(host, service, locations)
self.requests[host].remove(service)
self.requests[host].remove(service)
def save_server(self):
"""Save all operational data to files.
@ -429,14 +459,14 @@ class Santiago(object):
Save all files with the ``self.me`` prefix.
"""
for datum in ("hosting", "consuming"):
name = "%s_%s" % (self.me, datum)
for key in ("hosting", "consuming"):
name = "%s_%s" % (self.me, key)
try:
with open(name, "w") as output:
output.write(str(getattr(self, datum)))
output.write(str(getattr(self, key)))
except Exception as e:
logging.exception("Could not save %s as %s", datum, name)
logging.exception("Could not save %s as %s", key, name)
class SantiagoConnector(object):
"""Generic Santiago connector superclass.
@ -463,8 +493,8 @@ class SantiagoListener(SantiagoConnector):
method passes the request along to the Santiago host.
"""
def incoming_request(self, **kwargs):
self.santiago.incoming_request(**kwargs)
def incoming_request(self, request):
self.santiago.incoming_request(request)
class SantiagoSender(SantiagoConnector):
"""Generic Santiago Sender superclass.
@ -479,8 +509,8 @@ class SantiagoSender(SantiagoConnector):
if __name__ == "__main__":
# FIXME: convert this to the withsqlite setup.
logging.getLogger().setLevel(logging.DEBUG)
logging.raiseExceptions = False
cert = "santiago.crt"
listeners = { "https": { "socket_port": 8080,
"ssl_certificate": cert,
@ -488,23 +518,24 @@ if __name__ == "__main__":
senders = { "https": { "proxy_host": "localhost",
"proxy_port": 8118} }
mykey = "D95C32042EE54FFDB25EC3489F2733F40928D23A"
# mykey = "0928D23A" # my short key
# load hosting
try:
hosting = load_data(mykey, "hosting")
except IOError:
hosting = { "a": { "santiago": set( ["https://localhost:8080"] )},
"b": { "santiago": set( ["https://localhost:8080"] )},
mykey: { "santiago": set( ["https://localhost:8080"] )}}
# load consuming
try:
consuming = load_data(mykey, "consuming")
except IOError:
consuming = { "santiago": { mykey: set( ["https://localhost:8080"] ),
"b": set( ["https://localhost:8080"] ),
"a": set( ["someAddress.onion"] )}}
# load the Santiago
santiago_b = SimpleSantiago(listeners, senders,
hosting, consuming, mykey)
santiago_b = Santiago(listeners, senders,
hosting, consuming, mykey)
santiago_b.start()

267
ugly_hacks/santiago/test_santiago.py
View File

@ -43,6 +43,7 @@ import os
import sys
import unittest
import ast
import gnupg
import logging
import simplesantiago as santiago
@ -452,65 +453,6 @@ import utilities
#
# pass
class VerifyRequest(unittest.TestCase):
"""Are incoming requests handled correctly?
- Messages come with a request.
- Each message identifies the Santiago protocol version it uses.
- Messages come with a range of Santiago protocol versions I can reply with.
- Messages that don't share any of my versions are ignored (either the
client or I won't be able to understand the message).
Test this in a fairly hacky way.
"""
def setUp(self):
self.santiago = santiago.Santiago()
self.request = { "request_version": 1,
"reply_versions": [1],
"request": None }
def test_valid_message(self):
"""A known good request passes."""
self.assertTrue(self.santiago.verify_request(self.request))
def test_required_keys_are_required(self):
"""Messages without required keys fail.
The following keys are required in the un-encrypted part of the message:
- request
- request_version
- reply_versions
"""
for key in ("request", "request_version", "reply_versions"):
del self.request[key]
self.assertFalse(self.santiago.verify_request(self.request))
def test_require_protocol_version_overlap(self):
"""Clients that can't accept protocols I can send are ignored."""
santiago.Santiago.SUPPORTED_PROTOCOLS, unsupported = \
set(["e"]), santiago.Santiago.SUPPORTED_PROTOCOLS
self.assertFalse(self.santiago.verify_request(self.request))
santiago.Santiago.SUPPORTED_PROTOCOLS, unsupported = \
unsupported, santiago.Santiago.SUPPORTED_PROTOCOLS
def test_require_protocol_version_understanding(self):
"""I must ignore any protocol versions I can't understand."""
self.request["request_version"] = "e"
self.assertFalse(self.santiago.verify_request(self.request))
class UnpackRequest(unittest.TestCase):
"""Are requests unpacked as expected?
@ -518,6 +460,10 @@ class UnpackRequest(unittest.TestCase):
- Messages that aren't for me (that I can't decrypt) are ignored.
- Messages with invalid signatures are rejected.
- Only passing messages return the dictionary.
- Each message identifies the Santiago protocol version it uses.
- Messages come with a range of Santiago protocol versions I can reply with.
- Messages that don't share any of my versions are ignored (either the
client or I won't be able to understand the message).
- The message is unpacked correctly. This is a bit difficult because of the
number of overlapping data types.
@ -574,12 +520,13 @@ class UnpackRequest(unittest.TestCase):
"locations": [1],
"request_version": 1, "reply_versions": [1], }
self.ALL_KEYS = ("host", "client", "service", "locations", "reply_to",
"request_version", "reply_versions")
self.REQUIRED_KEYS = ("client", "host", "service",
"request_version", "reply_versions")
self.OPTIONAL_KEYS = ("locations", "reply_to")
self.LIST_KEYS = ("reply_to", "locations", "reply_versions")
self.ALL_KEYS = set(("host", "client", "service",
"locations", "reply_to",
"request_version", "reply_versions"))
self.REQUIRED_KEYS = set(("client", "host", "service",
"request_version", "reply_versions"))
self.OPTIONAL_KEYS = set(("locations", "reply_to"))
self.LIST_KEYS = set(("reply_to", "locations", "reply_versions"))
def test_valid_message(self):
"""A message that should pass does pass normally."""
@ -704,6 +651,28 @@ class UnpackRequest(unittest.TestCase):
for attribute in ("union", "intersection"):
self.assertTrue(hasattr(unpacked[key], attribute))
def test_require_protocol_version_overlap(self):
"""Clients that can't accept protocols I can send are ignored."""
santiago.Santiago.SUPPORTED_PROTOCOLS, unsupported = \
set(["e"]), santiago.Santiago.SUPPORTED_PROTOCOLS
self.request = self.wrap_message(str(self.request))
self.assertFalse(self.santiago.unpack_request(self.request))
santiago.Santiago.SUPPORTED_PROTOCOLS, unsupported = \
unsupported, santiago.Santiago.SUPPORTED_PROTOCOLS
def test_require_protocol_version_understanding(self):
"""The service must ignore any protocol versions it can't understand."""
self.request["request_version"] = "e"
self.request = self.wrap_message(str(self.request))
self.assertFalse(self.santiago.unpack_request(self.request))
class HandleRequest(unittest.TestCase):
"""Process an incoming request, from a client, for to host services.
@ -762,23 +731,20 @@ class HandleRequest(unittest.TestCase):
self.assertTrue(self.santiago.requested)
def test_unwilling_client(self):
"""Don't handle the request if the cilent isn't trusted."""
def test_unwilling_source(self):
"""Don't handle the request if the cilent or proxy isn't trusted.
self.client = 0
Ok, so, "isn't trusted" is the wrong turn of phrase here. Technically,
it's "this Santiago isn't willing to host services for", but the
former's much easier to type.
self.test_call()
"""
for key in ("client", ):
setattr(self, key, 0)
self.assertFalse(self.santiago.requested)
self.test_call()
def test_unwilling_proxy(self):
"""Don't handle the request if the proxy isn't trusted."""
self.from_ = 0
self.test_call()
self.assertFalse(self.santiago.requested)
self.assertFalse(self.santiago.requested)
def test_learn_services(self):
"""New reply_to locations are learned."""
@ -791,7 +757,152 @@ class HandleRequest(unittest.TestCase):
self.assertEqual(self.santiago.consuming["santiago"][self.keyid],
set([1, 2]))
# class HandleReply(unittest.TestCase):
# """
# def handle_reply(self, from_, to, host, client,
# service, locations, reply_to):
# "Process a reply from a Santiago service.
# The last call in the chain that makes up the Santiago system, we now
# take the reply from the other Santiago server and learn any new service
# locations, if we've requested locations for that service."
# """
# def test_valid_message(self):
# """A valid message should teach new service locations."""
# self.fail()
# def test_no_request_to_host(self):
# """If I haven't asked the host for any services, ignore the reply."""
# self.fail()
# def test_no_request_for_service(self):
# """If I haven't asked the host for this service, ignore the reply."""
# self.fail()
# def test_not_to_me(self):
# """Ignore messages to another Santiago service.
# if not self.i_am(to):
# """
# self.fail()
# def test_for_other_client(self):
# """Ignore messages that another Santiago is the client for.
# if not self.i_am(client):
# """
# self.fail()
# def test_learn_santiago_locations(self):
# """New Santiago locations are learned."""
# self.fail()
# def test_learn_service_locations(self):
# """New service locations are learned."""
# self.fail()
# def test_dequeue_service_request(self):
# """Don't accept further service requests after the request is handled.
# Of course, this has its limits. Multiple requests to the same host
# would create multiple outstanding requests. Should they? Think on that.
# """
# self.fail()
class OutgoingRequest(unittest.TestCase):
"""Are outgoing requests properly formed?
Here, we'll use a faux Santiago Sender that merely records and decodes the
request when it goes out.
"""
class TestRequestSender(object):
"""A barebones sender that records details about the request."""
def __init__(self):
self.gpg = gnupg.GPG(use_agent = True)
def outgoing_request(self, request, destination):
"""Decrypt and record the pertinent details about the request."""
self.destination = destination
self.crypt = request
self.request = ast.literal_eval(str(self.gpg.decrypt(str(request))))
def setUp(self):
"""Create an encryptable request."""
self.keyid = utilities.load_config().get("pgpprocessor", "keyid")
self.santiago = santiago.Santiago(
me = self.keyid,
consuming = { "santiago": { self.keyid: ( "https://1", )}})
self.request_sender = OutgoingRequest.TestRequestSender()
self.santiago.senders = { "https": self.request_sender }
self.host = self.keyid
self.client = self.keyid
self.service = "santiago"
self.reply_to = [ "https://1" ]
self.locations = [1]
self.request_version = 1
self.reply_versions = [1]
self.request = {
"host": self.host, "client": self.client,
"service": self.service,
"reply_to": self.reply_to, "locations": self.locations,
"request_version": self.request_version,
"reply_versions": self.reply_versions }
def outgoing_call(self):
"""A short-hand for calling outgoing_request with all 8 arguments."""
self.santiago.outgoing_request(
None, None, self.host, self.client,
self.service, self.locations, self.reply_to)
def test_valid_message(self):
"""Are valid messages properly encrypted and delivered?"""
self.outgoing_call()
self.assertEqual(self.request_sender.request,
self.request)
self.assertEqual(self.request_sender.destination, self.reply_to[0])
def test_queue_service_request(self):
"""Add the host's service to the request queue."""
self.outgoing_call()
self.assertTrue(self.service in self.santiago.requests[self.host])
def test_transparent_unwrapping(self):
"""Is the unwrapping process transparent?"""
import urlparse, urllib
self.outgoing_call()
request = {"request": str(self.request_sender.crypt) }
self.assertEqual(request["request"],
urlparse.parse_qs(urllib.urlencode(request))["request"][0])
if __name__ == "__main__":
logging.disable(logging.CRITICAL)
unittest.main()