# Copyright 2002-2004 Nick Mathewson.  See LICENSE for licensing information.
# $Id: test.py,v 1.220 2005/12/02 19:56:35 nickm Exp $

"""mixminion.tests

   Unit tests for all Mixminion functionality.

   Usage:
   >>> import mixminion.tests
   >>> mixminion.tests.testAll()
   """

__pychecker__ = 'no-funcdoc maxlocals=100'

import base64
import cPickle
import cStringIO
import gzip
import operator
import os
import re
import socket
import stat
import struct
import sys
import threading
import time
import types
from string import atoi

# Not every post-2.0 version of Python has a working 'unittest' module, so
# we include a copy with mixminion, as 'mixminion._unittest'.
try:
    import unittest
except ImportError:
    import mixminion._unittest as unittest

# If the environment variable MM_COVERAGE is set, then start running the
# coverage analysis tool.  You'll need to install the file 'coverage.py':
# it's included in the contrib directory.
if os.environ.get("MM_COVERAGE"):
    import coverage
    coverage.erase()
    coverage.start()

import mixminion.testSupport
from mixminion.testSupport import mix_mktemp, suspendLog, resumeLog, \
     replaceAttribute, undoReplacedAttributes, replaceFunction, \
     getReplacedFunctionCallLog, clearReplacedFunctionCallLog

import mixminion.BuildMessage as BuildMessage
import mixminion.ClientMain
import mixminion.ClientUtils
import mixminion.Config
import mixminion.Crypto as Crypto
import mixminion.Filestore
import mixminion.Fragments
import mixminion.MMTPClient
import mixminion.NetUtils
import mixminion.Packet
import mixminion.ServerInfo
import mixminion.ThreadUtils
import mixminion.TLSConnection
import mixminion._minionlib as _ml
import mixminion.server.MMTPServer
import mixminion.server.Modules
import mixminion.server.Pinger
import mixminion.server.ServerConfig
import mixminion.server.ServerKeys
import mixminion.server.ServerMain
import mixminion.directory.ServerList
import mixminion.directory.ServerInbox
import mixminion.directory.DirFormats
import mixminion.directory.DirMain
import mixminion.directory.Directory
from mixminion.Common import *
from mixminion.Common import Log, _FileLogHandler, _ConsoleLogHandler
from mixminion.Config import _ConfigFile, ConfigError, _parseInt
from mixminion.Crypto import *
from mixminion.Packet import *
from mixminion.server.HashLog import HashLog
from mixminion.server.Modules import *
from mixminion.server.PacketHandler import *
from mixminion.server.ServerQueue import *
from mixminion.server.ServerKeys import generateServerDescriptorAndKeys, \
     generateCertChain

# Set this flag to 1 in order to have all RSA keys and diffie-hellman params
# generated independently.  Otherwise, we cache diffie-hellman parameters
# on disk, and only generate a small number of RSA keys.
USE_SLOW_MODE = 0

# Flag: are we on a platform with windowsy permissions?
ON_WINDOWS = (sys.platform in ('cygwin', 'win32'))

# Flag: are we on win32?
ON_WIN32 = (sys.platform == 'win32')


#----------------------------------------------------------------------
# Misc helper functions
def hexread(s):
    """Helper function.  Converts a hexidecimal string into a binary string.
       For example, hexread('0A0B') == '\x0A\x0B'."""
    assert (len(s) % 2) == 0
    s = s.upper()
    r = []
    hexvals = "0123456789ABCDEF"
    for i in range(len(s) / 2):
        v1 = hexvals.index(s[i*2])
        v2 = hexvals.index(s[i*2+1])
        c = (v1 << 4) + v2
        assert 0 <= c < 256
        r.append(chr(c))
    return "".join(r)

def findFirstDiff(s1, s2):
    """Helper function.  Returns the first index i for which s1[i] != s2[i],
       or -1 s1 == s2."""
    if s1 == s2:
        return -1
    last = min(len(s1), len(s2))
    for i in xrange(last):
        if s1[i] != s2[i]:
            return i
    return last

def floatEq(f1,f2,tolerance=.00001):
    """Return true iff f1 is very close to f2."""
    if min(f1, f2) != 0:
        return abs(f1-f2)/min(f1,f2) < tolerance
    else:
        return abs(f1-f2) < .00001

def fileURL(fname):
    """Return a file url suitable for a file named 'fname'"""
    fname = os.path.abspath(fname)
    return "file:%s"%fname

#----------------------------------------------------------------------
# DNS override
def overrideDNS(overrideDict,delay=0):
    """Helper function: temporarily replace the DNS functionality in
       NetUtils to lookup entries in overrideDict instead of on the
       network, and pause for 'delay' seconds before returning an answer.
    """
    def getIPs_replacement(addr,overrideDict=overrideDict,delay=delay):
        v = overrideDict.get(addr)
        if delay: time.sleep(delay)
        if v is None:
            raise socket.error, "not there"
        elif '.' in v:
            return [ (mixminion.NetUtils.AF_INET, v, time.time()) ]
        else:
            return [ (mixminion.NetUtils.AF_INET6, v, time.time()) ]

    replaceAttribute(mixminion.NetUtils, "getIPs", getIPs_replacement)

#----------------------------------------------------------------------
# RSA key caching functionality

# Map from (n, bits) to a RSA key with bits bits.  Used to cache RSA keys
# for different purposes.
_generated_rsa_keys = {}
_generated_rsa_keys[(0,2048)] = mixminion.testSupport.TEST_KEYS_2048[0]
_generated_rsa_keys[(1,2048)] = mixminion.testSupport.TEST_KEYS_2048[1]
_generated_rsa_keys[(2,2048)] = mixminion.testSupport.TEST_KEYS_2048[2]
_generated_rsa_keys[(3,2048)] = mixminion.testSupport.TEST_KEYS_2048[3]
_generated_rsa_keys[(4,2048)] = mixminion.testSupport.TEST_KEYS_2048[4]
_generated_rsa_keys[(5,2048)] = mixminion.testSupport.TEST_KEYS_2048[5]
def getRSAKey(n,bits):
    """Return the n'th of an arbitrary number of cached 'bits'-bit RSA keys,
       generating them as necessary."""
    try:
        return _generated_rsa_keys[(n,bits)]
    except KeyError:
        if bits > 1024:
            print "[generating %d-bit key #%d..."%(bits,n),
            sys.stdout.flush()
        k = _pk_generate_orig(bits)
        if bits > 1024:
            print "done]",
            sys.stdout.flush()
        _generated_rsa_keys[(n,bits)] = k
        return k

# Functions to override Crypto.pk_generate to avoid generating a zillion RSA
# keys.
_pk_generate_orig = Crypto.pk_generate
_pk_generate_idx = 0
def _pk_generate_replacement(bits=1024,e=65537):
    if bits == 2048:
        global _pk_generate_idx
        _pk_generate_idx = (_pk_generate_idx + 1) % 6
        return getRSAKey(_pk_generate_idx, bits)
    else:
        return getRSAKey(0, bits)

if not USE_SLOW_MODE:
    Crypto.pk_generate = _pk_generate_replacement

#----------------------------------------------------------------------
# Add some helpful functions to unittest.TestCase

class TestCase(unittest.TestCase):
    """Extended version of unittest.TestCase with a few extra
       'assertFoo' functions."""
    def __init__(self, *args, **kwargs):
        unittest.TestCase.__init__(self, *args, **kwargs)
    def assertFloatEq(self, f1, f2, tolerance=.00001):
        """Fail unless f1 and f2 are very close to one another."""
        if not floatEq(f1, f2, tolerance):
            self.fail("%s != %s" % (f1, f2))
    def assertLongStringEq(self, s1, s2):
        """Fail unless the string s1 equals the string s2.  If they aren't
           equal, give a failure message that highlights the first point at
           which they differ."""
        if s1 != s2:
            d = findFirstDiff(s1, s2)
            self.fail("Strings unequal.  First difference at %s: %r vs %r"
                      % (d, s1[d:d+10], s2[d:d+10]))
    def assertUnorderedEq(self, l1, l2):
        """Fail unless the lists l1 and l2 have the same elements.  The
           order of elements in the two lists need not be the same."""
        l1 = list(l1)[:]
        l2 = list(l2)[:]
        l1.sort()
        l2.sort()
        self.assertEquals(l1, l2)
    def assertStartsWith(self, s1, s2):
        """Fail unless string s1 begins with string s2."""
        if not s1.startswith(s2):
            if len(s1) > min(40, len(s2)+5):
                s1 = s1[:len(s2)+5]+"..."
            self.fail("%r does not start with %r"%(s1,s2))
    def assertEndsWith(self, s1, s2):
        """Fail unless string s1 ends with string s2."""
        if not s1.endswith(s2):
            if len(s1) > min(40, len(s2)+5):
                s1 = "..."+s1[-(len(s2)+5):]
            self.fail("%r does not end with %r"%(s1,s2))
    def assertIn(self, item, lst):
        """Fail unless 'item in lst'"""
        if not (item in lst):
            self.fail("%r not in %r"%(item,lst))
    def assertNotIn(self, item, lst):
        """Fail unless 'item not in lst'"""
        if (item in lst):
            self.fail("%r in %r"%(item,lst))
    def assertIEquals(self, i1, i2):
        self.assertEquals(long(i1), long(i2))

#----------------------------------------------------------------------
# Tests for common functionality

class MiscTests(TestCase):
    def testDiv(self):
        self.assertEquals(floorDiv(10,1), 10)
        self.assertEquals(floorDiv(10,2), 5)
        self.assertEquals(floorDiv(10,3), 3)
        self.assertEquals(floorDiv(10,11), 0)
        self.assertEquals(floorDiv(0,11), 0)
        self.assertEquals(floorDiv(-1,1), -1)
        self.assertEquals(floorDiv(-1,2), -1)
        self.assertEquals(floorDiv(-10,3), -4)
        self.assertEquals(floorDiv(-10,-3), 3)

        self.assertEquals(ceilDiv(10,1), 10)
        self.assertEquals(ceilDiv(10,2), 5)
        self.assertEquals(ceilDiv(10,3), 4)
        self.assertEquals(ceilDiv(10,11), 1)
        self.assertEquals(ceilDiv(0,11), 0)
        self.assertEquals(ceilDiv(-1,1), -1)
        self.assertEquals(ceilDiv(-1,2), 0)
        self.assertEquals(ceilDiv(-10,3), -3)
        self.assertEquals(ceilDiv(-10,-3), 4)

    def testVersions(self):
        vstr = mixminion.version_tuple_to_string
        parse = mixminion.parse_version_string
        cmp_v = mixminion.cmp_versions
        last = None
        for t,s,c in [((1,0,0,100,-1),   "1.0.0",       None),
                      ((0,0,3,0,-1),     "0.0.3alpha",  '<'),
                      ((0,5,22,50,6),    "0.5.22beta6", '>'),
                      ((0,5,22,99,6),    "0.5.22rc6",   '>'),
                      ((0,5,22,99,6),    "0.5.22rc6",   '='),
                      ((0,0,3,100,9),    "0.0.3.9",     '<'),
                      ((0,0,3,66,9),     "0.0.3(66)9",  '<'),
                      ((0,0,3,66,-1),    "0.0.3(66)",   '<'),
                      ((0,0,3,66,0),     "0.0.3(66)0",  '>'),
                      ((0,0,3,'fred',-1),"0.0.3fred",   'EX'),
                      ((0,0,3,'fred',8), "0.0.3fred8",  '<'),
                      ((0,0,3,'fred',0), "0.0.3fred0",  '>'),
                      ((0,0,3,'code',0), "0.0.3code0",  "EX"),
                      ]:
            self.assertEquals(vstr(t), s)
            self.assertEquals(parse(s), t)
            if not last:
                continue

            if c == 'EX':
                self.assertRaises(ValueError, cmp, last, t)
            elif c == '<':
                self.assertEquals(cmp_v(last,t), -1)
            elif c == '=':
                self.assertEquals(cmp_v(last,t), 0)
            elif c == '>':
                self.assertEquals(cmp_v(last,t), 1)
            else:
                print "Huh?"

            last = t

    def testTimeFns(self):
        # This isn't a very good test.
        now = int(time.time())
        max_sec_per_day = 24*60*60+ 1
        # Sample 1000 evenly spaced points, making sure...
        for t in xrange(10, now, floorDiv(now, 1000)):
            yyyy,MM,dd,hh,mm,ss = time.gmtime(t)[:6]
            # 1. That previousMidnight returns the same day, at midnight.
            pm = previousMidnight(t)
            yyyy2,MM2,dd2,hh2,mm2,ss2 = time.gmtime(pm)[:6]
            self.assertEquals((yyyy2,MM2,dd2), (yyyy,MM,dd))
            self.assertEquals((0,0,0), (hh2,mm2,ss2))
            self.failUnless(pm <= t and 0 <= (t-pm) <= max_sec_per_day)
            # 2. That previousMidnight is repeatable
            self.assertEquals(previousMidnight(t), pm)
            # 3. That previousMidnight is idempotent
            self.assertEquals(previousMidnight(pm), pm)

            # 4. That succeedingMidnight returns the next day, at midnight.
            sm = succeedingMidnight(t)
            yyyy2,MM2,dd2,hh2,mm2,ss2 = time.gmtime(sm)[:6]
            self.assertEquals((0,0,0), (hh2, mm2, ss2))
            if dd2 == dd + 1:
                self.assertEquals((yyyy2, MM2), (yyyy, MM))
            elif MM2 == MM + 1:
                self.assertEquals((yyyy2, dd2), (yyyy, 1))
            else:
                self.assertEquals((yyyy2, MM2, dd2), (yyyy+1, 1, 1))
            # 5. That succeedingMidnight is repeatable
            self.assertEquals(succeedingMidnight(t), sm)
            # 6. That sm(pm(x)) = sm(x)
            self.assertEquals(succeedingMidnight(previousMidnight(t)),
                              succeedingMidnight(t))
            # 7. That pm(sm(x)) = sm(x)
            self.assertEquals(previousMidnight(succeedingMidnight(t)),
                              succeedingMidnight(t))

        now = time.time()
        ft = formatFnameTime(now)
        fd = formatFnameDate(now)
        tm = time.localtime(now)
        self.assertEquals(ft, "%04d%02d%02d%02d%02d%02d" % tm[:6])
        self.assertEquals(fd, "%04d%02d%02d" % tm[:3])

    def test_isSMTPMailbox(self):
        # Do we accept good addresses?
        for addr in ("Foo@bar.com", "a@b", "a@b.c.d.e", "a!b.c@d", "z@z",
                     "$@com"):
            self.assert_(isSMTPMailbox(addr))

        # Do we reject bad addresses?
        for addr in ("(foo)@bar.com", "z.d" "z@", "@z", "@foo.com", "aaa",
                     "foo.bar@", "foo\177@bar.com", "foo@bar\177.com",
                     "foo@bar;cat /etc/shadow;echo ","foo bar@baz.com",
                     "a@b@c", "foo@[127.0.0.1]", "foo@127.0.0.1", "foo@127"):
            self.assert_(not isSMTPMailbox(addr))

    def test_intervalset(self):
        eq = self.assertEquals
        nil = IntervalSet()
        nil2 = IntervalSet()
        nil._checkRep()
        self.assert_(nil.isEmpty())
        self.assert_(nil == nil2)
        eq(repr(nil), "IntervalSet([])")
        eq([], nil.getIntervals())
        nil3 = IntervalSet([(10, 0)])
        eq([], nil3.getIntervals())

        oneToTen = IntervalSet([(1,10)])
        fourToFive = IntervalSet([(4,5)])
        zeroToTen = IntervalSet([(0,10)])
        zeroToTwenty = IntervalSet([(0,20)])
        tenToTwenty = IntervalSet([(10,20)])
        oneToTwenty = IntervalSet([(1,20)])
        fifteenToFifty = IntervalSet([(15,50)])

        eq(zeroToTen.getIntervals(), [(0, 10)])
        for iset in oneToTen, fourToFive, zeroToTen, zeroToTwenty, oneToTwenty:
            iset._checkRep()

        checkEq = self._intervalEq

        # Tests for addition: A + B, where...
        #   1. A and B are empty.
        checkEq(nil+nil, nil, [])
        #   2. Just A or B is empty.
        checkEq(nil+oneToTen, oneToTen+nil, oneToTen, [(1,10)])
        #   3. A contains B, or vice versa.
        checkEq(oneToTen+fourToFive, fourToFive+oneToTen, oneToTen)
        checkEq(oneToTen+zeroToTwenty, zeroToTwenty)
        #   4. A == B
        checkEq(oneToTen+oneToTen, oneToTen)
        #   5. A and B are disjoint and don't touch.
        checkEq(oneToTen+fifteenToFifty, fifteenToFifty+oneToTen,
                [(1,10),(15,50)])
        #   6. A and B are disjoint and touch
        checkEq(oneToTen+tenToTwenty, tenToTwenty+oneToTen, oneToTwenty)
        #   7. A and B overlap on one side only.
        checkEq(oneToTwenty+fifteenToFifty,
                fifteenToFifty+oneToTwenty,
                "IntervalSet([(1,50)])")
        #   8. A nice complex situation.
        fromPrimeToPrime = IntervalSet([(2,3),(5,7),(11,13),(17,19),(23,29)])
        fromSquareToSquare = IntervalSet([(1,4),(9,16),(25,36)])
        fromFibToFib = IntervalSet([(1,1),(2,3),(5,8),(13,21),(34,55)])
        x = fromPrimeToPrime.copy()
        x += fromSquareToSquare
        x += fromSquareToSquare
        checkEq(fromPrimeToPrime+fromSquareToSquare,
                fromSquareToSquare+fromPrimeToPrime,
                x,
                [(1,4),(5,7),(9,16),(17,19),(23,36)])
        checkEq(fromSquareToSquare+fromFibToFib,
                [(1,4),(5,8),(9,21),(25,55)])
        checkEq(fromPrimeToPrime+fromFibToFib,
                [(2,3),(5,8),(11,21),(23,29),(34,55)])

        # Now, subtraction!
        #  1. Involving nil.
        checkEq(nil-nil, nil, [])
        checkEq(fromSquareToSquare-nil, fromSquareToSquare)
        checkEq(nil-fromSquareToSquare, nil)
        #  2. Disjoint ranges.
        checkEq(fourToFive-tenToTwenty, fourToFive)
        checkEq(tenToTwenty-fourToFive, tenToTwenty)
        #  3. Matching on one side
        checkEq(oneToTwenty-oneToTen, tenToTwenty)
        checkEq(oneToTwenty-tenToTwenty, oneToTen)
        checkEq(oneToTen-oneToTwenty, nil)
        checkEq(tenToTwenty-oneToTwenty, nil)
        #  4. Overlapping on one side
        checkEq(fifteenToFifty-oneToTwenty, [(20,50)])
        checkEq(oneToTwenty-fifteenToFifty, [(1,15)])
        #  5. Overlapping in the middle
        checkEq(oneToTen-fourToFive, [(1,4),(5,10)])
        checkEq(fourToFive-oneToTen, nil)
        #  6. Complicated
        checkEq(fromPrimeToPrime-fromSquareToSquare,
                [(5,7),(17,19),(23,25)])
        checkEq(fromSquareToSquare-fromPrimeToPrime,
                [(1,2),(3,4),(9,11),(13,16),(29,36)])
        checkEq(fromSquareToSquare-fromFibToFib,
                [(1,2),(3,4),(9,13),(25,34)])
        checkEq(fromFibToFib-fromSquareToSquare,
                [(5,8),(16,21),(36,55)])
        #  7. Identities
        for a in (fromPrimeToPrime, fromSquareToSquare, fromFibToFib, nil):
            for b in (fromPrimeToPrime, fromSquareToSquare, fromFibToFib, nil):
                checkEq(a-b+b, a+b)
                checkEq(a+b-b, a-b)

        ## Test intersection
        # 1. With nil
        checkEq(nil*nil, nil*fromFibToFib, oneToTen*nil, nil, [])
        # 2. Self
        for iset in oneToTen, fromSquareToSquare, fourToFive:
            checkEq(iset, iset*iset)
        # 3. A disjoint from B
        checkEq(oneToTen*fifteenToFifty, fifteenToFifty*oneToTen, nil)
        # 4. A disjoint from B but touching.
        checkEq(oneToTen*tenToTwenty, tenToTwenty*oneToTen, nil)
        # 5. A contains B at the middle.
        checkEq(oneToTen*fourToFive, fourToFive*oneToTen, fourToFive)
        # 6. A contains B at one end
        checkEq(oneToTen*oneToTwenty, oneToTwenty*oneToTen, oneToTen)
        checkEq(tenToTwenty*oneToTwenty, oneToTwenty*tenToTwenty, tenToTwenty)
        # 7. A and B overlap without containment.
        checkEq(fifteenToFifty*oneToTwenty, oneToTwenty*fifteenToFifty,
                [(15,20)])
        # 8. Tricky cases
        checkEq(fromPrimeToPrime*fromSquareToSquare,
                fromSquareToSquare*fromPrimeToPrime,
                [(2,3),(11,13),(25,29)])
        checkEq(fromPrimeToPrime*fromFibToFib,
                fromFibToFib*fromPrimeToPrime,
                [(2,3),(5,7),(17,19)])
        checkEq(fromSquareToSquare*fromFibToFib,
                fromFibToFib*fromSquareToSquare,
                [(2,3),(13,16),(34,36)])
        # 9. Identities
        for a in (fromPrimeToPrime, fromSquareToSquare, fromFibToFib, oneToTen,
                  fifteenToFifty, nil):
            self.assert_((not a) == a.isEmpty() == (a == nil))
            for b in (fromPrimeToPrime, fromSquareToSquare, fromFibToFib,
                      oneToTen, fifteenToFifty, nil):
                checkEq(a*b,b*a)
                checkEq(a-b, a*(a-b), (a-b)*a)
                checkEq(b*(a-b), (a-b)*b, nil)
                checkEq(a-b, a-a*b)
                checkEq((a-b)+a*b, a)
                checkEq((a-b)*(b-a), nil)
                checkEq((a-b)+(b-a)+a*b, a+b)

        ## Contains / getIntervalContaining
        t = self.assert_
        # 1. With nil
        t(5 not in nil)
        t(oneToTen not in nil)
        t(fromFibToFib not in nil)
        eq((None,None), nil.getIntervalContaining(5))
        # 2. Self in self
        for iset in nil, oneToTen, tenToTwenty, fromSquareToSquare:
            t(iset in iset)
        # 3. Simple sets: closed below, open above.
        t(1 in oneToTen)
        t(2 in oneToTen)
        t(9.9 in oneToTen)
        t(10 not in oneToTen)
        t(0 not in oneToTen)
        t(11 not in oneToTen)
        eq((1,10), oneToTen.getIntervalContaining(1))
        eq((1,10), oneToTen.getIntervalContaining(9.9))
        eq((None,None), oneToTen.getIntervalContaining(0))
        eq((None,None), oneToTen.getIntervalContaining(10))
        eq((None,None), oneToTen.getIntervalContaining(11))
        # 4. Simple sets: A contains B.
        t(fourToFive in oneToTen) # contained wholly
        t(oneToTen in zeroToTen) #contains on one side.
        t(oneToTwenty not in oneToTen) #disjoint on one side
        t(oneToTen not in tenToTwenty) #disjoint but touching
        t(fourToFive not in tenToTwenty) #disjoint, not touching
        # 5. Complex sets: closed below, open above
        t(0 not in fromSquareToSquare)
        t(1 in fromSquareToSquare)
        t(2 in fromSquareToSquare)
        t(4 not in fromSquareToSquare)
        t(8 not in fromSquareToSquare)
        t(9 in fromSquareToSquare)
        t(15 in fromSquareToSquare)
        t(16 not in fromSquareToSquare)
        t(35 in fromSquareToSquare)
        t(36 not in fromSquareToSquare)
        t(100 not in fromSquareToSquare)
        eq((None,None), fromSquareToSquare.getIntervalContaining(0))
        eq((1,4), fromSquareToSquare.getIntervalContaining(1))
        eq((1,4), fromSquareToSquare.getIntervalContaining(2))
        eq((None,None), fromSquareToSquare.getIntervalContaining(4))
        eq((None,None), fromSquareToSquare.getIntervalContaining(8))
        eq((9,16), fromSquareToSquare.getIntervalContaining(9))
        eq((9,16), fromSquareToSquare.getIntervalContaining(15))
        eq((None,None), fromSquareToSquare.getIntervalContaining(16))
        eq((25,36), fromSquareToSquare.getIntervalContaining(35))
        eq((None,None), fromSquareToSquare.getIntervalContaining(36))
        eq((None,None), fromSquareToSquare.getIntervalContaining(49))

        # SpanLength
        eq(0, nil.spanLength())
        eq(0, nil2.spanLength())
        eq(0, nil3.spanLength())
        eq(9, oneToTen.spanLength())
        eq(1, fourToFive.spanLength())
        eq(20, zeroToTwenty.spanLength())
        eq(20, zeroToTwenty.spanLength())
        eq(13, fromPrimeToPrime.spanLength())
        eq(21, fromSquareToSquare.spanLength())
        eq(33, fromFibToFib.spanLength())

    def _intervalEq(self, a, *others):
        eq = self.assertEquals
        for b in others:
            if isinstance(b, IntervalSet):
                eq(a,b)
                b._checkRep()
            elif isinstance(b, types.StringType):
                eq(repr(a), b)
            elif isinstance(b, types.ListType):
                eq(a.getIntervals(), b)
            else:
                raise MixError()
            a._checkRep()

    def test_openUnique(self):
        d = mix_mktemp()
        os.mkdir(d)
        dX = os.path.join(d,"X")
        f, fn = openUnique(dX)
        f.write("X")
        f.close()
        self.assertEquals(fn, dX)

        f, fn = openUnique(dX)
        f.write("X")
        f.close()
        self.assertEquals(fn, dX+".1")

        f, fn = openUnique(dX)
        f.write("X")
        f.close()
        self.assertEquals(fn, dX+".2")

    def test_lockfile(self):
        #XXXX The 'locked' cases need to get tested on windows.  Right now
        #XXXX that's hard, because they don't seem to block one another unless
        #XXXX you try them from separate processes.

        fn = mix_mktemp()
        LF1 = Lockfile(fn)
        LF2 = Lockfile(fn)
        LF1.acquire("LF1")
        if not ON_WINDOWS:
            self.assertEquals("LF1", readFile(fn))
        if not ON_WIN32:
            self.assertRaises(LockfileLocked, LF2.acquire, blocking=0)
        LF1.replaceContents("new longer contents")
        if not ON_WINDOWS:
            self.assertEquals("new longer contents", readFile(fn))
        LF1.replaceContents("shorter")
        if not ON_WINDOWS:
            self.assertEquals("shorter", readFile(fn))
        if not ON_WIN32:
            self.assertRaises(LockfileLocked, LF2.acquire, blocking=0)
        LF1.release()
        LF2.acquire("LF2",1)
        if not ON_WINDOWS:
            self.assertEquals("LF2", readFile(fn))
            self.assertRaises(LockfileLocked, LF1.acquire, blocking=0)

        # Now try recursivity.
        LF2.acquire()
        if not ON_WIN32:
            self.assertRaises(LockfileLocked, LF1.acquire, blocking=0)
        LF2.release()
        if not ON_WIN32:
            self.assertRaises(LockfileLocked, LF1.acquire, blocking=0)
        LF2.release()

        if ON_WIN32:
            return

        LF1.acquire(blocking=1)

        # Now try a blocking lock.  We need to block in another process
        # because of some platforms' implementations of threading.
        releasedFile = mix_mktemp()
        writeFile(releasedFile, "nobody")
        pid = os.fork()
        if pid == 0: # child
            LF2.acquire("LF2", blocking=1)
            if readFile(releasedFile) == 'parent':
                writeFile(releasedFile, "GOOD")
            else:
                writeFile(releasedFile, "BAD")
            os._exit(0)

        time.sleep(.1)
        writeFile(releasedFile, 'parent')
        LF1.release()
        os.waitpid(pid, 0)
        self.assertEquals("GOOD", readFile(releasedFile))

    def test_encodeBase64(self):
        longish = "xyzzyasjklsadjsakldjsakldjsakldjskljaskldjsadkljsa"*10
        d32 = encodeBase64(longish, lineWidth=32)
        d64 = encodeBase64(longish, lineWidth=64)
        self.assertEquals(longish, base64.decodestring(d32))
        self.assertEquals(longish, base64.decodestring(d64))
        for enc, maxlen in ((d32, 33), (d64,65)):
            lines = enc.split("\n")
            for line in lines[-1]:
                self.assertEquals(len(line), maxlen)
            self.assert_(len(lines[-1]) <= maxlen)

    def test_armor(self):
        inp1 = "xyzzy111"*10
        inp2 = "Hello\nWorld!\n--Me.\n"
        # Simple base64
        self.assertEquals(armorText(inp1, "FOO BAR", (), 1),
"""-----BEGIN FOO BAR-----

eHl6enkxMTF4eXp6eTExMXh5enp5MTExeHl6enkxMTF4eXp6eTExMXh5enp5MTEx
eHl6enkxMTF4eXp6eTExMXh5enp5MTExeHl6enkxMTE=
-----END FOO BAR-----\n""")
        # With headers
        self.assertEquals(armorText(inp1, "FOO BAR",
                                    [("H1", "xyz"),("H2", "a b")], 1),
"""-----BEGIN FOO BAR-----
H1: xyz
H2: a b

eHl6enkxMTF4eXp6eTExMXh5enp5MTExeHl6enkxMTF4eXp6eTExMXh5enp5MTEx
eHl6enkxMTF4eXp6eTExMXh5enp5MTExeHl6enkxMTE=
-----END FOO BAR-----\n""")

        # No base64
        self.assertEquals(armorText(inp1, "FOO BAR", (), 0),
"""-----BEGIN FOO BAR-----

xyzzy111xyzzy111xyzzy111xyzzy111xyzzy111xyzzy111xyzzy111xyzzy111xyzzy111xyzzy111
-----END FOO BAR-----\n""")

        # No base64, hyphen escape.
        self.assertEquals(armorText(inp2, "FOO BAR", [("H3", "text")], 0),
"""-----BEGIN FOO BAR-----
H3: text

Hello
World!
- --Me.
-----END FOO BAR-----\n""")

        # Test unarmor, succeeding cases.
        for inp in inp1, inp2:
            for mode in 0,1:
                for headers in [], [("H-x", "abcde fg")]:
                    enc = armorText(inp, "THIS THAT", headers, mode)
                    enc = "asdasdkjlasd\nHey kids!\n"+enc
                    tp, h, b = unarmorText(enc, ["THIS THAT", "FRED"],
                                           mode)[0]
                    self.assertEquals(tp, "THIS THAT")
                    self.assertEquals(h, headers)
                    self.assertEquals(b.strip(), inp.strip())
                    if not mode:
                        self.assertEndsWith(b, "\n")

        # Test unarmor and [...]
        enc = armorText(inp2*50, "MUNGED", [], base64=1)
        enc = enc.split("\n")
        enc[-10:-5] = [ "[...]" ]
        enc = "\n".join(enc)
        self.assertRaises(UIError, unarmorText, enc, ["MUNGED"], 1)

        # Test armor and \r\n
        enc = armorText(inp2*50, "MUNGED", [], base64=1)
        enc = enc.replace("\n","\r\n")
        tp, h, b = unarmorText(enc, ["MUNGED"], 1)[0]
        self.assertEquals(b, inp2*50)

        # Test base64fn and concatenation.
        enc1 = armorText(inp2, "THIS THAT", [("H-64", "0")], 0)
        enc2 = armorText(inp2, "THIS THAT", [("H-64", "1")], 1)
        def base64fn(tp, h, assert_=self.assert_):
            assert_(tp=="THIS THAT")
            assert_(len(h) == 1)
            assert_(len(h[0]) == 2)
            assert_(h[0][0] == "H-64")
            return int(h[0][1])
        dec = unarmorText(enc1+enc2, ["THIS THAT"], base64fn=base64fn)
        self.assertEquals(len(dec), 2)
        for mode in 0, 1:
            tp,h,b = dec[mode]
            self.assertEquals(tp, "THIS THAT")
            self.assertEquals(h, [("H-64", str(mode))])
            self.assertEquals(b, inp2)

        # Test skipping unwanted types
        enc3 = armorText(inp2, "OTHER", [("H-64", "1")], 1)
        dec2 = unarmorText(enc1+enc3+enc2, ["THIS THAT"], base64fn=base64fn)
        self.assertEquals(dec2, dec)

        # Test skipping broken armor.
        self.assertRaises(ValueError,
                          unarmorText,"-----BEGIN X-----\n\n", ["Y"], 1)
        self.assertRaises(ValueError,
                          unarmorText,
                          "-----BEGIN X-----\n"
                          ":B:C\n\n"
                          "A B C\n-----END X-----\n", ["X"], 0)
        self.assertRaises(ValueError,
                          unarmorText,
                          "-----BEGIN X-----\n"
                          "A: X\n\n"
                          "A B C\n-----END X-----\n", ["X"], 1)

    def test_clearableQueue(self):
        #FFFF This test is inadequate for weird multithreaded
        q = mixminion.ThreadUtils.ClearableQueue()
        self.assert_(q.empty())
        q.put(1)
        q.put(2)
        self.assertEquals(2, q.qsize())
        self.assertEquals(1, q.get())
        q.clear()
        self.assert_(q.empty())
        self.assertRaises(mixminion.ThreadUtils.QueueEmpty, q.get_nowait)

    def test_rwlock(self):
        RWLock = mixminion.ThreadUtils.RWLock
        lock = RWLock()
        # XXXX007 This only tests non-blocking cases
        lock.read_in()
        lock.read_in()
        lock.read_out()
        lock.read_out()
        lock.write_in()
        lock.write_out()
        lock.read_in()
        lock.read_out()

    def test_englishSequence(self):
        es = englishSequence
        self.assertEquals("none", es([]))
        self.assertEquals("fred", es(["fred"]))
        self.assertEquals("fred and joe", es(["fred", "joe"]))
        self.assertEquals("fred, joe, and bob", es(["fred", "joe", "bob"]))
        self.assertEquals("fred, jr; joe; and bob",
                          es(["fred, jr", "joe", "bob"]))
        self.assertEquals("fred, jr; and bob",
                          es(["fred, jr", "bob"]))
        self.assertEquals("Kernighan and Ritchie; and Laurel and Hardy",
                          es(["Kernighan and Ritchie", "Laurel and Hardy"]))
        self.assertEquals("Kernighan and Ritchie; or Laurel and Hardy",
                          es(["Kernighan and Ritchie", "Laurel and Hardy"],
                             compound="or"))
        self.assertEquals("fred", es(["fred"], compound="or"))
        self.assertEquals("fred or joe", es(["fred", "joe"], compound="or"))
        self.assertEquals("fred, joe, or bob",
                          es(["fred", "joe", "bob"], compound="or"))

    def test_fileops(self):
        # XXXXX Test more file ops.
        fn = mix_mktemp()
        for _ in 1,2:
            f = open(fn, 'w')
            tst = [ "I think the fact\n",
                    "that we are not currently under siege\n",
                    "by unscrupulous minions\n",
                    "speaks for itself\n" ]
            f.write("".join(tst))
            f.close()
            f = open(fn, 'r')
            lst = [ item for item in iterFileLines(f) ]
            f.close()
            self.assertEquals(lst, tst)
            tst.append("unterminated line")

#----------------------------------------------------------------------

class MinionlibCryptoTests(TestCase):
    """Tests for cryptographic C extensions."""
    def test_sha1(self):
        s1 = _ml.sha1

        # A test vector from the SHA1 spec
        self.assertEquals(s1("abc"),
              hexread("A9993E364706816ABA3E25717850C26C9CD0D89D"))

        # Another test vector from the SHA1 spec
        s = s1("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq")
        self.assertEquals(s,
               hexread("84983E441C3BD26EBAAE4AA1F95129E5E54670F1"))

        # Make sure that we fail gracefully on non-string input.
        self.failUnlessRaises(TypeError, s1, 1)

    def test_xor(self):
        xor = _ml.strxor

        # Try a few known-value XORs.
        self.assertEquals(xor("abc", "\000\000\000"), "abc")
        self.assertEquals(xor("abc", "abc"), "\000\000\000")
        self.assertEquals(xor("\xEF\xF0\x12", "\x11\x22\x35"), '\xFE\xD2\x27')

        # Make sure that the C doesn't (cringe) modify the strings out from
        # under us.
        a = "aaaa"
        self.assertEquals(xor(a,"\000\000\000a"), "aaa\000")
        self.assertEquals(a, "aaaa")
        self.assertEquals(xor("\000\000\000a",a), "aaa\000")
        self.assertEquals(a, "aaaa")

        # Check for error msg on XORing strings of unequal length.
        self.failUnlessRaises(TypeError, xor, "a", "bb")

    def test_aes(self):
        crypt = _ml.aes_ctr128_crypt

        # First, try one of the test vectors from the AES spec.
        key = txt = "\x80" + "\x00" * 15
        key = _ml.aes_key(key)
        expected = hexread("8EDD33D3C621E546455BD8BA1418BEC8")
        self.failUnless(crypt(key, txt, 0) == expected)
        self.failUnless(crypt(key, txt) == expected)

        # Now, make sure that the counter implementation is sane.
        self.failUnless(crypt(key, " "*100, 0)[1:] == crypt(key, " "*99, 1))
        self.failUnless(crypt(key, " "*100, 0)[30:] == crypt(key, " "*70, 30))

        # Make sure ctr_crypt works the same everywhere.
        expected2 = hexread("351DA02F1CF68C4BED393BC71274D181892FC420CA9E9995"
                            "C6E5E9744920020DB854019CB1CEB6BAD055C64F60E63B91"
                            "5917930EB30972BCB3942E6904252F26")
        self.failUnless(crypt(key, " "*64, 0xABCD) == expected2)

        # Counter mode is its own inverse
        self.failUnless(crypt(key,crypt(key, " "*100, 0),0) == " "*100)

        # Try a different key to be sure.
        teststr = """I have seen the best ciphers of my generation
                     Destroyed by cryptanalysis, broken, insecure,
                     Implemented still in cryptographic libraries"""
        key2 = _ml.aes_key("xyzz"*4)
        self.assertEquals(teststr,crypt(key2,crypt(key2,teststr)))

        # Try generating the same test vector again, but this time in PRNG
        # mode
        expected2 = hexread("0EDD33D3C621E546455BD8BA1418BEC8")
        self.assertEquals(expected2, crypt(key, "", 0, len(expected2)))
        # PRNG mode ignores input.
        self.assertEquals(expected2, crypt(key, "Z", 0, len(expected2)))
        # Try an offset with prng mode.
        self.assertEquals(expected2[5:], crypt(key, "", 5, len(expected2)-5))
        # Make sure that PRNG mode with negative count yields ""
        self.assertEquals("", crypt(key,"",0,-1))

        # Can't use a non-key object.
        self.failUnlessRaises(TypeError, crypt, "a", teststr)
        # Can't make a key from a short string...
        self.failUnlessRaises(TypeError, _ml.aes_key, "a")
        # ...or a long string.
        self.failUnlessRaises(TypeError, _ml.aes_key, "a"*17)

    def test_openssl_seed(self):
        # Just try seeding openssl a couple of times, and make sure it
        # doesn't crash.
        _ml.openssl_seed("Hello")
        _ml.openssl_seed("")

    def test_oaep(self):
        _add = Crypto._add_oaep_padding
        _check = Crypto._check_oaep_padding
        # Perform OAEP tests with C implementation of OAEP and Python
        # implementation too.
        for add,check in ((_ml.add_oaep_padding, _ml.check_oaep_padding),
                          (_add, _check)):
            self.do_test_oaep(add, check)

        # Make sure they can invert one another.
        self.assertEquals("a",_check(_ml.add_oaep_padding("a", "b", 128),
                                         "b",128))
        self.assertEquals("a",_ml.check_oaep_padding(_add("a","b",128),
                                                         "b",128))

    def do_test_oaep(self, add, check):
        strxor = _ml.strxor
        # Check_oaep inverts add_oaep successfully.
        x = add("A", "B", 128)

        self.assertEquals("A",check(x, "B", 128))

        # 86 bytes can be used with size=128
        add("A"*86, "B",128)
        # But 300 is too much,
        self.failUnlessRaises(_ml.CryptoError,
                              add,"A"*300, "B", 128)
        # And so is even 87.
        self.failUnlessRaises(_ml.CryptoError,
                              add,"A"*87, "B", 128)
        # Changing a character at the beginning keeps it from checking.
        ch = strxor(x[0], '\x01')
        self.failUnlessRaises(_ml.CryptoError,
                              check,ch+x[1:],"B",128)
        # Changing a character at the end keeps it from checking.
        ch = strxor(x[-1], '\x01')
        self.failUnlessRaises(_ml.CryptoError,
                              check,x[:-1]+ch,"B",128)

    def test_rsa(self):
        p = getRSAKey(1,1024)

        #for all of SIGN, CHECK_SIG, ENCRYPT, DECRYPT...
        for pub1 in (0,1):
            for enc1 in (0,1):
                msg = "Now is the time for all anonymous parties"
                x = _ml.add_oaep_padding(msg, "B", 128)
                x2 = p.crypt(x, pub1, enc1);
                # ...Encryption inverts decryption...
                x3 = p.crypt(x2, [1,0][pub1], [1,0][enc1]);
                self.failUnless(x3 == x)
                # ...And oaep is preserved.
                x4 = _ml.check_oaep_padding(x3, "B", 128)
                self.failUnless(x4 == msg)

        # Fail if there is not enough padding
        self.failUnlessRaises(_ml.CryptoError,p.crypt,"X",1,1)
        # Fail if there is too much padding
        self.failUnlessRaises(_ml.CryptoError,p.crypt,x+"ZZZ",1,1)

        ###
        # Test key equality and fingerprinting.
        self.assert_(pk_same_public_key(p, p))
        self.assert_(not pk_same_public_key(p, getRSAKey(2,1024)))
        self.assert_(len(pk_fingerprint(p))==40)
        self.assertNotEquals(pk_fingerprint(p),
                             pk_fingerprint(getRSAKey(2,1024)))
        ####
        # Test key encoding
        padhello = _ml.add_oaep_padding("Hello", "B", 128)
        for public in (0,1):
            #encode(decode(encode(x))) == x.
            x = p.encode_key(public)
            p2 = _ml.rsa_decode_key(x,public)
            x3 = p2.encode_key(public)
            self.assertEquals(x,x3)
            # decode(encode(x)) encrypts the same as x.
            self.assertEquals(p.crypt(padhello,public,1),
                              p2.crypt(padhello,public,1))
            self.assert_(pk_same_public_key(p, p2))
            self.assertEquals(pk_fingerprint(p),
                              pk_fingerprint(p2))

        # encoding public keys to/from their moduli.
        self.assertEquals(p.get_modulus_bytes(),1024 >> 3)
        n,e = p.get_public_key()
        # Let p2 be the public portion of p.
        p2 = _ml.rsa_make_public_key(n,e)
        self.assertEquals((n,e), p2.get_public_key())
        self.assertEquals(65537,e)
        self.assertEquals(p.encode_key(1), p.encode_key(1))

        # Let P3 be the public key component of p.  Encrypt some messages
        # using p...
        p3 = _ml.rsa_decode_key(p.encode_key(1),1)
        msg1 = p.crypt(padhello, 1,1)
        msg2 = p.crypt(padhello, 1,1)
        msg3 = p.crypt(padhello, 1,1)
        self.assertEquals(padhello, p.crypt(msg1,0,0))
        self.assertEquals(padhello, p.crypt(msg2,0,0))
        self.assertEquals(padhello, p.crypt(msg3,0,0))
        # And make sure that neither p2 nor p3 can decode them.
        self.failUnlessRaises(TypeError, p2.crypt, msg1, 0, 0)
        self.failUnlessRaises(TypeError, p3.crypt, msg1, 0, 0)
        # And make sure that we can't encode either as a private key
        self.failUnlessRaises(TypeError, p2.encode_key, 0)
        self.failUnlessRaises(TypeError, p3.encode_key, 0)

        # Test PEM encoding (we need this encoding because it's the
        # most widely used way to store encrypted private keys.(
        tf = mix_mktemp()
        tf_pub = tf + "1"
        tf_prv = tf + "2"
        tf_enc = tf + "3"

        p.PEM_write_key(open(tf_pub,'w'), 1)
        p.PEM_write_key(open(tf_prv,'w'), 0)
        p.PEM_write_key(open(tf_enc,'w'), 0, "top sekrit")
        p2 = _ml.rsa_PEM_read_key(open(tf_pub, 'r'), 1)
        self.assertEquals(p.get_public_key(), p2.get_public_key())

        p2 = _ml.rsa_PEM_read_key(open(tf_prv, 'r'), 0)
        self.assertEquals(p.encode_key(0), p2.encode_key(0))

        self.failUnlessRaises(_ml.CryptoError,
                              _ml.rsa_PEM_read_key,
                              open(tf_enc, 'r'), 0)

        p2 = _ml.rsa_PEM_read_key(open(tf_prv, 'r'), 0, "top sekrit")
        self.assertEquals(p.encode_key(0), p2.encode_key(0))

#----------------------------------------------------------------------
class MinionlibFECTests(TestCase):
    def do_fec_test(self, k, n, sz):
        r = getCommonPRNG()

        eq = self.assertEquals
        fec = _ml.FEC_generate(k,n)
        eq((k,n), fec.getParameters())

        inpChunks = [ r.getBytes(sz) for i in xrange(k) ]
        inp = "".join(inpChunks)
        outChunks = [ fec.encode(i, inpChunks) for i in xrange(n) ]
        for i in xrange(n):
            if i < k:
                eq(outChunks[i], inpChunks[i])
            eq(len(outChunks[i]), sz)

        numberedChunks = [ (i, outChunks[i]) for i in xrange(n) ]

        for i in xrange(200):
            chk = r.shuffle(numberedChunks, k)
            #print [ i for i,_ in chk ]
            out = fec.decode(chk)
            eq(out, inpChunks)
            eq("".join(out), inp)

    def test_good_fec(self):
        self.do_fec_test(3,5,10)
        self.do_fec_test(10,20,1024)
        self.do_fec_test(30,40,1024)
        self.do_fec_test(30,40,1)
        self.do_fec_test(3,4,128)
        self.do_fec_test(3,3,2048)

    def test_bad_fec(self):

        self.assertRaises(_ml.FECError,_ml.FEC_generate,5,3)
        self.assertRaises(_ml.FECError,_ml.FEC_generate,0,10)
        self.assertRaises(_ml.FECError,_ml.FEC_generate,10,300)

        r = getCommonPRNG()
        fec = _ml.FEC_generate(3,5)
        inp = [ r.getBytes(256) for i in xrange(3) ]
        self.assertRaises(_ml.FECError,fec.encode, -1, inp)
        self.assertRaises(_ml.FECError,fec.encode, 5, inp)
        self.assertRaises(_ml.FECError,fec.encode, 1, inp[0:2])
        self.assertRaises(_ml.FECError,fec.encode, 1, inp*2)
        self.assertRaises(_ml.FECError,fec.encode, 1, inp[0:2]+[3])
        self.assertRaises(_ml.FECError,fec.encode, 1, 5)

        cInp = [ (i, fec.encode(i, inp)) for i in xrange(5) ]
        fec.decode(cInp[2:5])
        self.assertRaises(_ml.FECError,fec.decode, inp)
        self.assertRaises(_ml.FECError,fec.decode, [1,2,3])
        self.assertRaises(_ml.FECError,fec.decode, cInp)
        self.assertRaises(_ml.FECError,fec.decode, cInp[:-1])
        self.assertRaises(_ml.FECError,fec.decode, cInp[0:2]+[2,'x'])

#----------------------------------------------------------------------

class CryptoTests(TestCase):
    """Tests for Python cryptographic library"""
    def test_initcrypto(self):
        init_crypto()

    def test_wrappers(self):
        # Test simple wrappers over _minionlib functionality.  Mainly, just
        # test that _ml.foo and foo do the same thing.
        self.assertEquals(_ml.sha1("xyzzy"), sha1("xyzzy"))
        k = _ml.aes_key("xyzy"*4)
        self.assertEquals(_ml.aes_ctr128_crypt(k,"hello",0),
                          ctr_crypt("hello",k))
        self.assertEquals(_ml.aes_ctr128_crypt(k,"hello",99),
                          ctr_crypt("hello",k,99))
        self.assertEquals(_ml.aes_ctr128_crypt(k,"",0,99), prng(k,99))
        self.assertEquals(_ml.aes_ctr128_crypt(k,"",3,99), prng(k,99,3))
        self.assertEquals(prng(k,100,0),prng(k,50,0)+prng(k,50,50))

    def test_rsa(self):

        eq = self.assertEquals
        k512 = getRSAKey(0,512)
        k1024 = getRSAKey(0,1024)

        eq(512>>3, k512.get_modulus_bytes())
        eq(1024>>3, k1024.get_modulus_bytes())

        # Check pk_get_modulus sanity
        self.failUnless((1L<<511) < pk_get_modulus(k512) < (1L<<513))
        self.failUnless((1L<<1023) < pk_get_modulus(k1024) < (1L<<1024))
        self.assertEquals(pk_get_modulus(k512),
                        pk_get_modulus(pk_from_modulus(pk_get_modulus(k512))))

        # Make sure that public keys can be made from moduli, and used to
        # encrypt and decrypt.
        msg="Good hello"
        pub512 = pk_from_modulus(pk_get_modulus(k512))
        pub1024 = pk_from_modulus(pk_get_modulus(k1024))

        eq(msg, pk_decrypt(pk_encrypt(msg, k512),k512))
        eq(msg, pk_decrypt(pk_encrypt(msg, pub512),k512))
        eq(msg, pk_decrypt(pk_encrypt(msg, k1024),k1024))
        eq(msg, pk_decrypt(pk_encrypt(msg, pub1024),k1024))

        # Make sure that CH_OAEP(RSA()) inverts pk_encrypt.
        eq(msg, _ml.check_oaep_padding(
                    k512.crypt(pk_encrypt(msg,k512), 0, 0),
                    Crypto.OAEP_PARAMETER, 64))

        # test signing
        eq(pk_check_signature(pk_sign(msg, k1024),pub1024), msg)
        eq(pk_check_signature(pk_sign(msg, k1024),k1024), msg)
        self.failUnlessRaises(TypeError,
                              pk_sign, msg, pub1024)
        self.failUnlessRaises(CryptoError,
                              pk_check_signature,
                              pk_sign(msg, k1024)+"X",
                              pub1024)

        # Make sure we can still encrypt after we've encoded/decoded a
        # key.
        encoded = pk_encode_private_key(k512)
        decoded = pk_decode_private_key(encoded)
        eq(msg, pk_decrypt(pk_encrypt(msg, pub512),decoded))

        # Test pickling
        init_crypto()
        s = cPickle.dumps(k512)
        self.assertEquals(cPickle.loads(s).get_public_key(),
                          k512.get_public_key())
        s = cPickle.dumps(k512, 1)
        self.assertEquals(cPickle.loads(s).get_public_key(),
                          k512.get_public_key())

    def test_trng(self):
        # Make sure that the true rng is at least superficially ok.
        self.assertNotEquals(trng(40), trng(40))

    def test_lioness(self):
        enc = lioness_encrypt
        dec = lioness_decrypt

        # Check basic cipher properties.
        key = ("ABCDE"*4, "ABCDF"*4, "DECBA"*4, "VWXYZ"*4)
        plain = "The more it snows the more it goes on snowing"*10
        self.assertNotEquals(plain, enc(plain,key))
        self.assertNotEquals(plain, dec(plain,key))
        self.assertEquals(len(plain), len(enc(plain,key)))
        self.assertEquals(len(plain), len(dec(plain,key)))
        self.assertEquals(plain, dec(enc(plain,key),key))
        self.assertEquals(plain, enc(dec(plain,key),key))

        # Walk through a LIONESS encryption to check for correct values.
        # Check getLionessKeys too.
        s = "ABCDE"*4
        key1 = sha1(s+"foo")
        key2 = key1[:-1]+strxor(key1[-1], chr(1))
        key3 = key1[:-1]+strxor(key1[-1], chr(2))
        key4 = key1[:-1]+strxor(key1[-1], chr(3))

        left = plain[:20]
        right = plain[20:]
        right = ctr_crypt(right, sha1(key1+left+key1)[:16])
        left  = strxor(left, sha1(key2+right+key2))
        right = ctr_crypt(right, sha1(key3+left+key3)[:16])
        left  = strxor(left, sha1(key4+right+key4))

        key = (key1,key2,key3,key4)
        self.assertEquals(left+right, lioness_encrypt(plain,key))
        self.assertEquals(key, Keyset("ABCDE"*4).getLionessKeys("foo"))

        u = "Hello world"*2
        w = whiten(u)
        self.assertNotEquals(w, u)
        self.assertEquals(unwhiten(w), u)
        u = "xyzprdlty"*100
        w = whiten(u)
        self.assertNotEquals(w, u)
        self.assertEquals(unwhiten(w), u)

    def test_bear(self):
        enc = bear_encrypt
        dec = bear_decrypt

        # Check basic cipher properties.
        key = ("ABCDE"*4, "QRSTU"*4)
        plain = "The more it snows the more it goes on snowing"*10
        self.assertNotEquals(plain, enc(plain,key))
        self.assertNotEquals(plain, dec(plain,key))
        self.assertEquals(len(plain), len(enc(plain,key)))
        self.assertEquals(len(plain), len(dec(plain,key)))
        self.assertEquals(plain, dec(enc(plain,key),key))
        self.assertEquals(plain, enc(dec(plain,key),key))

        # Walk through a BEAR encryption to check for correct values.
        # Check getLionessKeys too.
        s = "ABCDE"*4
        key1 = sha1(s+"foo")
        key2 = key1[:-1]+strxor(key1[-1], chr(1))

        left = plain[:20]
        right = plain[20:]
        left = strxor(left, sha1(key1+right+key1))
        right = ctr_crypt(right, sha1(left)[:16])
        left = strxor(left, sha1(key2+right+key2))

        key = (key1,key2)
        self.assertEquals(left+right, bear_encrypt(plain,key))
        self.assertEquals(key, Keyset("ABCDE"*4).getBearKeys("foo"))

    def test_keyset(self):
        s = sha1
        x = _ml.strxor
        # Make sure that keyset.get expected
        k = Keyset("a")
        eq = self.assertEquals
        eq(s("aFoo")[:10], k.get("Foo",10))
        eq(s("aBar")[:16], k.get("Bar"))

        # Make sure that keyset.getLionessKeys works as expected.
        z19 = "\x00"*19
        eq((s("aBaz"),               x(s("aBaz"), z19+"\x01"),
            x(s("aBaz"),z19+"\x02"), x(s("aBaz"), z19+"\x03")),
           k.getLionessKeys("Baz"))

    def test_aesprng(self):
        # Make sure that AESCounterPRNG is really repeatable.
        key ="aaab"*4
        PRNG = AESCounterPRNG(key)
        self.assert_(prng(key,100000) == (
                          PRNG.getBytes(5)+PRNG.getBytes(16*1024-5)+
                          PRNG.getBytes(50)+PRNG.getBytes(32*1024)+
                          PRNG.getBytes(9)+PRNG.getBytes(10)+
                          PRNG.getBytes(15)+PRNG.getBytes(16000)+
                          PRNG.getBytes(34764)))

        # Check getInt, getFloat.
        for i in xrange(1,10000,17):
            self.failUnless(0 <= PRNG.getInt(10) < 10)
            self.failUnless(0 <= PRNG.getInt(i) < i)

        # Check getNormal
        tot = 0
        for i in xrange(1,1000):
            v = PRNG.getNormal(5,1)
            self.failUnless(0 <= v <= 10)
            tot += v
        self.failUnless(4900<tot<5100)

        for i in xrange(100):
            self.failUnless(0 <= PRNG.getFloat() < 1)

        # Test the pick method
        lst = [1, 2, 3]
        count = [0,0,0,0]
        for _ in xrange(100):
            count[PRNG.pick(lst)]+=1
        self.assert_(count[0]==0)
        self.assert_(0 not in count[1:])
        self.assertRaises(IndexError, PRNG.pick, [])

        lst = range(100)
        # Test shuffle(0)
        self.assertEquals(PRNG.shuffle(lst,0), [])
        self.assertEquals(lst, range(100))
        # Make sure shuffle only shuffles the last n.
        PRNG.shuffle(lst,10)
        later = [ item for item in lst[10:] if item >= 10 ]
        s = later[:]
        s.sort()
        self.failUnless(later == s)

        # Make sure shuffle actually shuffles all positions.
        lists = [  ]
        for i in xrange(6):
            lists.append(PRNG.shuffle(lst)[:])
        # This will fail accidentally once in 10,000,000,000 attempts.
        for crossSection in zip(*lists):
            allEq = 1
            for z in crossSection:
                if z != crossSection[0]: allEq = 0
            self.failIf(allEq)
        foundUnmoved = 0
        for lst in lists:
            for inorder, shuffled in zip(lst, range(100)):
                if inorder == shuffled:
                    foundUnmoved = 1
                    break
            if foundUnmoved: break
        self.failUnless(foundUnmoved)
        for lst in lists:
            self.assertUnorderedEq(lst, range(100))

#----------------------------------------------------------------------

class PacketTests(TestCase):
    def test_subheader(self):
        s = Subheader(3,0,"abcdeabcdeabcdef",
                      "ABCDEFGHIJABCDEFGHIJ",
                      1, "Hello")

        expected = "\003\000abcdeabcdeabcdef"+\
                   "ABCDEFGHIJABCDEFGHIJ\000\005\000\001Hello"
        # test packing
        self.assertEquals(s.pack(), expected)
        self.assertEquals(s.getOverflow(), "")
        self.assertEquals(s.getUnderflowLength(), 256 - 42 - 42 - 5)
        self.assertEquals(s.getOverflowLength(), 0)

        # test unpacking,
        for more in "", "There is more":
            s = parseSubheader(s.pack()+more)
            self.assertEquals(s.major, 3)
            self.assertEquals(s.minor, 0)
            self.assertEquals(s.secret, "abcde"*3+"f")
            self.assertEquals(s.digest, "ABCDEFGHIJ"*2)
            self.assertEquals(s.routingtype, 1)
            self.assertEquals(s.routinglen, 5)
            self.assertEquals(s.routinginfo, "Hello")
            self.assertEquals(s.pack(), expected)
            self.assertEquals(s.underflow, more)
            self.assertEquals(s.getUnderflowLength(), 256-42-42-s.routinglen)

        ts_eliot = ("Who is the third who walks always beside you? / "+
                    "When I count, there are only you and I together / "+
                    "But when I look ahead up the white road / "+
                    "There is always another one walking beside you")

        s = Subheader(3,9,"abcdeabcdeabcdef",
                      "ABCDEFGHIJABCDEFGHIJ",
                      300, ts_eliot, len(ts_eliot))

        self.assertEquals(len(ts_eliot), 186)

        # test extended subeaders
        expected = ("\003\011abcdeabcdeabcdefABCDEFGHIJABCDEFGHIJ"+
                    "\000\272\001\054Who is the third who walks always"+
                    " beside you? / When I count, there are only you and I"+
                    " together / But when I look ahead up the white road /"+
                    " There is always another one walk")
        self.assertEquals(len(expected), mixminion.Packet.MAX_SUBHEADER_LEN)
        self.assertEquals(s.pack(), expected)

        overflow = s.getOverflow()
        self.assertEquals(len(overflow), 14)
        self.assertEquals(overflow, "ing beside you")

        # test parsing extended subheaders
        s = parseSubheader(expected)
        self.assertEquals(s.major, 3)
        self.assertEquals(s.minor, 9)
        self.assertEquals(s.secret, "abcde"*3+"f")
        self.assertEquals(s.digest, "ABCDEFGHIJ"*2)
        self.assertEquals(s.routingtype, 300)
        self.assertEquals(s.routinglen, 186)
        self.assertEquals(s.getOverflowLength(), 14)

        s.appendOverflow("ing beside you")
        self.assertEquals(s.routinginfo, ts_eliot)
        self.assertEquals(s.getExitAddress(), ts_eliot[20:])
        self.assertEquals(s.getTag(), ts_eliot[:20])
        self.assertEquals(s.pack(), expected)

        # Underlong subheaders must fail
        self.failUnlessRaises(ParseError,
                              parseSubheader, "a"*(41))
        # overlong subheaders must fail
        self.failUnlessRaises(ParseError,
                              parseSubheader, "a"*(256))

    def test_message(self):
        # Make sure we can pull the headers and payload of a message apart
        # correctly.
        # (Generate a nice random string to make sure we're slicing right.)
        m = Crypto.prng("HappyFunAESKey!!", 32768)
        msg = parsePacket(m)
        self.assert_(msg.pack() == m)
        self.assert_(msg.header1 == m[:2048])
        self.assert_(msg.header2 == m[2048:4096])
        self.assert_(msg.payload == m[4096:])
        self.failUnlessRaises(ParseError, parsePacket, m[:-1])
        self.failUnlessRaises(ParseError, parsePacket, m+"x")

    def test_ipv4info(self):
        # Check the IPV4Info structure used to hold the addresses for the
        # FWD and SWAP_FWD routing types.
        ri = hexread("12F400BCBBE30011223344556677889900112233445566778899")
        inf = parseIPV4Info(ri)
        self.assertEquals(inf.ip, "18.244.0.188")
        self.assertEquals(inf.port, 48099)
        self.assertEquals(inf.keyinfo, ri[-20:])
        self.assertEquals(len(inf.pack()), 26)
        self.assertEquals(inf.pack(), ri)
        self.assertEquals(IPV4Info("18.244.0.188", 48099, ri[-20:]).pack(),
                          ri)

        self.assertEquals(inf, parseRelayInfoByType(FWD_IPV4_TYPE,inf.pack()))
        self.assertEquals(inf, parseRelayInfoByType(SWAP_FWD_IPV4_TYPE,inf.pack()))

        self.failUnlessRaises(ParseError, parseIPV4Info, ri[:-1])
        self.failUnlessRaises(ParseError, parseIPV4Info, ri+"x")

    def test_mmtphostinfo(self):
        keyid = "zbcd"*5
        ri = "\x30\x55"+keyid+"the.hostname.is.here"
        inf = parseMMTPHostInfo(ri)
        self.assertEquals(inf.hostname, "the.hostname.is.here")
        self.assertEquals(inf.port, 0x3055)
        self.assertEquals(inf.keyinfo, keyid)
        self.assertEquals(inf.pack(), ri)
        self.assertEquals(MMTPHostInfo("the.hostname.is.here", 0x3055,
                                       keyid).pack(), ri)

        self.assertEquals(inf, parseRelayInfoByType(FWD_HOST_TYPE,inf.pack()))
        self.assertEquals(inf, parseRelayInfoByType(SWAP_FWD_HOST_TYPE,inf.pack()))

        self.failUnlessRaises(ParseError, parseMMTPHostInfo, "z")
        self.failUnlessRaises(ParseError, parseMMTPHostInfo, "\x30\x55"+keyid)
        self.failUnlessRaises(ParseError, parseMMTPHostInfo,
                              "\x30\x55"+keyid+"_.com")


    def test_replyblock(self):
        # Try parsing an example 'reply block' object
        key = "\x99"*16
        r = ("SURB\x00\x01"+"\x00\x00\x00\x00"+("Z"*2048)+"\x00\x0A"+"\x00\x01"
             +key+("F"*10))
        rb = parseReplyBlock(r)
        self.assertEquals(rb.timestamp, 0)
        self.assertEquals(rb.header, "Z"*2048)
        self.assertEquals(rb.routingType, 1)
        self.assertEquals(rb.routingInfo, "F"*10)
        self.assertEquals(rb.encryptionKey, key)
        self.assertEquals(r, rb.pack())
        # Now try regenerating it.
        rb = ReplyBlock(header="Z"*2048,useBy=0,rt=1,ri="F"*10,key=key)
        self.assertEquals(r, rb.pack())

        # Now try two blocks.
        r += ("SURB\x00\x01"+"\x00\x00\x00\x00"+("Z"*2048)+"\x00\x0A"+
              "\x00\x01"
             +key+("G"*10))
        rb = parseReplyBlocks(r)
        self.assertEquals(2, len(rb))
        self.assertEquals(rb[0].timestamp, 0)
        self.assertEquals(rb[1].routingInfo, "G"*10)

    def test_payloads(self):
        # Checks for payload structure functions.

        # First, generate some plausible singleton payloads.
        contents = ("payload"*(4*1024))[:28*1024 - 22]
        digest = "HASH"*5
        singleton_payload_1 = "\x00\xff"+digest+contents
        singleton_payload_2 = singleton_payload_1[:-38] #efwd overhead
        # Make sure that parsePayload works as expected.
        p1 = parsePayload(singleton_payload_1)
        p2 = parsePayload(singleton_payload_2)
        self.failUnless(p1.isSingleton() and p2.isSingleton())
        self.assertEquals(p1.size,255)
        self.assertEquals(p2.size,255)
        self.assertEquals(p1.hash,digest)
        self.assertEquals(p2.hash,digest)
        self.assertEquals(p1.data,contents)
        self.assertEquals(p2.data,contents[:-38])
        self.assertEquals(p1.getContents(), contents[:255])
        self.assertEquals(p2.getContents(), contents[:255])
        self.assertEquals(p1.pack(),singleton_payload_1)
        self.assertEquals(p2.pack(),singleton_payload_2)

        # Try SingletonPayload constructor and pack functions
        self.assertEquals(singleton_payload_1,
                          SingletonPayload(255, digest, contents).pack())
        self.assertEquals(singleton_payload_2,
                          SingletonPayload(255, digest, contents[:-38]).pack())

        # Impossible payload lengths
        self.failUnlessRaises(ParseError,parsePayload,singleton_payload_1+"a")
        self.failUnlessRaises(ParseError,parsePayload,singleton_payload_2+"a")
        self.failUnlessRaises(ParseError,parsePayload,singleton_payload_2[:-1])
        # Impossible value for size field
        bad = "\x7fff" + singleton_payload_1[2:]
        self.failUnlessRaises(ParseError,parsePayload,bad)

        ## Now, for the fragment payloads.
        msgID = "This is a message123"
        assert len(msgID) == 20
        contents = contents[:28*1024 - 47]
        frag_payload_1 = "\x80\x00\x02"+digest+msgID+"\x00\x01\x00\x00"+contents
        frag_payload_2 = frag_payload_1[:-38] # efwd overhead
        p1 = parsePayload(frag_payload_1)
        p2 = parsePayload(frag_payload_2)
        self.failUnless(not p1.isSingleton() and not p2.isSingleton())
        self.assertEquals(p1.index,2)
        self.assertEquals(p2.index,2)
        self.assertEquals(p1.hash,digest)
        self.assertEquals(p2.hash,digest)
        self.assertEquals(p1.msgID,msgID)
        self.assertEquals(p2.msgID,msgID)
        self.assertEquals(p1.msgLen,64*1024)
        self.assertEquals(p2.msgLen,64*1024)
        self.assertEquals(p1.data,contents)
        self.assertEquals(p2.data,contents[:-38])
        self.assertEquals(p1.pack(),frag_payload_1)
        self.assertEquals(p2.pack(),frag_payload_2)

        self.assertEquals(frag_payload_1,
                  FragmentPayload(2,digest,msgID,64*1024,contents).pack())
        self.assertEquals(frag_payload_2,
                  FragmentPayload(2,digest,msgID,64*1024,contents[:-38]).pack())

        # Impossible payload lengths
        self.failUnlessRaises(ParseError,parsePayload,frag_payload_1+"a")
        self.failUnlessRaises(ParseError,parsePayload,frag_payload_2+"a")
        self.failUnlessRaises(ParseError,parsePayload,frag_payload_2[:-1])

        # Impossible message sizes
        min_payload_1 = "\x80\x00\x02"+digest+msgID+"\x00\x00\x6F\xD2"+contents
        bad_payload_1 = "\x80\x00\x02"+digest+msgID+"\x00\x00\x6F\xD1"+contents
        min_payload_2 = "\x80\x00\x02"+digest+msgID+"\x00\x00\x6F\xAC"+contents[:-38]
        bad_payload_2 = "\x80\x00\x02"+digest+msgID+"\x00\x00\x6F\xAB"+contents[:-38]
        min_payload_3 = "\x80\x00\x02"+digest+msgID+"\x00\x00\x6F\xD1"+contents[:-38]
        parsePayload(min_payload_1)
        parsePayload(min_payload_2)
        parsePayload(min_payload_3)
        self.failUnlessRaises(ParseError,parsePayload,bad_payload_1)
        self.failUnlessRaises(ParseError,parsePayload,bad_payload_2)

    def testTextEncodedMessage(self):
        tem = TextEncodedMessage
        ptem = parseTextEncodedMessages
        eq = self.assertEquals
        start = "-----BEGIN TYPE III ANONYMOUS MESSAGE-----\n"
        pt = "Message-type: plaintext\n"
        end =   "-----END TYPE III ANONYMOUS MESSAGE-----\n"

        # Test generation: text case
        mt1 = tem("Hello, whirled","TXT")
        eq(mt1.pack(), start+pt+"\nHello, whirled\n"+end)
        mt2 = tem("Hello, whirled\n", "TXT")
        eq(mt2.pack(), start+pt+"\nHello, whirled\n"+end)
        mt3 = tem("Decoding-handle: gotcha!\nFoobar\n", "TXT")
        eq(mt3.pack(), start+pt+"\nDecoding-handle: gotcha!\nFoobar\n"+end)
        # Text generation: binary case
        v = hexread("00D1E50FED1F1CE5")*12
        v64 = encodeBase64(v)
        mb1 = tem(v, "BIN")
        eq(mb1.pack(), start+"""\
Message-type: binary

ANHlD+0fHOUA0eUP7R8c5QDR5Q/tHxzlANHlD+0fHOUA0eUP7R8c5QDR5Q/tHxzl
ANHlD+0fHOUA0eUP7R8c5QDR5Q/tHxzlANHlD+0fHOUA0eUP7R8c5QDR5Q/tHxzl
"""+end)
        eq(mb1.pack(), start+"Message-type: binary\n\n"+v64+end)
        # Overcompressed
        ml1 = tem(v, "LONG")
        eq(ml1.pack(), start+"Message-type: overcompressed\n\n"+v64+end)
        # Encoded
        menc1 = tem(v, "ENC", "9"*20)
        tag64 = base64.encodestring("9"*20).strip()
        eq(menc1.pack(), start+
           "Message-type: encrypted\nDecoding-handle: "+tag64+"\n\n"+v64+end)

        # Test parsing: successful cases
        p = ptem(mt1.pack())[0]
        eq(p.pack(), mt1.pack())
        eq(p.getContents(), "Hello, whirled\n")
        self.assert_(p.isText())
        p = ptem("This message is a test of the emergent broadcast system?\n"
                 +mt2.pack())[0]
        eq(p.pack(), mt2.pack())
        eq(p.getContents(), "Hello, whirled\n")
        # Two concatenated message.
        s = mb1.pack() + "\n\n" + ml1.pack()
        p, p2 = ptem(s)
        eq(p.pack(), mb1.pack())
        self.assert_(p.isBinary())
        eq(p.getContents(), v)
        eq(p2.pack(), ml1.pack())
        self.assert_(p2.isOvercompressed())
        eq(p2.getContents(), v)
        # An encoded message
        p = ptem(menc1.pack())[0]
        eq(p.pack(), menc1.pack())
        eq(p.getContents(), v)
        self.assert_(p.isEncrypted())
        eq(p.getTag(), "9"*20)

    def testHeaders(self):
        emh = encodeMessageHeaders
        eMh = encodeMailHeaders
        pmh = parseMessageAndHeaders
        eq = self.assertEquals

        # Nil headers
        eq(emh({}), "\n")

        # Parse and unparse headers
        encoded = emh({"ABC": "x y zzy", "X": "<42>"}) + "Hello whirled"
        eq(encoded, "ABC:x y zzy\nX:<42>\n\nHello whirled")
        m,h = pmh(encoded)
        eq(m, "Hello whirled")
        eq(h, {"ABC": "x y zzy", "X": "<42>"})

        # Handle leading newlines properly
        encoded = emh({}) + "A short message"
        eq(encoded, "\nA short message")
        eq(("A short message", {}), pmh(encoded))
        eq(("\nA short message", {}), pmh("\n"+encoded))

        # Parse headers with problems.
        try:
            suspendLog()
            m,h = pmh("A message that doesn't start with a header")
            eq(m, "A message that doesn't start with a header")
            eq(h, {})
            # Over-long header gets skipped.
            encoded = "X:"+("Y"*1000)+"\nY:X\n\nZ"
            m,h = pmh(encoded)
            eq(h, {"Y": "X"})
            eq(m, "Z")
        finally:
            resumeLog()

        eq("\n", eMh())
        eq("SUBJECT:foo\n\n", eMh(subject='foo'))
        eq("FROM:fred@bluto\n\n", eMh(fromAddr='fred@bluto'))
        eq("FROM:a\nIN-REPLY-TO:b\nREFERENCES:c\nSUBJECT:d\n\n",
           eMh(fromAddr='a',inReplyTo='b',references='c',subject='d'))
        self.assertRaises(MixError, eMh, fromAddr='fred"')
        self.assertRaises(MixError, eMh, subject=("X"*910))

#----------------------------------------------------------------------
class HashLogTests(TestCase):
    def test_hashlog(self):
        # Create a new,empty hashlog.
        fname = mix_mktemp(".db")

        # (We put the hashlog in a list so that we can pass the list to our
        #  internal helper functions, and change its contents later on.  If
        #  only we could rely on nested scopes (added to Python 2.1), this
        #  would be easier.)
        h = [HashLog(fname, "Xyzzy")]

        notseen=lambda hash_,self=self,h=h:self.assert_(not h[0].seenHash(hash_))
        seen = lambda hash_,self=self,h=h: self.assert_(h[0].seenHash(hash_))
        log = lambda hash_,h=h: h[0].logHash(hash_)

        # Make sure that an empty hash contains nothing, including NUL strings
        # and high-ascii strings.
        notseen("a"*20)
        notseen("\000"*20)
        notseen("\277"*20)
        # Log a value, and make sure that only that value is now in the log
        log("a"*20)
        notseen("\000"*20)
        notseen("b"*20)
        seen("a"*20)

        # Try a second value; make sure both values are now there.
        log("b"*20)
        seen("b"*20)
        seen("a"*20)

        # Try logging a string of NULs
        log("\000"*20)
        seen("\000"*20)
        notseen("\001"*20)
        notseen(("\000"*19)+"\001")
        notseen("\001"+("\000"*19))

        # Try double-logging.
        log("\000"*20)
        seen("\000"*20)

        # Try logging a string of ESCs
        log("\277"*20)
        seen("\277"*20)

        # And a nice plain ascii string
        log("abcde"*4)
        seen("abcde"*4)
        seen("a"*20)
        h[0].sync()
        seen("a"*20)

        # Now reopen the log, and make sure it has all its original contents.
        h[0].close()
        h[0] = HashLog(fname, "Xyzzy")
        seen("a"*20)
        seen("b"*20)
        seen("\277"*20)
        seen("abcde"*4)
        seen("\000"*20)
        # (and no other contents)
        notseen(" "*20)
        notseen("\001"*20)

        # Now add more, close again, and see if our latest adddition went in.
        notseen("ddddd"*4)
        log("ddddd"*4)
        seen("ddddd"*4)

        h[0].sync()
        h[0].close()
        h[0] = HashLog(fname, "Xyzzy")
        seen("ddddd"*4)

        # Make sure that hashlog still works when we sync.
        log("Abcd"*5)
        log("Defg"*5)
        seen("Abcd"*5)
        seen("Defg"*5)
        h[0].sync()
        #   (This violates HashLog's encapsulation, but let's make double-sure
        #    that we've really flushed the journal to disk.)
        self.assertEquals(0, len(h[0].journal))
        seen("Abcd"*5)
        seen("Defg"*5)
        log("Ghij"*5)
        seen("Ghij"*5)

        h[0].close()

#----------------------------------------------------------------------
class NetUtilTests(TestCase):
    def testGetIP(self):
        overridedict = {}
        if hasattr(socket, 'getaddrinfo'):
            def override_getaddrinfo(name,port,overridedict=overridedict):
                v = overridedict.get(name)
                if v:
                    r = []
                    for addr in v:
                        if '.' in addr:
                            r.append( (mixminion.NetUtils.AF_INET, -1, -1, name, (addr,port) ) )
                        else:
                            r.append( (mixminion.NetUtils.AF_INET6, -1, -1, name, (addr,port)) )
                    return r
                else:
                    raise socket.error, "foo"
            replaceAttribute(socket, "getaddrinfo", override_getaddrinfo)
        else:
            def override_gethostbyname(name,overridedict=overridedict):
                v = overridedict.get(name)
                if v:
                    return v[0]
                else:
                    raise socket.error, "foo"
            replaceAttribute(socket, "gethostbyname", override_gethostbyname)

        overridedict['revolving.restaurant'] = [ '30.1.0.50', "00FE::3" ]
        now = time.time()
        try:
            r = mixminion.NetUtils.getIPs('revolving.restaurant')

            self.assertEquals((socket.AF_INET, "30.1.0.50"), r[0][:2])
            self.assertFloatEq(now, r[0][2])

            if hasattr(socket, 'getaddrinfo'):
                self.assertEquals(2, len(r))
                self.assertEquals((mixminion.NetUtils.AF_INET6, "00FE::3"), r[1][:2])
                self.assertFloatEq(now, r[1][2])
                self.assertEquals((socket.AF_INET, "30.1.0.50"),
                       mixminion.NetUtils.getIP("revolving.restaurant",1)[:2])
                self.assertEquals((mixminion.NetUtils.AF_INET6, "00FE::3"),
                       mixminion.NetUtils.getIP("revolving.restaurant",0)[:2])
            else:
                self.assertEquals(1, len(r))
                self.assertEquals((socket.AF_INET, "30.1.0.50"),
                       mixminion.NetUtils.getIP("revolving.restaurant",0)[:2])
                self.assertEquals((socket.AF_INET, "30.1.0.50"),
                       mixminion.NetUtils.getIP("revolving.restaurant",1)[:2])

            self.assertRaises(socket.error, mixminion.NetUtils.getIPs,
                              "nowhere.nowhen.nohow")
            self.assertEquals(("NOENT", "foo"),
                      mixminion.NetUtils.getIP("nowhere.nowhen.nohow")[:2])
        finally:
            undoReplacedAttributes()

    def testGetProtocolSupport(self):
        ps = mixminion.NetUtils.getProtocolSupport()
        self.assertEquals(len(ps),2)
        self.assertEquals(ps[0], 1) # IPv4 support is required.
        self.assert_(ps[1] in (0,1))

    def testNameIsStaticIP(self):
        nisi = mixminion.NetUtils.nameIsStaticIP
        from mixminion.NetUtils import AF_INET, AF_INET6
        self.assertEquals(nisi("foo"), None)
        self.assertEquals(nisi("18.244.0.0")[:2], (AF_INET, "18.244.0.0"))
        self.assertEquals(nisi("::F00f")[:2], (AF_INET6, "::F00f"))
        self.assertEquals(nisi("4starts-with-digit.tld"), None)
        self.assertEquals(nisi("bogus-with-a-colon:wow"), None)

#----------------------------------------------------------------------

# Dummy PRNG class that just returns 0-valued bytes.  We use this to make
# generated padding predictable in our BuildMessage tests below.
class FakePRNG:
    def getBytes(self,n):
        return "\x00"*n

class FakeServerInfo:
    """Represents a Mixminion server, and the information needed to send
       messages to it."""
    def __init__(self, addr, port, key, keyid, ip4=0):
        assert key.get_modulus_bytes() == 256
        self.addr = addr
        self.port = port
        self.key = key
        self.keyid = keyid
        self.ip4 = ip4

    def getNickname(self): return "N(%s:%s)"%(self.addr,self.port)
    def getIP(self): return self.addr
    def getPort(self): return self.port
    def getPacketKey(self): return self.key
    def getKeyDigest(self): return self.keyid
    def supportsPacketVersion(self): return 1

    def getRoutingInfo(self):
        if self.ip4:
            return self.getIPV4Info()
        else:
            return self.getMMTPHostInfo()
    def getIPV4Info(self):
        return IPV4Info(self.addr, self.port, self.keyid)
    def getMMTPHostInfo(self):
        return MMTPHostInfo(self.addr, self.port, self.keyid)
    def getRoutingFor(self,other,swap):
        if other.ip4:
            tp = [FWD_IPV4_TYPE,SWAP_FWD_IPV4_TYPE][swap]
        else:
            tp = [FWD_HOST_TYPE,SWAP_FWD_HOST_TYPE][swap]
        return (tp, other.getRoutingInfo().pack())


class BuildMessageTests(TestCase):
    def setUp(self):
        self.pk1 = getRSAKey(0,2048)
        self.pk2 = getRSAKey(1,2048)
        self.pk3 = getRSAKey(2,2048)
        self.pk512 = getRSAKey(0,512)
        self.server1 = FakeServerInfo("127.0.0.1", 1, self.pk1, "X"*20)
        self.server2 = FakeServerInfo("127.0.0.2", 3, self.pk2, "Z"*20)
        self.server3 = FakeServerInfo("127.0.0.3", 5, self.pk3, "Q"*20)

    def test_compression(self):
        # Make sure that our compression helper functions work properly.

        p = AESCounterPRNG()
        longMsg = p.getBytes(100)*2 + str(dir(Crypto))

        # Make sure compression is reversible.
        for m in ("", "a", "\000", "xyzzy"*10, ("glossy glossolalia.."*2)[32],
                  longMsg):
            c = BuildMessage.compressData(m)
            self.assertEquals(m, uncompressData(c))

        self.failUnlessRaises(ParseError, uncompressData, "3")

        for _ in xrange(20):
            for _ in xrange(20):
                m = p.getBytes(p.getInt(1000))
                try:
                    uncompressData(m)
                except ParseError:
                    pass
        #FFFF Find a decent test vector.

    def test_payload_helpers(self):
        "test helpers for payload encoding"

        p = AESCounterPRNG()
        for _ in xrange(10):
            t = BuildMessage._getRandomTag(p)
            self.assertEquals(20, len(t))
            self.assertEquals(0, ord(t[0])&0x80)

        b = p.getBytes(28*1024)
        self.assert_(not BuildMessage._checkPayload(b))

        for m in (p.getBytes(3000), p.getBytes(10000), "", "q", "blznerty"):
            for ov in 0, 42-20+16: # encrypted forward overhead
                plds = BuildMessage.encodeMessage(m,ov,"",p)
                assert len(plds) == 1
                assert BuildMessage.getNPacketsToEncode(m,ov,"") == 1
                pld = plds[0]
                self.assertEquals(28*1024, len(pld)+ov)
                comp = compressData(m)
                self.assertStartsWith(pld[22:], comp)
                self.assertEquals(sha1(pld[22:]),pld[2:22])
                self.assert_(BuildMessage._checkPayload(pld))
                self.assertEquals(len(comp), ord(pld[0])*256+ord(pld[1]))
                self.assertEquals(0, ord(pld[0])&0x80)
                py = mixminion.Packet.parsePayload(pld)
                self.assertEquals(m, py.getUncompressedContents())

        self.failUnlessRaises(MixError, BuildMessage._decodeForwardPayload, b)

        # Check impossibly long messages
        pldSize = "\x7f\xff"+pld[2:] #sha1(pld[22:])+pld[22:]
        self.failUnlessRaises(ParseError,
                              BuildMessage._decodeForwardPayload,pldSize)

    def test_buildheader_1hop(self):
        bhead = BuildMessage._buildHeader

        head = bhead([self.server1], ["9"*16], 99, "Hi mom", AESCounterPRNG())
        self.do_header_test(head,
                            (self.pk1,),
                            ["9"*16,],
                            (99,),
                            ("Hi mom",))

    def test_buildheader_2hops(self):
        bhead = BuildMessage._buildHeader
        # 2 hops
        head = bhead([self.server1, self.server2],
                     ["9"*16, "1"*16], 99, "Hi mom", AESCounterPRNG())

        host = mixminion.Packet.MMTPHostInfo
        self.do_header_test(head,
                            (self.pk1, self.pk2),
                            ["9"*16, "1"*16],
                            (FWD_HOST_TYPE, 99),
                            (host("127.0.0.2",3,"Z"*20).pack(),
                             "Hi mom"))

    def test_buildheader_3hops(self):
        bhead = BuildMessage._buildHeader
        # 3 hops
        secrets = ["9"*16, "1"*16, "z"*16]
        head = bhead([self.server1, self.server2, self.server3], secrets,
                      99, "Hi mom", AESCounterPRNG())
        pks = (self.pk1,self.pk2,self.pk3)
        rtypes = (FWD_HOST_TYPE, FWD_HOST_TYPE, 99)
        rinfo = (mixminion.Packet.MMTPHostInfo("127.0.0.2", 3, "Z"*20).pack(),
                 mixminion.Packet.MMTPHostInfo("127.0.0.3", 5, "Q"*20).pack(),
                 "Hi mom")
        self.do_header_test(head, pks, secrets, rtypes, rinfo)

    def do_header_test(self, head, pks, secrets, rtypes, rinfo):
        """Unwraps and checks the layers of a single header.
                    head: the header to check
                    pks: sequence of public keys for hops in the path
                    secrets: sequence of master secrets for the subheaders
                    rtypes: sequenece of expected routing types
                    rinfo: sequenece of expected routing info's.

           If secrets is None, takes secrets from the headers without
             checking.

           Returns a tuple of (a list of the secrets encountered,
                               a list of routinginfo strings,
                               the tag from the last routinginfo)
        """
        tag = None
        retsecrets = []
        retinfo = []
        if secrets is None:
            secrets = [None] * len(pks)
        # Is the header the right length?
        self.assertEquals(len(head), mixminion.Packet.HEADER_LEN)
        # Go through the hops one by one, simulating the decoding process.
        for pk, secret, rt, ri in zip(pks, secrets,rtypes,rinfo):
            # Decrypt the first subheader.
            subh = mixminion.Packet.parseSubheader(pk_decrypt(head[:256], pk))
            # If we're expecting a given secret in this subheader, check it.
            if secret:
                self.assertEquals(subh.secret, secret)
            else:
                secret = subh.secret
            retsecrets.append(secret)
            # Check the version, the digest, and the routing type.
            self.assertEquals(subh.major, mixminion.Packet.MAJOR_NO)
            self.assertEquals(subh.minor, mixminion.Packet.MINOR_NO)
            self.assertEquals(subh.digest, sha1(head[256:]))
            self.assertEquals(subh.routingtype, rt)

            # Key to decrypt the rest of the header
            ks = Keyset(secret)
            key = ks.get(HEADER_SECRET_MODE)

            # If we have an exit type, the first 20 bytes of the routinginfo
            # are a decoding tag; extract it.
            if rt < 0x100: # extra bytes for tag
                ext = 0
            else:
                ext = 20
                if ri:
                    tag = subh.routinginfo[:20]

            # Check the routinginfo.  This is a little different for regular
            # and extended subheaders...
            if not subh.getOverflowLength():
                if ri:
                    self.assertEquals(subh.routinginfo[ext:], ri)
                    self.assertEquals(subh.routinglen, len(ri)+ext)
                else:
                    retinfo.append(subh.routinginfo)
                size = 256
            else:
                more = ctr_crypt(head[256:256+subh.getOverflowLength()], key)
                self.assert_(len(ri+more)>
                             mixminion.Packet.MAX_ROUTING_INFO_LEN)
                subh.appendOverflow(more)
                size = 256 + subh.getOverflowLength()
                if ri:
                    self.assertEquals(subh.routinginfo[ext:], ri)
                    self.assertEquals(subh.routinglen, len(ri)+ext)
                else:
                    retinfo.append(subh.routinginfo)

            # Decrypt and pad the rest of the header.
            prngkey = ks.get(RANDOM_JUNK_MODE)
            head = ctr_crypt(head[size:]+
                       prng(prngkey,size-len(subh.underflow)), key, size-256)
            head = subh.underflow + head
            self.assertEquals(len(head), 2048)

        return retsecrets, retinfo, tag

    def test_header_overflow(self):
        bhead = BuildMessage._buildHeader

        secrets = ["9"*16]
        longStr = "Foo"*100
        head = bhead([self.server1], secrets, 99, longStr, AESCounterPRNG())
        pks = (self.pk1,)
        rtypes = (99,)
        rinfo = (longStr,)

        self.do_header_test(head, pks, secrets, rtypes, rinfo)

        # Now try a header with extended **intermediate** routing info.
        # Since this never happens in the wild, we fake it.
        tag = "dref"*5
        longStr2 = "BAR"*100

        def getLongRoutingInfo(longStr2=longStr2,tag=tag):
            return MBOXInfo(tag+longStr2)

        server4 = FakeServerInfo("127.0.0.1", 1, self.pk1, "X"*20)
        server4.getRoutingInfo = getLongRoutingInfo

        secrets.append("1"*16)
        head = bhead([self.server2, server4], secrets, 99,
                     longStr,
                     AESCounterPRNG())
        pks = (self.pk2,self.pk1)
        rtypes = (FWD_HOST_TYPE,99)
        rinfo = (tag+longStr2,longStr)
        self.do_header_test(head, pks, secrets, rtypes, rinfo)

        # Now we make sure that overlong routing info fails.
        self.failUnlessRaises(MixError, bhead,
                              [self.server2, server4], secrets, 99, "Z"*2048,
                              AESCounterPRNG())

    def test_constructmessage(self):
        consMsg = BuildMessage._constructMessage

        h1 = "abcdefgh"*(2048 >> 3)
        h2 = "aBcDeFgH"*(2048 >> 3)

        ######
        ### non-reply case
        secrets1 = [ x * 16 for x in ("s","q","m","s","h")]
        secrets2 = [ x * 16 for x in ("o","s","f","r","g")]
        pld = """
           Everyone has the right to freedom of opinion and expression; this
           right includes freedom to hold opinions without interference and
           to seek, receive and impart information and ideas through any
           media and regardless of frontiers.
           """
        pld += "\000"*(28*1024-len(pld))

        message = consMsg(secrets1, secrets2, h1, h2, pld)

        self.assertEquals(len(message), mixminion.Packet.PACKET_LEN)
        msg = mixminion.Packet.parsePacket(message)
        head1, head2, payload = msg.header1, msg.header2, msg.payload
        self.assert_(h1 == head1)

        for path in secrets1, secrets2:
            for s in path:
                ks = Keyset(s)
                hkey = ks.getLionessKeys(HEADER_ENCRYPT_MODE)
                pkey = ks.getLionessKeys(PAYLOAD_ENCRYPT_MODE)
                if path is secrets1:
                    head2 = lioness_decrypt(head2, hkey)
                payload = lioness_decrypt(payload, pkey)

            if path is secrets1:
                swapkey = Crypto.lioness_keys_from_header(head2)
                payload = lioness_decrypt(payload, swapkey)

                swapkey = Crypto.lioness_keys_from_payload(payload)
                head2 = lioness_decrypt(head2, swapkey)

        self.assert_(head2 == h2)
        self.assert_(payload == pld)

        ######
        ### Reply case
        message = consMsg(secrets1, None, h1, h2, pld)
        self.assertEquals(len(message), mixminion.Packet.PACKET_LEN)
        msg = mixminion.Packet.parsePacket(message)
        head1, head2, payload = msg.header1, msg.header2, msg.payload
        self.assert_(h1 == head1)

        for s in secrets1:
            ks = Keyset(s)
            hkey = ks.getLionessKeys(HEADER_ENCRYPT_MODE)
            pkey = ks.getLionessKeys(PAYLOAD_ENCRYPT_MODE)
            head2 = lioness_decrypt(head2, hkey)
            payload = lioness_decrypt(payload, pkey)

        swapkey = Crypto.lioness_keys_from_header(head2)
        payload = lioness_decrypt(payload, swapkey)

        swapkey = Crypto.lioness_keys_from_payload(payload)
        head2 = lioness_decrypt(head2, swapkey)

        self.assert_(head2 == h2)
        self.assert_(payload == pld)

    def do_message_test(self, msg,
                        header_info_1,
                        header_info_2,
                        payload, decoder=None):
        """Decrypts the layers of a message one by one, checking them for
           correctness.
                      msg: the message to examine
                      header_info_1: a tuple of (pks,secrets,rtypes,rinfo)
                            as used by do_header_test for the first header.
                      header_info_2: a tuple of (pks,secrets,rtypes,rinfo)
                            as used by do_header_test for the second header.
                      payload: The beginning of the expected decrypted payload.
                      decoder: A function to call on the exit payload before
                            comparing it to 'payload'.  Takes payload,tag;
                            returns string.
           """
        # Check header 1, and get secrets
        sec, _, _ = self.do_header_test(msg[:2048], *header_info_1)
        h2 = msg[2048:4096]
        p = msg[4096:]
        # Do decryption steps for header 1.
        for s in sec:
            ks = Keyset(s)
            p = lioness_decrypt(p,ks.getLionessKeys(PAYLOAD_ENCRYPT_MODE))
            h2 = lioness_decrypt(h2,ks.getLionessKeys(HEADER_ENCRYPT_MODE))
        p = lioness_decrypt(p,Crypto.lioness_keys_from_header(h2))
        h2 = lioness_decrypt(h2,Crypto.lioness_keys_from_payload(p))

        sec, _, tag = self.do_header_test(h2, *header_info_2)
        for s in sec:
            ks = Keyset(s)
            p = lioness_decrypt(p,ks.getLionessKeys(PAYLOAD_ENCRYPT_MODE))

        if decoder is None:
            p = BuildMessage._decodeForwardPayload(p).getUncompressedContents()
        else:
            p = decoder(p, tag)

        self.assertEquals(payload, p[:len(payload)])

    def test_build_fwd_message(self):

        bfm = BuildMessage.buildForwardPacket
        befm = BuildMessage.buildEncryptedForwardPacket
        payloadF = BuildMessage.encodeMessage("Hello!!!!",0)[0]

        m = bfm(payloadF, 500, "Goodbye",
                [self.server1, self.server2],
                [self.server3, self.server2])

        self.do_message_test(m,
                             ( (self.pk1, self.pk2), None,
                               (FWD_HOST_TYPE, SWAP_FWD_HOST_TYPE),
                               (self.server2.getRoutingInfo().pack(),
                                self.server3.getRoutingInfo().pack()) ),
                             ( (self.pk3, self.pk2), None,
                               (FWD_HOST_TYPE, 500),
                               (self.server2.getRoutingInfo().pack(),
                                "Goodbye") ),
                             "Hello!!!!")
        m = bfm(payloadF, 500, "Goodbye", [self.server1], [self.server3])

        messages = {}

        def decoder0(p,t,messages=messages):
            messages['fwd'] = (p,t)
            return BuildMessage._decodeForwardPayload(p).getUncompressedContents()

        self.do_message_test(m,
                             ( (self.pk1,), None,
                               (SWAP_FWD_HOST_TYPE,),
                               (self.server3.getRoutingInfo().pack(),) ),
                             ( (self.pk3,), None,
                               (500,),
                               ("Goodbye",) ),
                             "Hello!!!!",
                             decoder=decoder0)

        # Drop message gets no tag, random payload
        m = bfm(BuildMessage.buildRandomPayload(),
                DROP_TYPE, "", [self.server1], [self.server3])

        def decoderDrop(p,t,self=self):
            self.assertEquals(None, t)
            self.failIf(BuildMessage._checkPayload(p))
            return ""

        self.do_message_test(m,
                             ( (self.pk1,), None,
                               (SWAP_FWD_HOST_TYPE,),
                               (self.server3.getRoutingInfo().pack(),) ),
                             ( (self.pk3,), None,
                               (DROP_TYPE,),
                               ("",) ),
                             "",
                             decoder=decoderDrop)

        # Encrypted forward message
        rsa1, rsa2 = self.pk1, self.pk512
        payloadE = BuildMessage.encodeMessage(
            "<<<<Hello>>>>"*100,ENC_FWD_OVERHEAD)[0]
        assert BuildMessage.getNPacketsToEncode("<<<<Hello>>>>"*100,
                                                ENC_FWD_OVERHEAD) == 1
        for rsakey in rsa1,rsa2:
            m = befm(payloadE, 500, "Phello",
                     [self.server1, self.server2],
                     [self.server3, self.server2],
                     rsakey)
            def decoder(p,t,key=rsakey,messages=messages):
                messages['efwd'+str(key.get_modulus_bytes())] = (p,t)
                payload = BuildMessage._decodeEncryptedForwardPayload(p,t,key)
                return payload.getUncompressedContents()
            self.do_message_test(m,
                                 ( (self.pk1, self.pk2), None,
                                   (FWD_HOST_TYPE, SWAP_FWD_HOST_TYPE),
                                   (self.server2.getRoutingInfo().pack(),
                                    self.server3.getRoutingInfo().pack()) ),
                                 ( (self.pk3, self.pk2), None,
                                   (FWD_HOST_TYPE, 500),
                                   (self.server2.getRoutingInfo().pack(),
                                    "Phello") ),
                                 "<<<<Hello>>>>"*100,
                                 decoder=decoder)

        # Now do more tests on final messages: is the format as expected?
        p,t = messages['fwd']
        self.assertEquals(20, len(t))
        self.assertEquals(28*1024,len(p))
        self.assertEquals(0, ord(t[0]) & 0x80)
        comp = compressData("Hello!!!!")
        self.assertEquals(len(comp), ord(p[0])*256 +ord(p[1]))
        self.assertStartsWith(p[22:], comp)
        self.assertEquals(sha1(p[22:]), p[2:22])

        for rsakey in (rsa1, rsa2):
            n = rsakey.get_modulus_bytes()
            p,t = messages['efwd'+str(n)]
            mrsa, mrest = t+p[:n-20], p[n-20:]
            mrsa = pk_decrypt(mrsa, rsakey)
            sessionkey, rsa_rest = mrsa[:16], mrsa[16:]
            ks = Keyset(sessionkey)
            msg = rsa_rest + lioness_decrypt(mrest,
                              ks.getLionessKeys("END-TO-END ENCRYPT"))
            comp = compressData("<<<<Hello>>>>"*100)
            self.assert_(len(comp), ord(msg[0])*256 + ord(msg[1]))
            self.assertEquals(sha1(msg[22:]), msg[2:22])
            self.assertStartsWith(msg[22:], comp)

    def test_buildreply(self):
        brbi = BuildMessage._buildReplyBlockImpl
        brb = BuildMessage.buildReplyBlock
        brm = BuildMessage.buildReplyPacket

        ## Stateful reply blocks.
        reply, secrets_1, tag_1 = \
             brbi([self.server3, self.server1, self.server2,
                  self.server1, self.server3],
                 SMTP_TYPE,
                 "no-such-user@invalid", tag=("+"*20))
        hsecrets = secrets_1[:-1]
        hsecrets.reverse()

        pks_1 = (self.pk3, self.pk1, self.pk2, self.pk1, self.pk3)
        infos = (self.server1.getRoutingInfo().pack(),
                 self.server2.getRoutingInfo().pack(),
                 self.server1.getRoutingInfo().pack(),
                 self.server3.getRoutingInfo().pack())

        self.assert_(reply.routingInfo == self.server3.getRoutingInfo().pack())

        m = brm(BuildMessage.encodeMessage("Information???",0)[0],
                [self.server3, self.server1],
                reply)

        messages = {}
        def decoder(p,t,secrets=secrets_1,messages=messages):
            messages['repl'] = p,t
            payload = BuildMessage._decodeReplyPayload(p,secrets)
            return payload.getUncompressedContents()

        self.do_message_test(m,
                             ((self.pk3, self.pk1), None,
                              (FWD_HOST_TYPE,SWAP_FWD_HOST_TYPE),
                              (self.server1.getRoutingInfo().pack(),
                               self.server3.getRoutingInfo().pack())),
                             (pks_1, hsecrets,
                              (FWD_HOST_TYPE,FWD_HOST_TYPE,FWD_HOST_TYPE,FWD_HOST_TYPE,SMTP_TYPE),
                              infos+("no-such-user@invalid",)),
                             "Information???",
                             decoder=decoder)
        ## Stateless replies
        reply = brb([self.server3, self.server1, self.server2,
                      self.server1, self.server3], MBOX_TYPE,
                     "fred", "Tyrone Slothrop", 3)

        sec,(loc,), _ = self.do_header_test(reply.header, pks_1, None,
                            (FWD_HOST_TYPE,FWD_HOST_TYPE,FWD_HOST_TYPE,FWD_HOST_TYPE,MBOX_TYPE),
                            infos+(None,))

        self.assertEquals(loc[20:], "fred")

        # (Test reply block formats)
        self.assertEquals(reply.timestamp, 3)
        self.assertEquals(reply.routingType, SWAP_FWD_HOST_TYPE)
        self.assertEquals(reply.routingInfo,
                          self.server3.getRoutingInfo().pack())
        self.assertEquals(reply.pack(),
                          "SURB\x00\x01\x00\x00\x00\x03"+reply.header+
                         "\x00"+chr(len(self.server3.getRoutingInfo().pack()))+
                          "\x00\x04"+reply.encryptionKey+
                          self.server3.getRoutingInfo().pack())
        self.assertEquals(reply.pack(), parseReplyBlock(reply.pack()).pack())
        txt = reply.packAsText()
        self.assertStartsWith(txt,
            "-----BEGIN TYPE III REPLY BLOCK-----\nVersion: 0.2\n\n")
        self.assertEndsWith(txt,
            "-----END TYPE III REPLY BLOCK-----\n")
        parsed = parseTextReplyBlocks(txt)
        self.assertEquals(1, len(parsed))
        self.assertEquals(reply.pack(), parsed[0].pack())
        parsed2 = parseTextReplyBlocks((txt+"   9999 \n")*2)
        self.assertEquals(2, len(parsed2))
        self.assertEquals(reply.pack(), parsed2[1].pack())

        self.assertEquals([], parseTextReplyBlocks("X"))

        # test failing cases for parseTextReplyBlocks
        def fails(s, p=parseTextReplyBlocks, self=self):
            self.assertRaises(ParseError, p, s)

        fails("-----BEGIN TYPE III REPLY BLOCK-----\n"+
              "Version: 0.2\n\n"+
              "xyz\n"+
              "-----END TYPE III REPLY BLOCK-----\n")

        # Test decoding
        seed = loc[:20]
        prng = AESCounterPRNG(sha1(seed+"Tyrone SlothropGenerate")[:16])
        sec.reverse()
        self.assertEquals(sec, [ prng.getBytes(16) for _ in range(len(sec)) ])

        # _Gravity's Rainbow_, page 258.
        message = '''
              "...Is it any wonder the world's gone insane, with information
            come to the be the only medium of exchange?"
              "I thought it was cigarettes."
              "You dream."
                  -- Gravity's Rainbow, p.258 ''' # " <- for emacs python-mode
        payload = BuildMessage.encodeMessage(message,0)[0]
        m = brm(payload,
                [self.server3, self.server1],
                reply)

        def decoder2(p,t,messages=messages):
            messages['srepl'] = p,t
            payload = BuildMessage._decodeStatelessReplyPayload(p,t,
                                                         "Tyrone Slothrop")
            return payload.getUncompressedContents()

        self.do_message_test(m,
                             ((self.pk3, self.pk1), None,
                              (FWD_HOST_TYPE,SWAP_FWD_HOST_TYPE),
                              (self.server1.getRoutingInfo().pack(),
                               self.server3.getRoutingInfo().pack())),
                             (pks_1, None,
                              (FWD_HOST_TYPE,FWD_HOST_TYPE,FWD_HOST_TYPE,FWD_HOST_TYPE,MBOX_TYPE),
                              infos+("fred",)),
                             message,
                             decoder=decoder2)

        # Now test format of generated messages.
        p,t = messages['repl']
        self.assertEquals(t, tag_1)
        for s in secrets_1:
            ks = Keyset(s)
            p = lioness_encrypt(p, ks.getLionessKeys(
                               Crypto.PAYLOAD_ENCRYPT_MODE))
        comp = compressData('Information???')
        self.assertEquals(len(comp), ord(p[0])*256 +ord(p[1]))
        self.assertStartsWith(p[22:], comp)
        self.assertEquals(sha1(p[22:]), p[2:22])

        p,t = messages['srepl']
        self.assertEquals('\000', sha1(t+"Tyrone SlothropValidate")[-1])
        prng = AESCounterPRNG(sha1(t+"Tyrone SlothropGenerate")[:16])
        for _ in xrange(6): # 5 hops plus end-to-end
            s = prng.getBytes(16)
            ks = Keyset(s)
            p = lioness_encrypt(p, ks.getLionessKeys(
                                      Crypto.PAYLOAD_ENCRYPT_MODE))
        comp = compressData(message)
        self.assertEquals(len(comp), ord(p[0])*256 +ord(p[1]))
        self.assertStartsWith(p[22:], comp)
        self.assertEquals(sha1(p[22:]), p[2:22])

    def assertPayloadDecodesTo(self, decoded, encoded, tag, key, userKeys):
        p = BuildMessage.decodePayload(encoded, tag, key, userKeys)
        if p is None:
            self.assertEquals(decoded, p)
        else:
            self.assertEquals(decoded, p.getUncompressedContents())

    def test_fragments(self):
        msg = Crypto.getCommonPRNG().getBytes(30000)
        prefix = "Prefix!"
        payloads = BuildMessage.encodeMessage(msg, 0, prefix)
        self.assertEquals(len(payloads), 3)
        self.assertEquals(BuildMessage.getNPacketsToEncode(msg, 0, prefix), 3)
        p1 = BuildMessage.decodePayload(payloads[0], "")
        p2 = BuildMessage.decodePayload(payloads[1], "")
        p3 = BuildMessage.decodePayload(payloads[2], "")
        self.assert_(None not in [p1, p2, p3])

    def test_decoding(self):
        # Now we create a bunch of fake payloads and try to decode them.

        # Successful messages:
        payload = "All dreamers and sleepwalkers must pay the price, and "+\
          "even the invisible victim is responsible for the fate of all.\n"+\
          "   -- _Invisible Man_"

        comp = compressData(payload)
        self.assertEquals(len(comp), 109)
        encoded1 = (comp+ "RWE/HGW"*4096)[:28*1024-22]
        encoded1 = '\x00\x6D'+sha1(encoded1)+encoded1

        # Forward message.
        self.assertEquals(payload,
                          BuildMessage._decodeForwardPayload(encoded1).getUncompressedContents())
        # Encoded forward message
        efwd = (comp+"RWE/HGW"*4096)[:28*1024-22-38]
        efwd = '\x00\x6D'+sha1(efwd)+efwd
        rsa1 = getRSAKey(0,1024)
        key1 = Keyset("RWE "*4).getLionessKeys("END-TO-END ENCRYPT")
        efwd_rsa = pk_encrypt(("RWE "*4)+efwd[:70], rsa1)
        efwd_lioness = lioness_encrypt(efwd[70:], key1)
        efwd_t = efwd_rsa[:20]
        efwd_p = efwd_rsa[20:]+efwd_lioness
        self.assertEquals(payload,
             BuildMessage._decodeEncryptedForwardPayload(efwd_p,efwd_t,rsa1).getUncompressedContents())

        # Stateless reply
        tag = "To light my way out\xBE"
        passwd = "out I would have to burn every paper in the briefcase"
        self.assertEquals('\000', sha1(tag+passwd+"Validate")[-1])
        prng = AESCounterPRNG(sha1(tag+passwd+"Generate")[:16])
        secrets2 = [ prng.getBytes(16) for _ in xrange(5) ]
        m = encoded1
        s = secrets2[:]
        s.reverse()
        for k in s:
            key = Keyset(k).getLionessKeys(PAYLOAD_ENCRYPT_MODE)
            m = lioness_decrypt(m,key)
        self.assertEquals(payload,
                     BuildMessage._decodeStatelessReplyPayload(m,tag,passwd).getUncompressedContents())
        repl2, repl2tag = m, tag

        # Okay, now let's try out 'decodePayload' (and thereby test its
        # children).  First, we test all the cases that succeed; or that
        # fail and return None to indicate that another key might decrypt
        # the message.
        decodePayload = BuildMessage.decodePayload
        # fwd
        for pk in (self.pk1, None):
            for p in (passwd, None):
                for tag in ("zzzz"*5, "pzzz"*5):
                    self.assertPayloadDecodesTo(payload,
                                                encoded1, tag, pk, p)

        # efwd
        for p in (passwd, None):
            self.assertPayloadDecodesTo(payload, efwd_p, efwd_t, rsa1, p)
            self.assertPayloadDecodesTo(None, efwd_p, efwd_t, None, p)
            self.assertPayloadDecodesTo(None, efwd_p, efwd_t, self.pk2, p)

        # Stateful replies are disabled.

        # repl (stateless)
        for pk in (self.pk1, None):
            #for sd in (sdict, None): #Stateful replies are disabled
                self.assertPayloadDecodesTo(payload,
                                            repl2, repl2tag, pk, passwd)
                try:
                    suspendLog("INFO")
                    self.assertPayloadDecodesTo(payload, repl2, repl2tag, pk,
                                     userKeys={ "Fred": passwd, "": "z"*20 })
                finally:
                    s = resumeLog()
                self.assert_(stringContains(s,
                               "Decoded reply message to identity 'Fred'"))
                self.assertPayloadDecodesTo(None,
                                            repl2, repl2tag, pk, "Bliznerty")
                self.assertPayloadDecodesTo(None, repl2, repl2tag, pk,
                                   userKeys={ "Fred":"Bliznerty", "":"z"*20})
                self.assertPayloadDecodesTo(None, repl2, repl2tag, pk, None)

        # Try decoding a payload that looks like a zlib bomb.  An easy way to
        # get such a payload is to compress 25K of zeroes.
        nils = "\x00"*(25*1024)
        overcompressed_payload = \
             BuildMessage.encodeMessage(nils, 0)[0]
        p = BuildMessage.decodePayload(overcompressed_payload, "X"*20)
        self.failUnlessRaises(CompressedDataTooLong,
                              p.getUncompressedContents)

        # And now the cases that fail hard.  This can only happen on:
        #   1) *: Hash checks out, but zlib or size is wrong.  Already tested.
        #   2) EFWD: OAEP checks out, but hash is wrong.
        #   3) <omitted; stateful replies are gone>
        #   4) SREPLY: ---.

        # Bad efwd
        efwd_pbad = efwd_p[:-1] + chr(ord(efwd_p[-1])^0xaa)
        self.failUnlessRaises(MixError,
                              BuildMessage._decodeEncryptedForwardPayload,
                              efwd_pbad, efwd_t, rsa1)

        for p in (passwd, None):
            self.failUnlessRaises(MixError, decodePayload,
                                  efwd_pbad, efwd_t, rsa1, p)
            self.assertPayloadDecodesTo(None, efwd_pbad, efwd_t, self.pk2, p)

        # Bad repl
        repl2_bad = repl2[:-1] + chr(ord(repl2[-1])^0xaa)
        self.assertPayloadDecodesTo(None, repl2_bad, repl2tag, None, passwd)

#----------------------------------------------------------------------
# Having tested BuildMessage without using PacketHandler, we can now use
# BuildMessage to see whether PacketHandler is doing the right thing.
#
# (of course, we still need to build failing messages by hand)

class PacketHandlerTests(TestCase):
    def setUp(self):
        self.pk1 = getRSAKey(0,2048)
        self.pk2 = getRSAKey(1,2048)
        self.pk3 = getRSAKey(2,2048)
        self.tmpfile = mix_mktemp(".db")
        h = self.hlog = HashLog(self.tmpfile, "Z"*20)

        self.server1 = FakeServerInfo("127.0.0.1", 1, self.pk1, "X"*20)
        self.server2 = FakeServerInfo("127.0.0.2", 3, self.pk2, "Z"*20)
        self.server3 = FakeServerInfo("127.0.0.3", 5, self.pk3, "Q"*20)
        self.sp1 = PacketHandler([self.pk1], [h])
        self.sp2 = PacketHandler([self.pk2], [h])
        self.sp3 = PacketHandler([self.pk3], [h])
        self.sp2_3 = PacketHandler((self.pk2,self.pk3), (h,h))

    def tearDown(self):
        self.hlog.close()

    def do_test_chain(self, m, sps, routingtypes, routinginfo, payload,
                      appkey=None):
        """Routes a message through a series of servers, making sure that
           each one decrypts it properly and routes it correctly to the
           next.
                    m: the message to test
                    sps: sequence of PacketHandler objects for m's path
                    routingtypes: sequence of expected routingtype
                    routinginfo: sequence of expected routinginfo, excl tags
                    payload: beginning of expected final payload."""
        for sp, rt, ri in zip(sps,routingtypes,routinginfo):
            res = sp.processPacket(m)
            self.assert_(isinstance(res, DeliveryPacket) or
                         isinstance(res, RelayedPacket))
            if rt in (FWD_HOST_TYPE, SWAP_FWD_HOST_TYPE):
                self.assert_(not res.isDelivery())
                self.assertEquals(res.getAddress().pack(), ri)
                m = res.getPacket()
            else:
                self.assert_(res.isDelivery())
                self.assertEquals(res.getExitType(), rt)
                res.setTagged(res.getExitType() not in (DROP_TYPE,FRAGMENT_TYPE))
                self.assertEquals(res.getAddress(), ri)
                if appkey:
                    self.assertEquals(res.getApplicationKey(), appkey)

                self.assertStartsWith(res.getContents(), payload)
                break
        return res

    def test_successful(self):
        bfm = BuildMessage.buildForwardPacket
        # A two-hop/one-hop message.
        p = "Now is the time for all good men to come to the aid"
        m = bfm(BuildMessage.encodeMessage("\n"+p,0)[0],
                SMTP_TYPE, "nobody@invalid",
                [self.server1, self.server2], [self.server3])

        self.do_test_chain(m,
                           [self.sp1,self.sp2,self.sp3],
                           [FWD_HOST_TYPE, FWD_HOST_TYPE, SMTP_TYPE],
                           [self.server2.getRoutingInfo().pack(),
                            self.server3.getRoutingInfo().pack(),
                            "nobody@invalid"],
                           p)

        # A one-hop/one-hop message.
        m = bfm(BuildMessage.encodeMessage("\n"+p,0)[0],
                SMTP_TYPE, "nobody@invalid", [self.server1], [self.server3])

        self.do_test_chain(m,
                           [self.sp1,self.sp3],
                           [FWD_HOST_TYPE, SMTP_TYPE],
                           [self.server3.getRoutingInfo().pack(),
                            "nobody@invalid"],
                           p)

        # Try servers with multiple keys
        m = bfm(BuildMessage.encodeMessage("\n"+p,0)[0],
                SMTP_TYPE, "nobody@invalid", [self.server2], [self.server3])
        self.do_test_chain(m, [self.sp2_3, self.sp2_3], [FWD_HOST_TYPE, SMTP_TYPE],
                           [self.server3.getRoutingInfo().pack(),
                            "nobody@invalid"], p)

        # A 3/3 message with a long exit header.
        for i in (100,300):
            longemail = "f"*i+"@invalid"
            m = bfm(BuildMessage.encodeMessage("\n"+p,0)[0],
                    SMTP_TYPE, longemail,
                    [self.server1, self.server2, self.server1],
                    [self.server3, self.server1, self.server2])

            self.do_test_chain(m,
                               [self.sp1,self.sp2,self.sp1,
                                self.sp3,self.sp1,self.sp2],
                               [FWD_HOST_TYPE,FWD_HOST_TYPE,FWD_HOST_TYPE,
                                FWD_HOST_TYPE,FWD_HOST_TYPE,SMTP_TYPE],
                               [self.server2.getRoutingInfo().pack(),
                                self.server1.getRoutingInfo().pack(),
                                self.server3.getRoutingInfo().pack(),
                                self.server1.getRoutingInfo().pack(),
                                self.server2.getRoutingInfo().pack(),
                                longemail],
                               p)

    def test_deliverypacket(self):
        # Test out DeliveryPacket.*: with a plaintext ascii packet.
        bfm = BuildMessage.buildForwardPacket
        befm = BuildMessage.buildEncryptedForwardPacket

        h = "SUBJECT:cooper\n\n"
        p = "That gum you like, it's coming back in style."
        payload = BuildMessage.encodeMessage(h+p,0)[0]
        m = bfm(payload, SMTP_TYPE, "nobody@invalid", [self.server1],
                [self.server3])

        pkt = self.do_test_chain(m,
                                 [self.sp1,self.sp3],
                                 [FWD_HOST_TYPE, SMTP_TYPE],
                                 [self.server3.getRoutingInfo().pack(),
                                  "nobody@invalid"],
                                 p)

        self.assertEquals(SMTP_TYPE, pkt.getExitType())
        self.assertEquals("nobody@invalid", pkt.getAddress())
        self.assertEquals(20, len(pkt.getTag()))
        self.assertEquals(p, pkt.getContents())
        self.assert_(pkt.isDelivery())
        self.assert_(pkt.isPlaintext())
        self.failIf(pkt.isOvercompressed())
        self.assert_(pkt.isPrintingAscii())
        self.failIf(pkt.isError())
        self.assertEquals(p, pkt.getAsciiContents())
        self.assertEquals(base64.encodestring(pkt.getTag()).strip(),
                          pkt.getAsciiTag())
        self.assertEquals({"SUBJECT":"cooper"}, pkt.getHeaders())
        # with a plaintext, nonascii packet.
        pbin = hexread("0123456789ABCDEFFEDCBA9876543210")
        m = bfm(BuildMessage.encodeMessage("\n"+pbin,0)[0],
                SMTP_TYPE, "nobody@invalid",
                [self.server1], [self.server3])
        pkt = self.do_test_chain(m,
                                 [self.sp1,self.sp3],
                                 [FWD_HOST_TYPE, SMTP_TYPE],
                                 [self.server3.getRoutingInfo().pack(),
                                  "nobody@invalid"],
                                 pbin)
        self.assertEquals(pbin, pkt.getContents())
        self.assert_(pkt.isPlaintext())
        self.failIf(pkt.isPrintingAscii())
        self.assertEquals(encodeBase64(pkt.getContents()),
                          pkt.getAsciiContents())
        # with an overcompressed content
        pcomp = "          "*4096
        #      (forward, overcompressed)
        m = bfm(BuildMessage.encodeMessage(pcomp,0)[0],
                SMTP_TYPE, "nobody@invalid",
                [self.server1], [self.server3])
        pkt = self.do_test_chain(m,
                                 [self.sp1,self.sp3],
                                 [FWD_HOST_TYPE, SMTP_TYPE],
                                 [self.server3.getRoutingInfo().pack(),
                                  "nobody@invalid"],
                                 "")
        self.assert_(not pkt.isPlaintext())
        self.assert_(pkt.isOvercompressed())
        self.assert_(pkt.getAsciiContents(),
             encodeBase64(compressData(pcomp)))

        #      (enc-forward, overcompressed)
        m = befm(BuildMessage.encodeMessage(pcomp,ENC_FWD_OVERHEAD)[0],
                 SMTP_TYPE, "nobody@invalid", [self.server1],
                 [self.server3], getRSAKey(0,1024))
        pkt = self.do_test_chain(m,
                                 [self.sp1,self.sp3],
                                 [FWD_HOST_TYPE, SMTP_TYPE],
                                 [self.server3.getRoutingInfo().pack(),
                                  "nobody@invalid"],
                                 "")
        self.assert_(pkt.isEncrypted())
        self.assert_(not pkt.isPrintingAscii())
        self.assertEquals(len(pkt.getContents()), 28*1024)
        self.assertEquals(encodeBase64(pkt.getContents()),
                          pkt.getAsciiContents())

        # Header features
        #
        #    (ASCII msg with headers)
        h = {"FROM":'fred@foo', "SUBJECT":'Stuff'}
        p = encodeMessageHeaders(h) + "This is the message.\n"
        m = bfm(BuildMessage.encodeMessage(p,0)[0],
                SMTP_TYPE, "nobody@invalid",[self.server1], [self.server3])
        pkt = self.do_test_chain(m,
                                 [self.sp1, self.sp3],
                                 [FWD_HOST_TYPE, SMTP_TYPE],
                                 [self.server3.getRoutingInfo().pack(),
                                  "nobody@invalid"],
                                 "")
        self.assert_(pkt.isPrintingAscii())
        self.assert_(not pkt.isEncrypted())
        self.assertEquals(pkt.getContents(), "This is the message.\n")
        self.assertEquals(pkt.getHeaders(), h)

        #    (binary msg with headers.)
        body = "\x01\x02\x03\x04"*10
        p = encodeMessageHeaders(h) + body
        m = bfm(BuildMessage.encodeMessage(p,0)[0],
                SMTP_TYPE, "nobody@invalid",[self.server1], [self.server3])
        pkt = self.do_test_chain(m,
                                 [self.sp1, self.sp3],
                                 [FWD_HOST_TYPE, SMTP_TYPE],
                                 [self.server3.getRoutingInfo().pack(),
                                  "nobody@invalid"],
                                 "")
        self.assert_(not pkt.isPrintingAscii())
        self.assert_(not pkt.isEncrypted())
        self.assertEquals(pkt.getContents(), body)
        self.assertEquals(pkt.getHeaders(), h)


    def test_rejected(self):
        bfm = BuildMessage.buildForwardPacket
        brm = BuildMessage.buildReplyPacket
        brbi = BuildMessage._buildReplyBlockImpl

        # A long intermediate header needs to fail.
        server1X = FakeServerInfo("127.0.0.1", 1, self.pk1, "X"*20, 1)
        class _packable:
            def pack(self): return "x"*200
        server1X.getRoutingInfo = lambda _packable=_packable: _packable()

        zPayload = BuildMessage.encodeMessage("Z",0)[0]
        m = bfm(zPayload, MBOX_TYPE, "hello\000bye",
                [self.server2, server1X, self.server3],
                [server1X, self.server2, self.server3])
        self.failUnlessRaises(ParseError, self.sp2.processPacket, m)

        # Duplicate messages need to fail.
        m = bfm(zPayload, SMTP_TYPE, "nobody@invalid",
                [self.server1, self.server2], [self.server3])
        self.sp1.processPacket(m)
        self.failUnlessRaises(ContentError, self.sp1.processPacket, m)

        # Duplicate reply blocks need to fail
        reply,s,tag = brbi([self.server3], SMTP_TYPE, "fred@invalid")
        yPayload = BuildMessage.encodeMessage("Y",0)[0]
        m = brm(yPayload, [self.server2], reply)
        m2 = brm(yPayload, [self.server1], reply)
        m = self.sp2.processPacket(m).getPacket()
        self.sp3.processPacket(m)
        m2 = self.sp1.processPacket(m2).getPacket()
        self.failUnlessRaises(ContentError, self.sp3.processPacket, m2)

        # Even duplicate secrets need to go.
        prng = AESCounterPRNG(" "*16)
        reply1,s,t = brbi([self.server1], SMTP_TYPE, "fred@invalid",0,prng)
        prng = AESCounterPRNG(" "*16)
        reply2,s,t = brbi([self.server2], MBOX_TYPE, "foo",0,prng)
        m = brm(yPayload, [self.server3], reply1)
        m2 = brm(yPayload, [self.server3], reply2)
        m = self.sp3.processPacket(m).getPacket()
        self.sp1.processPacket(m)
        m2 = self.sp3.processPacket(m2).getPacket()
        self.failUnlessRaises(ContentError, self.sp2.processPacket, m2)

        # Drop gets dropped.
        m = bfm(zPayload, DROP_TYPE, "", [self.server2], [self.server2])
        m = self.sp2.processPacket(m).getPacket()
        res = self.sp2.processPacket(m)
        self.assertEquals(res,None)

        # Wrong server.
        m = bfm(zPayload, DROP_TYPE, "", [self.server1], [self.server2])
        self.failUnlessRaises(CryptoError, self.sp2.processPacket, m)
        self.failUnlessRaises(CryptoError, self.sp2_3.processPacket, m)

        # Plain junk in header
        m_x = ("XY"*64)+m[128:]
        self.failUnlessRaises(CryptoError, self.sp1.processPacket, m_x)

        # Bad message length
        m_x = m+"Z"
        self.failUnlessRaises(ParseError, self.sp1.processPacket, m_x)

        # Bad internal type
        try:
            # (We temporarily override the setting from 'BuildMessage',
            #  not Packet; BuildMessage has already imported a copy of this
            #  constant.)
            global SWAP_FWD_HOST_TYPE# override the copy used by FakeServerInfo
            save = SWAP_FWD_HOST_TYPE
            SWAP_FWD_HOST_TYPE = 50
            m_x = bfm(zPayload, 500, "", [self.server1], [self.server2])
        finally:
            SWAP_FWD_HOST_TYPE = save
        self.failUnlessRaises(ContentError, self.sp1.processPacket, m_x)

        # Subhead with bad length
        m_x = pk_encrypt("foo", self.pk1)+m[256:]
        self.failUnlessRaises(ContentError, self.sp1.processPacket, m_x)

        # Subhead we can't parse.
        m_x = pk_encrypt("f"*(256-42), self.pk1)+m[256:]
        self.failUnlessRaises(ContentError, self.sp1.processPacket, m_x)

        # Bad IPV4 info
        subh_real = pk_decrypt(m[:256], self.pk1)
        subh = parseSubheader(subh_real)
        subh.routingtype = FWD_IPV4_TYPE
        m_x = pk_encrypt(subh.pack()+subh.underflow, self.pk1)+m[256:]
        self.failUnlessRaises(ParseError, self.sp1.processPacket, m_x)

        # Bad Major or Minor
        subh = parseSubheader(subh_real)
        subh.major = 255
        m_x = pk_encrypt(subh.pack()+subh.underflow, self.pk1)+m[256:]
        self.failUnlessRaises(ContentError, self.sp1.processPacket, m_x)

        # Bad digest
        subh = parseSubheader(subh_real)
        subh.digest = " "*20
        m_x = pk_encrypt(subh.pack()+subh.underflow, self.pk1)+m[256:]
        self.failUnlessRaises(ContentError, self.sp1.processPacket, m_x)

        # Corrupt payload
        m = bfm(zPayload, MBOX_TYPE, "Z", [self.server1, self.server2],
                [self.server3])
        m_x = m[:-30] + " "*30
        assert len(m_x) == len(m)
        m_x = self.sp1.processPacket(m_x).getPacket()
        m_x = self.sp2.processPacket(m_x).getPacket()
        self.failUnlessRaises(CryptoError, self.sp3.processPacket, m_x)

#----------------------------------------------------------------------
# FILESTORE and QUEUE

class TestDeliveryQueue(DeliveryQueue):
    def __init__(self,d,now=None):
        DeliveryQueue.__init__(self,d,now=now)
        self._msgs = None
    def sendReadyMessages(self, *x, **y):
        self._msgs = None
        DeliveryQueue.sendReadyMessages(self, *x,**y)
    def _deliverMessages(self, msgList):
        self._msgs = msgList

class TestPerAddressDeliveryQueue(PerAddressDeliveryQueue):
    def __init__(self,d,now=None):
        PerAddressDeliveryQueue.__init__(self,d,now=now)
        self._msgs = None
    def sendReadyMessages(self, *x, **y):
        self._msgs = None
        PerAddressDeliveryQueue.sendReadyMessages(self, *x,**y)
    def _deliverMessages(self, msgList):
        self._msgs = msgList

class FStoreTestBase(TestCase):
    def unlink(self, fns):
        for f in fns:
            os.unlink(f)

class FilestoreTests(FStoreTestBase):
    def setUp(self):
        mixminion.Common.installSIGCHLDHandler()
        self.d1 = mix_mktemp("q1")
        self.d2 = mix_mktemp("q2")
        self.d3 = mix_mktemp("q3")
    def testCreateStore(self):
        Store = mixminion.Filestore.MixedStore

        # Nonexistent dir.
        self.failUnlessRaises(MixFatalError, Store, self.d1)
        # File in place of dir
        writeFile(self.d1, "   ")
        self.failUnlessRaises(MixFatalError, Store, self.d1)
        self.failUnlessRaises(MixFatalError, Store, self.d1, create=1)
        os.unlink(self.d1)

        # Try to create
        queue = Store(self.d1, create=1)
        self.failUnless(os.path.isdir(self.d1))
        if not ON_WINDOWS:
            self.assertEquals(0700, os.stat(self.d1)[stat.ST_MODE] & 0777)
        self.assertEquals(0, len(os.listdir(self.d1)))
        queue.queueMessage("Hello world 1")
        h2 = queue.queueMessage("Hello world 2")
        self.assertEquals(2, len(os.listdir(self.d1)))
        self.assertEquals(2, queue.count())

        # Make sure recreate doesn't bonk
        queue = Store(self.d1, create=1)

        # Use a queue we haven't just made.
        queue = Store(self.d1)
        self.assertEquals(2, queue.count())
        self.assertEquals(queue.messageContents(h2), "Hello world 2")
        queue.removeMessage(h2)
        self.assertEquals(1, queue.count())

        queue.removeAll(self.unlink)

    def testStoreOps(self):
        Store = mixminion.Filestore.MixedStore

        queue1 = Store(self.d2, create=1)
        queue2 = Store(self.d3, create=1)

        # Put 100 messages in queue1
        handles = [queue1.queueMessage("Sample message %s" % i)
                   for i in range(100)]
        hdict = {}
        for i in range(100): hdict[handles[i]] = i
        # Make sure that queue1 has all 100 elements
        self.assertEquals(queue1.count(), 100)
        self.assertEquals(len(handles), 100)

        # Get the messages in random order, and make sure the contents
        # of each one are correct
        foundHandles = queue1.pickRandom(100)
        self.assertEquals(len(foundHandles), 100)
        for h in foundHandles:
            self.failUnless(hdict.has_key(h))
            i = hdict[h]
            self.assertEquals("Sample message %s" %i,
                              queue1.messageContents(h))

        assert len(hdict) == len(handles) == 100

        # Move the first 30 messages to queue2
        q2h = []
        for h in handles[:30]:
            nh = queue2.queueMessage(queue1.messageContents(h))
            queue1.removeMessage(h)
            q2h.append(nh)

        # Look at the messages in queue2, 15 then 30 at a time.
        for group in queue2.pickRandom(15), queue2.pickRandom(30):
            seen = {}
            for h in group:
                c = queue2.messageContents(h)
                self.assertStartsWith(c, "Sample message ")
                i = atoi(c[15:])
                self.failIf(seen.has_key(i))
                seen[i]=1

        # Make sure that we got all 30 messages
        for i in range(30):
            self.failUnless(seen.has_key(i))

        # Remove messages 30..59 from queue1.
        for h in handles[30:60]:
            queue1.removeMessage(h)
        self.assertEquals(40, queue1.count())

        # Make sure that smaller pickRandoms work.
        L1 = queue1.pickRandom(10)
        L2 = queue1.pickRandom(10)
        self.failUnless(len(L1) == 10)
        self.failUnless(len(L2) == 10)
        self.failUnless(L1 != L2)

        # Test 'openMessage'
        f = queue1.openMessage(handles[60])
        s = f.read()
        f.close()
        self.assertEquals(s, "Sample message 60")

        # test successful 'openNewMessage'
        f, h = queue1.openNewMessage()
        f.write("z"*100)
        self.failUnlessRaises(IOError, queue1.messageContents, h)
        self.assertEquals(queue1.count(), 40)
        queue1.finishMessage(f,h)
        self.assertEquals(queue1.messageContents(h), "z"*100)
        self.assertEquals(queue1.count(), 41)

        # test aborted 'openNewMessage'
        f, h = queue1.openNewMessage()
        f.write("z"*100)
        queue1.abortMessage(f,h)
        self.failUnlessRaises(IOError, queue1.messageContents, h)
        self.assertEquals(queue1.count(), 41)
        self.assert_(not os.path.exists(os.path.join(self.d2, "msg_"+h)))

        # Test object functionality
        obj = [ ("A pair of strings", "in a tuple in a list") ]
        h1 = queue1.queueObject(obj)
        h2 = queue1.queueObject(6060842)
        self.assertEquals(obj, queue1.getObject(h1))
        self.assertEquals(6060842, queue1.getObject(h2))
        self.assertEquals(obj, cPickle.loads(queue1.messageContents(h1)))

        # Scrub both queues.
        queue1.removeAll(self.unlink)
        queue2.removeAll(self.unlink)
        queue1.cleanQueue(self.unlink)
        queue2.cleanQueue(self.unlink)

    def testMetadataStores(self):
        d_d = mix_mktemp("q_md")
        Store = mixminion.Filestore.StringMetadataStore

        queue = Store(d_d, create=1)
        h1 = queue.queueMessageAndMetadata("abc", [2,3])
        self.assertEquals(readPickled(os.path.join(d_d, "meta_"+h1)), [2,3])
        self.assertEquals(queue.getMetadata(h1), [2,3])
        h2 = queue.queueMessageAndMetadata("def", [5,6])
        h3 = queue.queueMessageAndMetadata("ghi", None)
        self.assertEquals(queue._metadata_cache, { h1 : [2,3], h2 : [5,6],
                                                   h3 : None })
        os.unlink(os.path.join(d_d, "meta_"+h3))
        queue = Store(d_d, create=0)
        self.assertEquals(queue.getMetadata(h2), [5,6])
        self.assertEquals(queue._metadata_cache, { h2 : [5,6] })
        try:
            suspendLog()
            queue.loadAllMetadata(lambda h: h)
        finally:
            s = resumeLog()
        self.assertEndsWith(s, "Missing metadata for file %s\n"%h3)
        self.assertEquals(queue._metadata_cache, { h1 : [2,3], h2 : [5,6],
                                                   h3: h3})
        self.assertEquals(readPickled(os.path.join(d_d, "meta_"+h3)), h3)
        queue.removeMessage(h2)
        self.assertEquals(queue._metadata_cache, { h1 : [2,3], h3: h3 })
        self.assert_(os.path.exists(os.path.join(d_d, "rmvm_"+h2)))
        self.assert_(os.path.exists(os.path.join(d_d, "rmv_"+h2)))
        queue.cleanQueue(self.unlink)
        self.assert_(not os.path.exists(os.path.join(d_d, "rmvm_"+h2)))
        self.assert_(not os.path.exists(os.path.join(d_d, "rmv_"+h2)))

    def testDBWrappers(self):
        d_parent = mix_mktemp("db")
        loc = os.path.join(d_parent, "db0")
        # Test unjournaled DB.
        class NumDB(mixminion.Filestore.DBBase):
            def __init__(self,loc):
                mixminion.Filestore.DBBase.__init__(self, loc, "numbers")
            def _encodeKey(self,k):
                return str(k)
            def _decodeKey(self,k):
                return int(k)
            def _encodeVal(self,v):
                return v.upper()
            def _decodeVal(self,v):
                return v.lower()
        db0 = NumDB(loc)
        db0[3] = 'three'
        db0[99] = 'Ninety-nine'
        self.assertEquals(db0[3], 'three')
        self.assertEquals(db0[99], 'ninety-nine')
        self.assertRaises(KeyError, lambda db0=db0:db0[10])
        self.assertRaises(KeyError, lambda db0=db0:db0["10"])
        self.assertEquals(db0.log["3"], 'THREE')
        db0[3] = 'iii'
        self.assertEquals(db0.log["3"], 'III')
        self.assertEquals(db0["3"], 'iii')
        self.assertEquals(db0.get(10), None)
        self.assertEquals(db0.get(10, "hi"), "hi")
        self.assert_(db0.has_key(3))
        self.assert_(not db0.has_key(" 3"))
        self.assert_(not db0.has_key("10"))
        db0[6]='six'
        del db0[6]
        self.assert_(not db0.has_key(6))
        self.assert_(not db0.log.has_key("6"))
        self.assertRaises(KeyError, lambda db0=db0:db0[6])
        self.assertUnorderedEq([3, 99], db0.keys())
        db0.close()
        db0 = NumDB(loc)
        self.assertUnorderedEq([3, 99], db0.keys())
        self.assert_(not db0.has_key(6))
        self.assertEquals(db0["3"], 'iii')
        db0.sync()
        db0[3] = 'three'
        db0.close()

        for fn in os.listdir(d_parent):
            self.assertStartsWith(fn, "db0")
            os.unlink(os.path.join(d_parent,fn))

        # Test journaled DB.
        loc = os.path.join(d_parent, "db1")
        jloc = loc+"_jrnl"
        class JNumDB(mixminion.Filestore.JournaledDBBase):
            def __init__(self,loc):
                mixminion.Filestore.JournaledDBBase.__init__(
                    self, loc, "numbers", 5, 20, None)
            def _encodeKey(self,k): return str(k)
            def _decodeKey(self,k): return int(k)
            def _encodeVal(self,v): return v
            def _decodeVal(self,v): return v
            def _jEncodeKey(self,k): return "%05d"%k
            def _jDecodeKey(self,k): return int(k)
            def _jEncodeVal(self,v): return "%20s"%v
            def _jDecodeVal(self,v): return v.strip()

        db1 = JNumDB(loc)
        db1[3] = "three"
        db1[10] = "ten"
        db1[8] = "eight"
        self.assertEquals(readFile(jloc),
                          "00003               three"
                          "00010                 ten"
                          "00008               eight")
        self.assertUnorderedEq(db1.log.keys(), [])
        self.assertUnorderedEq(db1.keys(), [10, 3, 8])
        self.assertEquals(db1[3], "three")
        self.assertEquals(db1[10], "ten")
        self.assertEquals(db1[8], "eight")
        self.assertRaises(KeyError, lambda db1=db1:db1[7])
        self.assert_(db1.has_key(8))
        self.assert_(not db1.has_key(5))
        db1.close()
        db1 = JNumDB(loc)
        self.assertUnorderedEq(db1.log.keys(), ['10', '3', '8'])
        self.assertUnorderedEq(db1.keys(), [10, 3, 8])
        self.assertUnorderedEq(db1.journal.keys(), [])
        self.assertEquals(readFile(jloc), "")
        self.assertEquals(db1[3], "three")
        self.assertEquals(db1[10], "ten")
        self.assertEquals(db1[8], "eight")
        self.assertRaises(KeyError, lambda db1=db1:db1[7])
        self.assert_(db1.has_key(8))
        self.assert_(not db1.has_key(5))
        db1[9] = "nine"
        self.assertEquals(readFile(jloc),
                          "00009                nine")
        self.assertEquals(db1[9], "nine")
        db1.sync()
        self.assertEquals(db1[9], "nine")
        self.assertEquals(db1[10], "ten")
        db1.close()
        for fn in os.listdir(d_parent):
            self.assertStartsWith(fn, "db1")
            os.unlink(os.path.join(d_parent,fn))

        # Test journaled value-less DB
        loc = os.path.join(d_parent, "db2")
        jloc = loc+"_jrnl"
        db2 = mixminion.Filestore.BooleanJournaledDBBase(loc, "numbers", 2)
        db2["02"] = 1
        db2["03"] = 1
        db2["05"] = 1
        db2["07"] = 1
        db2["11"] = 1
        db2["13"] = 1
        self.assertEquals(1, db2["02"])
        self.assertEquals(readFile(jloc), "020305071113")
        db2.close()
        db2 = mixminion.Filestore.BooleanJournaledDBBase(loc, "numbers", 2)
        self.assertEquals(readFile(jloc), "")
        self.assertEquals(1, db2["13"])
        self.assert_(db2.has_key("07"))
        self.assert_(not db2.has_key("08"))
        db2.close()

        # Test migration from unjournaled to journaled.
        loc = os.path.join(d_parent, "db3")
        jloc = loc+"_jrnl"
        db3 = NumDB(loc)
        db3[9] = "nine"
        db3[10] = "ten"
        db3[11] = "eleven"
        db3.close()
        db3 = JNumDB(loc)
        self.assertEquals(readFile(jloc), "")
        db3[12] = 'twelve'
        self.assertEquals(readFile(jloc), "00012              twelve")
        self.assertEquals(db3[10], "TEN")
        self.assertEquals(db3[12], "twelve")
        db3.close()
        for fn in os.listdir(d_parent):
            os.unlink(os.path.join(d_parent,fn))

        # WritethroughDict
        loc = os.path.join(d_parent, "db4")
        WD = mixminion.Filestore.WritethroughDict
        d = WD(loc, "testing")
        d["xyzzy"] = 91
        d["bliznert"] = (4,22,11,8)
        self.assertEquals(d["xyzzy"], 91)
        self.assertEquals(d["bliznert"], (4,22,11,8))
        d.close()

        d = WD(loc, "testing")
        self.assertEquals(d["xyzzy"], 91)
        self.assert_(d.has_key("bliznert"))
        d["bliznert"] = [ 12, "aeaeae" ]
        d["mulligan"] = ( "stately, plump", )
        self.assertEquals(d.get("mulligan"), ("stately, plump",))
        self.assertEquals(d.get("mulliganX"), None)
        self.assertEquals(d["bliznert"], [12, "aeaeae"])
        del d['xyzzy']
        self.assert_(not d.has_key("xyzzy"))
        d.close()

        d = WD(loc, "testing")
        try:
            _ = d["xyzzy"]
        except KeyError:
            pass
        else:
            self.fail()
        self.assertEquals(d.get("mulligan"), ("stately, plump",))
        self.assertEquals(d["bliznert"], [12, "aeaeae"])
        self.assertUnorderedEq(d.keys(), ["mulligan","bliznert"])
        d.close()

class _TestAddr:
    def __init__(self,s):
        self.s = s
    def __str__(self):
        return self.s
    def __hash__(self):
        return hash(self.s+"_but different.")
    def __cmp__(self, o):
        return cmp(self.s, o.s)

class QueueTests(FStoreTestBase):
    def setUp(self):
        mixminion.Common.installSIGCHLDHandler()
    def testDeliveryQueues(self):
        d_d = mix_mktemp("qd")

        now = 10000 # time.time()
        queue = TestDeliveryQueue(d_d, now)
        queue.setRetrySchedule([10, 10, 10, 10]) # Retry up to 40 sec.
        # First, make sure the queue stores messages correctly.
        h1 = queue.queueDeliveryMessage("Message 1", now=now)
        h2 = queue.queueDeliveryMessage("Message 2", now=now)
        self.assertEquals(("Message 1", now, None, now), queue._inspect(h1))
        self.assertEquals(("Message 2", now, None, now), queue._inspect(h2))

        # Call sendReadyMessages to begin 'sending' msg1 and msg2.
        queue.sendReadyMessages(now)
        msgs = [ (msg.getHandle(), msg.getMessage()) for msg in queue._msgs ]
        self.assertEquals(2, len(msgs))
        # _deliverMessages should have gotten them both.
        self.failUnless((h1, "Message 1") in msgs)
        self.failUnless((h2, "Message 2") in msgs)
        # Add msg3, and acknowledge that msg1 succeeded.  msg2 is now in limbo
        h3 = queue.queueDeliveryMessage("Message 3", now=now)
        queue.deliverySucceeded(h1)
        # Only msg3 should get sent out, since msg2 is still in progress.
        queue.sendReadyMessages(now+1)
        msgs = [ (msg.getHandle(), msg.getMessage()) for msg in queue._msgs ]
        self.assertEquals([(h3, "Message 3")], msgs)

        # Now, make sure that msg1 is gone from the pool.
        self.assertUnorderedEq(queue.getAllMessages(), [h2, h3])

        # Now, fail msg2 retriably, and fail msg3 hard.  Only one message
        # should be left.  (It will have a different handle from the old
        # msg2.)
        queue.deliveryFailed(h2, retriable=1, now=now+4)
        queue.deliveryFailed(h3, retriable=0, now=now+4)
        allHandles = queue.getAllMessages()
        h4 = allHandles[0]
        queue.cleanQueue(self.unlink)
        self.assertUnorderedEq(os.listdir(d_d), ["meta_"+h4, "msg_"+h4])
        self.assertEquals([h4], queue.getAllMessages())
        self.assertEquals(("Message 2", now, now, now+10), queue._inspect(h2))

        # Reload the queue; and try that again.
        queue = TestDeliveryQueue(d_d, now+4)
        queue.setRetrySchedule([10, 10, 10, 10]) # Retry up to 40 sec.
        self.assertEquals([h4], queue.getAllMessages())
        self.assertEquals(("Message 2", now, now, now+10), queue._inspect(h2))

        # When we try to send messages again after 5 seconds, nothing happens.
        queue.sendReadyMessages(now+5)
        self.assertEquals([], queue._msgs)
        # When we try to send again after after 11 seconds, message 2 fires.
        queue.sendReadyMessages(now+11)
        msgs = [ (msg.getHandle(), msg.getMessage()) for msg in queue._msgs ]
        self.assertEquals([(h4, "Message 2")], msgs)
        self.assertEquals(h2, h4)
        queue._msgs[0].failed(retriable=1, now=now+15)
        self.assertEquals(("Message 2", now, now+11, now+20),
                          queue._inspect(h2))
        # At 31 seconds, message 2 fires.
        h5 = queue.getAllMessages()[0]
        queue.sendReadyMessages(now+31)
        msgs = [ (msg.getHandle(), msg.getMessage()) for msg in queue._msgs ]
        self.assertEquals([(h5, "Message 2")], msgs)
        self.assertEquals(h5, h4)
        queue.deliveryFailed(h5, retriable=1, now=now+33)
        self.assertEquals(("Message 2", now, now+31, now+40),
                          queue._inspect(h2))
        # At 45 sec, it fires one last time.  It will have gotten up to #4
        # already.
        h6 = queue.getAllMessages()[0]
        queue.sendReadyMessages(now+45)
        msgs = [ (msg.getHandle(), msg.getMessage()) for msg in queue._msgs ]
        self.assertEquals([(h6, "Message 2")], msgs)
        self.assertEquals(h6, h5)
        queue.deliveryFailed(h6, retriable=1, now=now+100)
        # Now Message 2 is timed out.
        self.assertEquals([], queue.getAllMessages())

        queue.removeAll(self.unlink)
        queue.cleanQueue(self.unlink)

    def testPerAddressDeliveryQueue(self):
        PADQ = TestPerAddressDeliveryQueue
        A1 = _TestAddr("FirstAddress")
        A2 = _TestAddr("SecondAddress")
        A3 = _TestAddr("ThirdAddress")
        HOUR = 60*60
        MIN = 60
        RETRY = [ HOUR, HOUR, HOUR, HOUR*2, HOUR*4, HOUR*8 ]
        assert reduce(operator.add, RETRY) == 17*HOUR
        loc = mix_mktemp()
        start = time.time()
        q = PADQ(loc,now=start)
        q.setRetrySchedule(RETRY)
        h1 = q.queueDeliveryMessage("Message number one", A1, start)
        h2 = q.queueDeliveryMessage("Message number two", A2, start)
        h3 = q.queueDeliveryMessage("Message number three", A1, start)
        q._repOK()

        m, ds, as_ = q._inspect(h1)
        self.assertEquals(m, "Message number one")
        self.assertEquals(ds.address, A1)
        self.assertEquals(as_.address, A1)
        self.assert_(not ds.isPending())
        self.assertEquals(ds.lastAttempt, None)
        self.assertEquals(as_.lastSuccess, None)
        self.assertEquals(as_.lastFailure, None)
        self.assertEquals(as_.firstFailure, None)
        q.removeExpiredMessages(start)
        self.assertEquals(q.store.count(), 3)

        q.sendReadyMessages(start+10)
        msgs = self._pendingMsgDict(q._msgs)
        self.assertEquals(len(msgs), 3)
        self.assert_(q._inspect(h1)[1].isPending())
        self.assertEquals(msgs[h1].getAddress(), A1)
        self.assertEquals(msgs[h1].getMessage(), "Message number one")
        # Succeed on address 2.
        self.assertEquals(msgs[h2].getAddress(), A2)
        msgs[h2].succeeded(now=start+20)
        # Succeed on _one_ message from address 1, but not the other.
        msgs[h1].succeeded(now=start+30)
        msgs[h3].failed(now=start+40, retriable=1)
        self.assert_(not q._inspect(h3)[1].isPending())

        as_ = q.addressStateDB[str(A1)]
        self.assertIEquals(as_.lastSuccess-start, 30)
        self.assertIEquals(as_.firstFailure-start, 10)
        self.assertIEquals(as_.lastFailure-start, 10)
        self.assertIEquals(as_.nextAttempt-start, 10+HOUR)
        as_ = q.addressStateDB[str(A2)]
        self.assertIEquals(as_.lastSuccess-start, 20)
        self.assertEquals(as_.firstFailure, None)
        self.assertEquals(as_.lastFailure, None)
        self.assertIEquals(as_.nextAttempt-start, 20)

        # Queue two more messages; the one on A2 gets tried; the ones on
        # A1 doesn't get tried for a while.
        h4 = q.queueDeliveryMessage("This is message four", A1, start+MIN)
        h5 = q.queueDeliveryMessage("This is message five", A2, start+MIN)
        q.sendReadyMessages(start+2*MIN)
        msgs = self._pendingMsgDict(q._msgs)
        self.assertEquals(len(msgs), 1)
        self.assertEquals(msgs[h5].getMessage(), "This is message five")
        msgs[h5].succeeded(now=start+3*MIN)

        # Wait an "hour"; then, the A1 messages both get tried.
        q.sendReadyMessages(start+HOUR+MIN)
        msgs = self._pendingMsgDict(q._msgs)
        self.assertUnorderedEq(msgs.keys(), [h3, h4])
        as_ = q.addressStateDB[str(A1)]
        msgs[h3].failed(now=start+HOUR+MIN+5, retriable=1)
        msgs[h4].failed(now=start+HOUR+MIN+5, retriable=1)

        self.assertIEquals(as_.firstFailure-start, 10)
        self.assertIEquals(as_.lastFailure-start, HOUR+MIN)
        self.assertIEquals(as_.nextAttempt-start, 10+2*HOUR)

        h6 = q.queueDeliveryMessage("The sixth message", A1,
                                    now=start+HOUR*17-MIN)

        # 17 hours and 30 seconds after 'start', exactly one message
        # is ready to be removed.
        self.assertEquals(3, q.store.count())
        q.removeExpiredMessages(now=start+HOUR*17+30)
        self.assertEquals(2, q.store.count())
        self.assertUnorderedEq(q.store.getAllMessages(), [h4,h6])
        self.assertUnorderedEq(q.addressStateDB.keys(), [str(A1), str(A2)])
        # 17 hours and 2 minutes after 'start', another one is ready to be
        # removed.
        q.removeExpiredMessages(now=start+HOUR*17+MIN*2)
        self.assertUnorderedEq(q.store.getAllMessages(), [h6])
        self.assertUnorderedEq(q.addressStateDB.keys(), [str(A1), str(A2)])

        # 17 hours and 5 minutes: A2 expires (no activity for a long time.)
        q.removeExpiredMessages(now=start+HOUR*17+MIN*5)
        self.assertUnorderedEq(q.store.getAllMessages(), [h6])
        self.assertUnorderedEq(q.addressStateDB.keys(), [str(A1)])

        # Make sure that messages keep getting retried...
        as_ = q.addressStateDB[str(A1)]
        # (Reset nextDelivery on A1, since not all the delivery attempts
        # have really happened.)
        q.sendReadyMessages(now=start+HOUR*17+MIN*15)
        msgs = self._pendingMsgDict(q._msgs)
        msgs[h6].failed(now=start+HOUR*17+MIN*16, retriable=0)
        self.assertEquals(as_.nextAttempt - start, 25*HOUR+10)
        self.assertEquals([], q.store.getAllMessages())

        # Test reloading.
        q.sync()
        q.close()
        q = PADQ(loc,now=start+20*HOUR)
        q.setRetrySchedule(RETRY,now=start+20*HOUR)
        q._repOK()
        self.assertEquals(q._getAddressState(A1).nextAttempt - start,
                          25*HOUR+10)
        q.close()

        # Test upgrading
        loc2 = mix_mktemp()
        dq = TestDeliveryQueue(loc2, now=start)
        hA = dq.queueDeliveryMessage("Grandfathered msg 1", A2, now=start)
        hB = dq.queueDeliveryMessage("Grandfathered msg 2", A3, now=start+30)
        q = PADQ(loc2,now=start+MIN)
        self.assertUnorderedEq([hA,hB],q.store.getAllMessages())
        q.sendReadyMessages(now=start+2*MIN)
        msgs = self._pendingMsgDict(q._msgs)
        self.assertUnorderedEq(msgs.keys(), [hA,hB])
        self.assertEquals(msgs[hA].getAddress(),A2)
        self.assertEquals(msgs[hB].getAddress(),A3)
        q.close()

    def _pendingMsgDict(self, lst):
        d = {}
        for m in lst:
            d[m.getHandle()] = m
        return d

    def testMixPools(self):
        d_m = mix_mktemp("qm")

        # Trivial 'TimedMixPool'
        queue = TimedMixPool(d_m)
        h1 = queue.queueObject("Hello1")
        h2 = queue.queueObject("Hello2")
        h3 = queue.queueObject("Hello3")
        self.assertUnorderedEq(queue.getBatch(), [h1,h2,h3])

        # Now, test the CottrellMixPool.
        cmq = CottrellMixPool(d_m, 600, 6, sendRate=.3)
        # Not enough messages (<= 6) -- none will be sent.
        self.assertEquals([], cmq.getBatch())
        self.assertEquals([], cmq.getBatch())
        # 8 messages: 2 get sent
        for i in range(5):
            cmq.queueObject("Message %s"%i)
        self.assertEquals(8, cmq.count())
        b1, b2, b3 = cmq.getBatch(), cmq.getBatch(), cmq.getBatch()
        self.assertEquals(2, len(b1))
        self.assertEquals(2, len(b2))
        self.assertEquals(2, len(b3))

        # Make sure that all message batches are different.
        allEq = 1
        for x in xrange(13): #fails <one in a trillion
            b = cmq.getBatch()
            if b != b1:
                allEq = 0; break
        self.failIf(allEq)

        # Send 30 when there are 100 messages.
        for x in xrange(92):
            cmq.queueObject("Hello2 %s"%x)
        for x in xrange(10):
            self.assertEquals(30, len(cmq.getBatch()))

        # Binomial Cottrell pool
        bcmq = BinomialCottrellMixPool(d_m, 600, 6, sendRate=.3)
        # (Just make sure that we don't always return the same number of
        #  messages each time.)
        messageLens = []
        for i in range(31):
            b = bcmq.getBatch()
            messageLens.append(len(b))
        messageLens.sort()
        self.failIf(messageLens[0] == messageLens[-1])
        # (Fails less than once in 2 **30 tests.)
        self.assert_(messageLens[0] <= 30)
        self.assert_(messageLens[-1] >= 30)

        bcmq.removeAll(self.unlink)
        bcmq.cleanQueue(self.unlink)

#---------------------------------------------------------------------
# LOGGING
class LogTests(TestCase):
    def testLogging(self):

        # Create a new loghandler, and try sending a few messages to it.
        log = Log("INFO")
        self.assertEquals(log.getMinSeverity(), "INFO")
        log.handlers = []
        log.log("WARN", "This message should not appear")
        buf = cStringIO.StringIO()
        log.addHandler(_ConsoleLogHandler(buf))
        log.trace("Foo")
        self.assertEquals(buf.getvalue(), "")
        log.log("WARN", "Hello%sworld", ", ")
        self.assertEndsWith(buf.getvalue(), "[WARN] Hello, world\n")
        self.failUnless(buf.getvalue().index('\n') == len(buf.getvalue())-1)
        log.error("All your anonymity are belong to us")
        self.assertEndsWith(buf.getvalue(),
            "[ERROR] All your anonymity are belong to us\n")

        buf.truncate(0)

        # Try out error_exc.
        try:
            raise MixError()
        except:
            inf = sys.exc_info()
        log.error_exc(inf)
        log.error_exc(inf, "And so on")
        log.error_exc(inf, "And so %s", "on")

        # Try out file logging
        t = mix_mktemp("log")
        t1 = t+"1"

        log.addHandler(_FileLogHandler(t))
        log.info("Abc")
        log.info("Def")
        if ON_WIN32: #WWWW
            log.close()
            return

        os.rename(t,t1)
        log.info("Ghi")
        log.reset()
        log.info("Klm")
        log.close()
        self.assertEquals(readFile(t).count("\n") , 1)
        self.assertEquals(readFile(t1).count("\n"), 3)

    def testLogStream(self):
        stream = mixminion.Common.LogStream("STREAM", "WARN")
        suspendLog()
        try:
            print >>stream, "Testing", 1,2,3
            print >>stream
            print >>stream, "A\nB\nC"
            print >>stream, "X",
            print >>stream, "Y"
        finally:
            r = resumeLog()
        lines = [ l[26:] for l in r.split("\n") ]
        self.assertEquals(lines[0], "[WARN] ->STREAM: Testing 1 2 3")
        self.assertEquals(lines[1], "[WARN] ->STREAM: ")
        self.assertEquals(lines[2], "[WARN] ->STREAM: A")
        self.assertEquals(lines[3], "[WARN] ->STREAM: B")
        self.assertEquals(lines[4], "[WARN] ->STREAM: C")
        self.assertEquals(lines[5], "[WARN] ->STREAM: X Y")

    def testStatusLog(self):
        SL = mixminion.Common.StatusLog()
        self.assertEquals("[MIXMINION:] ABC\n",
                          SL.msg("ABC", ""))
        self.assertEquals("[MIXMINION:] ABC 42 HELLO\n",
                          SL.msg("ABC", "42 HELLO"))
        tmpfile = mix_mktemp()
        f = open(tmpfile, 'w')
        SL.setFD(f.fileno())
        SL.log("REVOLUTION_9", "number nine number nine")
        SL.log("REVOLUTION_9", "take this brother")
        SL.setFD(None)
        f.close()
        self.assertEquals(readFile(tmpfile),
                   "[MIXMINION:] REVOLUTION_9 number nine number nine\n"
                   "[MIXMINION:] REVOLUTION_9 take this brother\n")

#----------------------------------------------------------------------
# File paranoia


class FileParanoiaTests(TestCase):
    def ensureParanoia(self, whatkind):
        tempdir = mixminion.testSupport._MM_TESTING_TEMPDIR

        # If our tempdir doesn't exist and isn't private, we can't go on.
        try:
            checkPrivateDir(tempdir)
        except MixFatalError, e:
            self.fail(("Can't test %s paranoia, because something's\n"
                      +" wrong with %s: %s")%(whatkind, tempdir, str(e)))

    def testPrivateDirs(self):
        self.ensureParanoia("directory")

        # Pick a private directory under tempdir, but don't create it.
        noia = mix_mktemp("noia")
        subdir = os.path.join(noia, "subdir")

        # Nonexistent directory.
        self.failUnlessRaises(MixFatalError, checkPrivateDir, noia)
        # Bad permissions.
        os.mkdir(noia)
        if not ON_WINDOWS:
            os.chmod(noia, 0777)
            self.failUnlessRaises(MixFatalError, checkPrivateDir, noia)
            # Bad permissions on parent
            os.mkdir(subdir, 0700)
            self.failUnlessRaises(MixFatalError, checkPrivateDir, subdir)
            os.chmod(noia, 0755)
            checkPrivateDir(subdir)
            os.chmod(noia, 0700)
            checkPrivateDir(subdir)
            # Not writable by self
            os.chmod(subdir, 0600)
            self.failUnlessRaises(MixFatalError, checkPrivateDir, subdir)
            os.rmdir(subdir)
        # Not a directory
        writeFile(subdir, "x")
        os.chmod(subdir, 0700)
        self.failUnlessRaises(MixFatalError, checkPrivateDir, subdir)
        os.unlink(subdir)
        os.mkdir(subdir, 0700)

        # Helper fn: return mode,uid,isdir
        def mud(f):
            st = os.stat(f)
            return st[stat.ST_MODE]&0777, st[stat.ST_UID], os.path.isdir(f)

        # Now we test a directory we don't own...
        if not ON_WINDOWS and os.getuid() == 0:
            # If we're root, we can play with chown!
            # We don't own the directory
            os.chown(subdir, 1, 1)
            self.failUnlessRaises(MixFatalError, checkPrivateDir, subdir)
            os.chown(subdir, 0, os.getgid())
            # We don't own the parent
            os.chown(noia, 1, 1)
            self.failUnlessRaises(MixFatalError, checkPrivateDir, subdir)
            os.chown(noia, 0, os.getgid())
        elif not ON_WINDOWS:
            # We're not root.  We can't reliably find or make a directory
            # that's non-root and non-us.  Let's just make sure we don't
            # own temp.
            if os.path.exists("/tmp") and mud("/tmp")[1] != os.getuid():
                self.failUnlessRaises(MixFatalError, checkPrivateDir, "/tmp")

        if ON_WINDOWS:
            private = permissive = public = 0777
        else:
            private, permissive, public = 0700, 0777, 0755

        # Okay.  Now we try createPrivateDir a couple of times...
        old_mask = None
        try:
            # Make sure umask is lenient, so we can tell whether c-p-d is
            # strict.
            old_mask = os.umask(022)
            # 1. Create the world.
            os.rmdir(subdir)
            os.rmdir(noia)
            createPrivateDir(subdir)
            if not ON_WINDOWS:
                self.assertEquals((private,os.getuid(),1), mud(subdir))
                self.assertEquals((private,os.getuid(),1), mud(noia))
                # 2. Just create one dir.
                os.rmdir(subdir)
                os.chmod(noia, public)
                createPrivateDir(subdir)
                self.assertEquals((private,os.getuid(),1), mud(subdir))
                self.assertEquals((public,os.getuid(),1), mud(noia))
                # 3. Fail to create because of bad permissions
                os.rmdir(subdir)
                os.chmod(noia, permissive)
                self.failUnlessRaises(MixFatalError, createPrivateDir, subdir)
            # 4. Fail to create because of OSError
            os.rmdir(subdir)
            writeFile(subdir, 'W')
            self.failUnlessRaises(MixFatalError, createPrivateDir, subdir)
            os.unlink(subdir)
            # 5. Succeed: it's already there.
            os.chmod(noia, private)
            os.mkdir(subdir, private)
            createPrivateDir(subdir)
            # 6. Fail: it's already there, but has bad permissions
            if not ON_WINDOWS:
                os.chmod(subdir, permissive)
                self.failUnlessRaises(MixFatalError, createPrivateDir, subdir)
                os.chmod(subdir, private)
        finally:
            if old_mask is not None:
                os.umask(old_mask)

    def testPrivateFiles(self):
        self.ensureParanoia("file")
        dir = mix_mktemp()
        os.mkdir(dir, 0700)
        subdir = os.path.join(dir, "subdir")
        os.mkdir(subdir, 0777)
        os.chmod(subdir, 0777) # in case umask is set.

        # File doesn't exist.
        self.failUnlessRaises(MixFatalError,
                              checkPrivateFile, os.path.join(dir, "x"))

        # Parent not private.
        subdir_x = os.path.join(subdir, "x")
        writeFile(subdir_x, "zzz")
        os.chmod(subdir_x, 0600)
        if not ON_WINDOWS:
            self.failUnlessRaises(MixFatalError,
                                  checkPrivateFile, subdir_x)

        # File not owned by us. (???)
        if not ON_WINDOWS and os.getuid() == 0:
            os.chmod(subdir_x, 0600)
            os.chown(subdir_x, 1, 1)
            self.failUnlessRaises(MixFatalError, checkPrivateFile, subdir_x)
            os.chown(subdir_x, 0, os.getgid())
        else:
            if os.path.exists("/etc/shadow"):
                self.failUnlessRaises(MixFatalError, checkPrivateFile,
                                      "/etc/shadow")

        # File not private (fix, noex)
        os.chmod(subdir, 0700)
        os.chmod(subdir_x, 0777)
        try:
            suspendLog()
            checkPrivateFile(subdir_x)
        finally:
            s = resumeLog()
        if not ON_WINDOWS:
            self.assertEquals(0700, os.stat(subdir_x)[stat.ST_MODE] & 0777)
            self.assert_(stringContains(s,
                                 "Repairing permissions on file "+subdir_x))

        os.chmod(subdir_x, 0606)
        try:
            suspendLog()
            checkPrivateFile(subdir_x)
        finally:
            s = resumeLog()
        if not ON_WINDOWS:
            self.assertEquals(0600, os.stat(subdir_x)[stat.ST_MODE] & 0777)
            self.assert_(stringContains(s,
                                 "Repairing permissions on file "+subdir_x))

        # File not private, nofix.
        os.chmod(subdir_x, 0701)
        if not ON_WINDOWS:
            self.failUnlessRaises(MixFatalError, checkPrivateFile, subdir_x, 0)

#----------------------------------------------------------------------
# SIGHANDLERS
# FFFF Write tests here


#----------------------------------------------------------------------
# MMTP

# Run on a different port so we don't conflict with any actual servers
# running on this machine.
TEST_PORT = 40199

dhfile = pkfile = certfile = None

def _getTLSContext(isServer):
    "Helper function: create a new TLSContext object."
    global dhfile
    global pkfile
    global certfile
    if isServer:
        if dhfile is None:
            f = mix_mktemp()
            dhfile = f+"_dh"
            pkfile = f+"_pk"
            certfile = f+"_cert"
            dh_fname = os.environ.get("MM_TEST_DHPARAMS")
            got = 0
            if dh_fname and not USE_SLOW_MODE:
                dhfile = dh_fname
                try:
                    if not os.path.exists(dh_fname):
                        print "[Generating DH parameters...",
                        sys.stdout.flush()
                        _ml.generate_dh_parameters(dhfile, 0)
                        print "done.]",
                        sys.stdout.flush()
                        got = 1
                    else:
                        readFile(dh_fname)
                        got = 1
                except (OSError,TLSError):
                    print ("[Couldn't read/write %s]"%dh_fname),

            if not got:
                print "[Generating DH parameters (not caching)...",
                sys.stdout.flush()
                _ml.generate_dh_parameters(dhfile, 0)
                print "done.]",
                sys.stdout.flush()
            pk = getRSAKey(3,1024)
            pk.PEM_write_key(open(pkfile, 'w'), 0)
            ident = getRSAKey(0,2048)
            generateCertChain(certfile, pk, ident, "<testing>",
                              time.time(), time.time()+365*24*60*60)

        pk = _ml.rsa_PEM_read_key(open(pkfile, 'r'), 0)
        return _ml.TLSContext_new(certfile, pk, dhfile)
    else:
        return _ml.TLSContext_new()

def _getTLSContextKeyID():
    ident = getRSAKey(0,2048)
    keyid = sha1(ident.encode_key(1))
    return keyid

def _getMMTPServer(minimal=0,reject=0,port=TEST_PORT):
    """Helper function: create a new MMTP server with a listener connection
       Return a tuple of AsyncServer, ListenerConnection, list of received
       messages, and keyid."""
    server = mixminion.server.MMTPServer.AsyncServer()
    messagesIn = []
    def receivedHook(pkt,m=messagesIn):
        m.append(pkt)
    server.nJunkPackets = 0
    def junkCallback(server=server): server.nJunkPackets += 1
    def conFactory(sock, context=_getTLSContext(1),
                   receiveMessage=receivedHook,junkCallback=junkCallback,
                   reject=reject,server=server):
        tls = context.sock(sock, serverMode=1)
        sock.setblocking(0)
        con = mixminion.server.MMTPServer.MMTPServerConnection(sock,tls,
                                                               receiveMessage,
                                                         rejectPackets=reject)
        con.junkCallback = junkCallback
        server.register(con)
    def conFactoryMin(sock, context=_getTLSContext(1),server=server):
        tls = context.sock(sock, serverMode=1)
        sock.setblocking(0)
        con = mixminion.server.MMTPServer.MMTPServerConnection(sock,tls,
                                                               lambda m:None)
        con.junkCallback = lambda:None
        server.register(con)
    if minimal:
        conFactory = conFactoryMin
    listener = mixminion.server.MMTPServer.ListenConnection(
        socket.AF_INET, "127.0.0.1", port, 5, conFactory)
    server.register(listener)
    keyid = _getTLSContextKeyID()

    return server, listener, messagesIn, keyid

class FakeDeliverable:
    def __init__(self, s, isjunk=0):
        self._failed = self._succeeded = 0
        self._retriable = -1
        self._contents = s
        self._isjunk = isjunk
        if isjunk and not s:
            self._contents = getCommonPRNG().getBytes(32*1024)
    def getContents(self):
        return self._contents
    def isJunk(self):
        return self._isjunk
    def failed(self, retriable):
        assert not (self._failed or self._succeeded)
        self._failed = 1
        self._retriable = retriable
    def succeeded(self):
        assert not (self._failed or self._succeeded)
        self._succeeded = 1

class MMTPTests(TestCase):
    #XXXX This class is bulky, and has lots of cut-and-paste.  It could do
    #XXXX with a refactoring.
    def doTest(self, fn):
        """Wraps an underlying test function 'fn' to make sure we kill the
           MMTPServer, but don't block."""
        self.listener = self.server = None
        try:
            fn()
        finally:
            if self.listener is not None:
                self.server.remove(self.listener)
                self.listener.shutdown()
            if self.server is not None:
                count = 0
                while count < 100 and (self.server.connections):
                    self.server.process(0.1)
                    count = count + 1

    def testBlockingTransmission(self):
        self.doTest(self._testBlockingTransmission)

    def testNonblockingTransmission(self):
        self.doTest(self._testNonblockingTransmission)

    def testTimeout(self):
        self.doTest(self._testTimeout)

    def testRejected(self):
        self.doTest(self._testRejected)

    def _testBlockingTransmission(self):
        server, listener, packetsIn, keyid = _getMMTPServer()
        self.listener = listener
        self.server = server

        packets = ["helloxxx"*4096, "helloyyy"*4096]

        # Send m1, then junk, then renegotiate, then m2.
        server.process(0.1)
        routing = IPV4Info("127.0.0.1", TEST_PORT, keyid)
        t = threading.Thread(None,
                             mixminion.MMTPClient.sendPackets,
                             args=(routing,
                             [packets[0],"JUNK","RENEGOTIATE",packets[1]]))
        t.start()
        while len(packetsIn) < 2:
            server.process(0.1)
        while t.isAlive():
            server.process(0.1)
        t.join()

        for _ in xrange(3):
            server.process(0.1)

        self.failUnless(packetsIn == packets)
        self.assertEquals(1, server.nJunkPackets)

        # Now, with bad keyid.
        routing = IPV4Info("127.0.0.1", TEST_PORT, "Z"*20)
        t = threading.Thread(None,
                             self.failUnlessRaises,
                             args=(MixProtocolError,
                                   mixminion.MMTPClient.sendPackets,
                                   routing, packets))
        t.start()
        while t.isAlive():
            server.process(0.1)
        t.join()

    def testStallingTransmission(self):
        # XXXX I know this works, but there doesn't seem to be a good
        # XXXX way to test it.  It's hard to open a connection that
        # XXXX will surely stall.  For now, I'm disabling this test.
        if 1:
            return

        def threadfn(pausing):
            # helper fn to run in a different thread: bind a socket,
            # but don't listen.
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            sock.bind(("127.0.0.1", TEST_PORT))
            #sock.listen(5)
            #s, _ = sock.accept()
            while pausing[0] > 0:
                time.sleep(.1)
                pausing[0] -= .1
            time.sleep(2)
            #s.close()
            sock.close()
        pausing = [4]
        t = threading.Thread(None, threadfn, args=(pausing,))
        t.start()

        now = time.time()
        timedout = 0
        try:
            try:
                routing = IPV4Info("127.0.0.1", TEST_PORT, "Z"*20)
                mixminion.MMTPClient.sendPackets(routing, ["JUNK"],
                                                   connectTimeout=1)
                timedout = 0
            except mixminion.MMTPClient.TimeoutError:
                timedout = 1
        finally:
            passed = time.time() - now
            pausing[0] = 0
            t.join()

        self.assert_(passed < 2)
        self.assert_(timedout)

    def _testNonblockingTransmission(self):
        server, listener, packetsIn, keyid = _getMMTPServer()
        self.listener = listener
        self.server = server

        # Send m1, then junk, then renegotiate, then junk, then m2.
        dstr = mixminion.MMTPClient.DeliverableString
        tlscon = mixminion.TLSConnection.TLSConnection
        packets = ["helloxxx"*4096, "helloyyy"*4096]
        deliv = [FakeDeliverable(m) for m in packets]
        async = mixminion.server.MMTPServer.AsyncServer()
        clientcon = mixminion.server.MMTPServer.MMTPClientConnection(
            socket.AF_INET, "127.0.0.1", TEST_PORT, keyid)

        for d in [deliv[0],"JUNK","RENEGOTIATE","JUNK",deliv[1]]:
            if d == 'JUNK':
                clientcon.addPacket(dstr(isJunk=1))
            elif d == 'RENEGOTIATE':
                pass #XXXX implement or remove.
            else:
                clientcon.addPacket(d)

        async.register(clientcon)
        def clientThread(clientcon=clientcon, async=async):
            while clientcon.sock is not None:
                async.process(2)

        server.process(0.1)
        startTime = time.time()
        t = threading.Thread(None, clientThread)

        c = None
        t.start()
        while len(packetsIn) < 2:
            if c is None and len(server.connections) > 1:
                c = [ c for c in server.connections.values() if
                      isinstance(c, tlscon) ]
            server.process(0.1)
        while t.isAlive():
            server.process(0.1)
        t.join()
        endTime = time.time()

        self.assertEquals(len(packetsIn), len(packets))
        self.failUnless(packetsIn == packets)
        self.failUnless(c is not None)
        self.failUnless(len(c) == 1)
        self.failUnless(startTime <= c[0].lastActivity <= endTime)
        self.assertEquals(2, server.nJunkPackets)
        self.assert_(deliv[0]._succeeded)
        self.assert_(deliv[1]._succeeded)

        # Again, with bad keyid.
        deliv = [FakeDeliverable(p) for p in packets]
        clientcon = mixminion.server.MMTPServer.MMTPClientConnection(
            socket.AF_INET, "127.0.0.1", TEST_PORT, "Z"*20)
        for d in deliv:
            clientcon.addPacket(d)

        async.register(clientcon)
        def clientThread2(clientcon=clientcon, async=async):
            while not clientcon.isShutdown():
                async.process(2)

        try:
            suspendLog() # suppress warning
            server.process(0.1)
            t = threading.Thread(None, clientThread2)

            t.start()
            while t.isAlive():
                server.process(0.1)
            t.join()
        finally:
            resumeLog()  #unsuppress warning

        self.assert_(deliv[0]._failed)
        self.assert_(deliv[1]._failed)

    def _testTimeout(self):
        if 1:
            return #XXXX007 make this test work again.
        server, listener, packetsIn, keyid = _getMMTPServer()
        self.listener = listener
        self.server = server

        # This function wraps MMTPClient.sendPackets but catches exceptions.
        # Since we're going to run this function in a thread, we pass the
        # exception back through a list argument.
        def sendSlowlyAndCaptureException(exlst, pausing, targetIP, targetPort,
                                          targetKeyID, msgFast, msgSlow):
            try:
                con = mixminion.MMTPClient.BlockingClientConnection(
                    socket.AF_INET,targetIP,targetPort,targetKeyID)
                con.connect()
                con.sendPacket(msgFast)
                while pausing[0] > 0:
                    time.sleep(.1)
                    pausing[0] -= .1
                con.sendPacket(msgSlow)
                con.close()
            except:
                exlst.append(sys.exc_info())

        # Manually set the server's timeout threshold to 600 msec.
        server._timeout = 0.6
        server.process(0.1)
        excList = []
        pausing = [10]
        t = threading.Thread(None,
              sendSlowlyAndCaptureException,
              args=(excList, pausing, "127.0.0.1", TEST_PORT, keyid,
                    "helloxxx"*4096, "helloyyy"*4096))
        t.start()
        timedOut = 0 # flag: have we really timed out?
        try:
            suspendLog() # stop logging, but wait for the timeout message.
            while len(packetsIn) < 2:
                server.process(0.1)
                # If the number of connections changes around the call
                # to tryTimeout, the timeout has occurred.
                nConnections = len(server.readers)+len(server.writers)
                server.tryTimeout(time.time())
                if len(server.readers)+len(server.writers) < nConnections:
                    timedOut = 1
                    break
            # Did we really time out the connection, or did we end normally?
            self.assert_(timedOut)
        finally:
            logMessage = resumeLog()
        # Did we log the timeout?
        self.assert_(stringContains(logMessage, "timed out"))
        # Was the one message we expected in fact transmitted?
        self.assertEquals([packetsIn[0]], ["helloxxx"*4096])

        # Now stop the transmitting thread.  It will notice that its
        # connection has been forcibly closed.
        pausing[0] = 0
        t.join()
        # Was an exception raised?
        self.assertEquals(1, len(excList))
        # Was it the right exception?
        ex = excList[0][1]
        self.assert_(isinstance(ex, MixError))
        self.assert_(isinstance(ex.base, _ml.TLSClosed))

        for _ in xrange(3):
            server.process(0.1)

    def _testRejected(self):
        server, listener, packetsIn, keyid = _getMMTPServer(reject=1)
        self.listener = listener
        self.server = server

        packets = ["helloxxx"*4096, "helloyyy"*4096]
        # Send 2 packets -- both should be rejected.
        server.process(0.1)
        routing = IPV4Info("127.0.0.1", TEST_PORT, keyid)
        ok=[0];done=[0]
        def _t(routing=routing, packets=packets, ok=ok, done=done):
            try:
                mixminion.MMTPClient.sendPackets(routing,packets)
            except mixminion.Common.MixProtocolError:
                ok[0] = 1
            done[0] = 1

        t = threading.Thread(None, _t)
        t.start()
        while not done[0]:
            server.process(0.1)
        t.join()
        for _ in xrange(3):
            server.process(0.1)

        self.failUnless(ok[0])
        self.failUnless(len(packetsIn) == 0)

        # Send m1, then junk, then renegotiate, then junk, then m2.
        packets = ["helloxxx"*4096, "helloyyy"*4096]
        deliv = [FakeDeliverable(p) for p in packets]
        async = mixminion.server.MMTPServer.AsyncServer()

        clientcon = mixminion.server.MMTPServer.MMTPClientConnection(
           socket.AF_INET, "127.0.0.1", TEST_PORT, keyid)
        for d in deliv:
            clientcon.addPacket(d)

        async.register(clientcon)
        def clientThread(clientcon=clientcon, async=async):
            while not clientcon.isShutdown():
                async.process(2)

        t = threading.Thread(None, clientThread)
        t.start()
        while clientcon.sock is not None:
            server.process(0.1)
        while t.isAlive():
            server.process(0.1)
        t.join()
        self.assertEquals(len(packetsIn), 0)
        self.assertEquals(deliv[0]._retriable, 1)
        self.assertEquals(deliv[1]._retriable, 1)

#----------------------------------------------------------------------
# Config files

class TestConfigFile(_ConfigFile):
    _syntax = { 'Sec1' : {'__SECTION__': ('REQUIRE', None, None),
                          'Foo': ('REQUIRE', None, None),
                          'Bar': ('ALLOW', None, "default"),
                          'Baz': ('ALLOW', None, None),},
                'Sec2' : {'Fob': ('ALLOW*', None, None),
                          'Bap': ('REQUIRE', None, None),
                          'Quz': ('REQUIRE*', None, None), },
                'Sec3' : {'IntAS': ('ALLOW', "int", None),
                          'IntAS2': ('ALLOW', "int", None),
                          'IntASD': ('ALLOW', "int", "5"),
                          'IntASD2': ('ALLOW', "int", "5"),
                          'IntAM': ('ALLOW*', "int", None),
                          'IntAMD': ('ALLOW*', "int", ["5", "2"]),
                          'IntAMD2': ('ALLOW*', "int", ["5", "2"]),
                          'IntRS': ('REQUIRE', "int", None),
                          'Quz' : ('ALLOW', None, None) }
                }
    def __init__(self, fname=None, string=None, restrict=0):
        self._restrictFormat = restrict
        _ConfigFile.__init__(self,fname,string)

class ConfigFileTests(TestCase):
    def testValidFiles(self):

        TCF = TestConfigFile
        # Try a minimal file.
        shorterString = """[Sec1]\nFoo a\n"""
        f = TCF(string=shorterString)
        self.assertEquals(f['Sec1']['Foo'], 'a')
        # Try a slightly more spaceful version of the above.
        f = TCF(string="""\n\n[ Sec1 ]  \n  \n\nFoo a  \n""")
        self.assertEquals(f['Sec1']['Foo'], 'a')
        self.assertEquals(f['Sec2'], {})

        # Now, try all the syntactical possibilities, and all the fields
        longerString = """[Sec1]

Foo=  abcde f

Bar bar
Baz:
  baz
  and more baz
  and more baz
[Sec2]

# Comment
Bap +
Quz 99 99


Fob=1
Quz : 88
     88

[Sec3]
IntAS=9
IntASD=10
IntAMD=8
IntAMD=10
IntRS=5

  """

        f = TCF(string=longerString)
        self.assertEquals(f['Sec1']['Foo'], 'abcde f')
        self.assertEquals(f['Sec1']['Bar'], 'bar')
        self.assertEquals(f['Sec1']['Baz'], ' baz and more baz and more baz')
        self.assertEquals(f['Sec2']['Bap'], '+')
        self.assertEquals(f['Sec2']['Fob'], ['1'])
        self.assertEquals(f['Sec2']['Quz'], ['99 99', '88 88'])
        self.assertEquals(f.getSectionItems('Sec2'),
                          [ ('Bap', '+'),
                            ('Quz', '99 99'),
                            ('Fob', '1'),
                            ('Quz', '88 88') ])

        # Make sure that str(f) works.
        self.assertEquals(str(f),
           ("[Sec1]\nFoo: abcde f\nBar: bar\nBaz:  baz and more baz"+
            " and more baz\n\n[Sec2]\nBap: +\nQuz: 99 99\nFob: 1\n"+
            "Quz: 88 88\n\n[Sec3]\nIntAS: 9\nIntASD: 10\nIntAMD: 8\n"+
            "IntAMD: 10\nIntRS: 5\n\n"))
        # Test file input
        fn = mix_mktemp()

        writeFile(fn, longerString)
        f = TCF(fname=fn)
        self.assertEquals(f['Sec1']['Bar'], 'bar')
        self.assertEquals(f['Sec2']['Quz'], ['99 99', '88 88'])

        self.assertEquals(f['Sec3']['IntAS'], 9)
        self.assertEquals(f['Sec3']['IntAS2'], None)
        self.assertEquals(f['Sec3']['IntASD'], 10)
        self.assertEquals(f['Sec3']['IntASD2'], 5)
        self.assertEquals(f['Sec3']['IntAM'], [])
        self.assertEquals(f['Sec3']['IntAMD'], [8,10])
        self.assertEquals(f['Sec3']['IntAMD2'], [5,2])
        self.assertEquals(f['Sec3']['IntRS'], 5)

        # Test restricted mode
        s = "[Sec1]\nFoo: Bar\nBaz: Quux\n[Sec3]\nIntRS: 9\n"
        f = TCF(string=s, restrict=1)
        self.assertEquals(f['Sec1']['Foo'], "Bar")
        self.assertEquals(f['Sec3']['IntRS'], 9)

    def testBadFiles(self):
        def fails(string, self=self):
            self.failUnlessRaises(ConfigError, TestConfigFile, None, string)


        # Missing section header
        fails("Foo = Bar\n")
        # Invalid indentation on key
        fails("[Sec1]\n  Foo = Bar\n")
        # Invalid indentation on header
        fails("  [Sec1]\n  Foo = Bar\n")
        # Unrecognized key
        fails("[Sec1]\nFoo! Bar\n")

        fails("[Sec1]\nFoob: Bar\n") # No such key
        fails("[Sec1]\nFoo: Bar\nFoo: Bar\n") #  Duplicate key
        fails("[Sec1]\nBaz: 3\n") # Missing key
        fails("[Sec2]\nBap = 9\nQuz=6\n") # Missing section
        fails("[Sec1]\nFoo 1\n[Sec2]\nBap = 9\n") # Missing require*
        fails("[Sec1]\nFoo: Bar\n[Sec3]\nIntRS=Z\n") # Failed validation

        try:
            TestConfigFile(None,"[Sec1]\nFoo: 1\nBap:1\n")
        except ConfigError, e:
            self.assertEndsWith(str(e),
                                "Unrecognized key Bap on line 3. This "
                                "key belongs in Sec2, but appears in Sec1.")
        try:
            TestConfigFile(None,"[Sec1]\nFoo: 1\nQuz:1\n")
        except ConfigError, e:
            self.assertEndsWith(str(e),
                               "Unrecognized key Quz on line 3. This key "
                               "belongs in Sec2 or Sec3, but appears in Sec1.")

        try:
            TestConfigFile(None,"[Sec1]\nFoo: 1\nBan:1\n")
        except ConfigError, e:
            self.assertEndsWith(str(e), "Unrecognized key Ban on line 3")

        # now test the restricted format
        def failsR(string, self=self):
            self.failUnlessRaises(ConfigError, TestConfigFile, None, string, 1)
        failsR("[Sec1]\nFoo=Bar\n")
        failsR("[Sec1]\nFoo Bar\n")
        failsR("[Sec1]\n\nFoo: Bar\n")
        failsR("\n[Sec1]\nFoo: Bar\n")
        failsR("\n[Sec1]\nFoo: Bar\n\n")
        failsR("")
        failsR("\n")

    def testValidationFns(self):
        import mixminion.Config as C

        ## First, check valid cases.
        # boolean
        self.assertEquals(C._parseBoolean("yes"), 1)
        self.assertEquals(C._parseBoolean(" NO"), 0)
        # severity
        self.assertEquals(C._parseSeverity("error"), "ERROR")
        # serverMode
        self.assertEquals(C._parseServerMode(" relay "), "relay")
        self.assertEquals(C._parseServerMode("Local"), "local")
        # interval
        self.assertEquals(str(C._parseInterval(" 1 sec ")),"1 second")
        self.assertEquals(str(C._parseInterval(" 99 sec ")),"99 seconds")
        self.assertFloatEq(float(C._parseInterval("1.5 minutes")), 90)
        h2 = C._parseInterval("2 houRS")
        m120 = C._parseInterval("120 minutes")
        self.assertEquals(str(h2), "2 hours")
        self.assertEquals(str(m120), "120 minutes")
        self.assert_(int(h2) == int(m120) == 7200)
        m120.reduce()
        self.assertEquals(str(m120), "2 hours")
        # size
        self.assertEquals(C._parseSize(" 30 bytes"), 30L)
        self.assertEquals(C._parseSize(" 3000 b"), 3000L)
        self.assertEquals(C._parseSize("50k"), 50*1024L)
        self.assertEquals(C._parseSize("50k "), 50*1024L)
        self.assertEquals(C._parseSize("50 "), 50L)
        self.assertEquals(C._parseSize("50"), 50L)
        self.assertEquals(C._parseSize("12.3M"), long(12.3*(1<<20L)))
        # IntervalList
        self.assertEquals(C._parseIntervalList(" 5 sec, 1 min, 2 hours"),
                          [ 5, 60, 7200 ])
        self.assertEquals([5,5,5,5,5,5, 8*3600,8*3600,8*3600,8*3600,],
              C._parseIntervalList("5 sec for 30 sec, 8 hours for 1.3 days"))
        self.assertEquals([60], C._parseIntervalList("1 min for 1 min"))
        self.assertEquals([60,60], C._parseIntervalList("1 min for 1.5 min"))
        self.assertEquals([60,60],
               C._parseIntervalList("EVERY  1 min for 1.5 min"))
        # int
        self.assertEquals(C._parseInt("99"), 99)
        # IP
        self.assertEquals(C._parseIP("192.168.0.1"), "192.168.0.1")
        # IP6
        self.assertEquals(C._parseIP6("::"), "::")
        self.assertEquals(C._parseIP6("A::"), "A::")
        self.assertEquals(C._parseIP6("::A"), "::A")
        self.assertEquals(C._parseIP6("FE0F::A"), "FE0F::A")
        self.assertEquals(C._parseIP6("1:2:3:4:5:6:7:8"), "1:2:3:4:5:6:7:8")
        self.assertEquals(C._parseIP6("1:2:3::5:6:7:8"), "1:2:3::5:6:7:8")
        self.assertEquals(C._parseIP6("1:2:3:4:5:6:1.2.3.4"),
                          "1:2:3:4:5:6:1.2.3.4")
        self.assertEquals(C._parseIP6("FFFF::1.2.3.4"),"FFFF::1.2.3.4")
        self.assertEquals(C._parseIP6("::000:1.2.3.4"),"::000:1.2.3.4")
        # Host
        self.assertEquals(C._parseHost("foo.bar.baz "), "foo.bar.baz")
        # AddressSet
        pa = C._parseAddressSet_allow
        self.assertEquals(pa("*"), ("0.0.0.0", "0.0.0.0", 48099, 48099))
        self.assertEquals(pa("192.168.0.1/255.255.0.0"),
                          ("192.168.0.1", "255.255.0.0", 48099, 48099))
        self.assertEquals(pa("192.168.0.1 /  255.255.0.0  23-99"),
                          ("192.168.0.1", "255.255.0.0", 23, 99))
        self.assertEquals(pa("192.168.0.1 /  255.255.0.0  23"),
                          ("192.168.0.1", "255.255.0.0", 23, 23))
        self.assertEquals(pa("192.168.0.1"),
                          ("192.168.0.1", "255.255.255.255", 48099, 48099))
        self.assertEquals(pa("192.168.0.1",0),
                          ("192.168.0.1", "255.255.255.255", 0, 65535))

        # Command
        if not sys.platform == 'win32':
            # WIN32 This should get implemented for Windows.
            c = C._parseCommand("ls -l")
            self.assert_(os.path.exists(c[0]) and c[0].endswith("/ls"))
            self.assertEquals(c[1], ['-l'])
            c = C._parseCommand("rm")
            self.assert_(os.path.exists(c[0]) and c[0].endswith("/rm"))
            self.assertEquals(c[1], [])
            self.assertEquals(C._parseCommand("/bin/ls"), ("/bin/ls", []))
            self.failUnless(C._parseCommand(sys.executable)[0] is not None)

        # Base64
        self.assertEquals(C._parseBase64(" YW\nJj"), "abc")
        self.assertEquals(C._parseBase64(" Y W\nJ j"), "abc")
        # Hex
        self.assertEquals(C._parseHex(" C0D0"), "\xC0\xD0")
        self.assertEquals(C._parseHex(" C0\n D 0"), "\xC0\xD0")
        # Date
        tm = C._parseDate("2002/05/30")
        self.assertEquals(time.gmtime(tm)[:6], (2002,5,30,0,0,0))
        tm = C._parseDate("2000/01/01")
        self.assertEquals(time.gmtime(tm)[:6], (2000,1,1,0,0,0))
        tm = C._parseDate("2000-05-03")
        self.assertEquals(time.gmtime(tm)[:6], (2000,5,3,0,0,0))
        # Time
        tm = C._parseTime("2001-12-25 06:15:10")
        self.assertEquals(time.gmtime(tm)[:6], (2001,12,25,6,15,10))
        tm = C._parseTime("2001-12-25 06:15:10.623")
        self.assertEquals(time.gmtime(tm)[:6], (2001,12,25,6,15,10))
        # nicknames
        self.assertEquals(C._parseNickname("Mrs-Premise"), "Mrs-Premise")
        # Filenames
        self.assertEquals(C._parseFilename(" ab/c/d"), "ab/c/d")
        self.assertEquals(C._parseFilename("  ~/ab/c/d"),
                          os.path.expanduser("~/ab/c/d"))
        self.assertEquals(C._parseFilename(" 'ab/c d/d' "), "ab/c d/d")
        self.assertEquals(C._parseFilename('  "ab/c d/d" '), "ab/c d/d")

        SC = mixminion.server.ServerConfig
        # Fractions
        self.assertFloatEq(SC._parseFraction("90 %"), .90)
        self.assertFloatEq(SC._parseFraction(" 90%"), .90)
        self.assertFloatEq(SC._parseFraction(".02"), .02)
        self.assertFloatEq(SC._parseFraction("1"), 1)
        self.assertFloatEq(SC._parseFraction("0"), 0)
        self.assertFloatEq(SC._parseFraction("100%"), 1)
        self.assertFloatEq(SC._parseFraction("0%"), 0)
        # Mix algorithms
        self.assertEquals(SC._parseMixRule(" Cottrell"), "CottrellMixPool")
        self.assertEquals(SC._parseMixRule("binomialCottrell"),
                          "BinomialCottrellMixPool")
        self.assertEquals(SC._parseMixRule("TIMED"), "TimedMixPool")


        ##
        # Now, try the failing cases.
        def fails(fn, val, self=self):
            self.failUnlessRaises(ConfigError, fn, val)

        fails(C._parseBoolean, "yo")
        fails(C._parseBoolean, "'yo'")
        fails(C._parseBoolean, "")
        fails(C._parseSeverity, "really bad")
        fails(C._parseServerMode, "whatever")
        fails(C._parseInterval, "seconds")
        fails(C._parseInterval, "15")
        fails(C._parseInterval, " 10 intervals")
        fails(C._parseIntervalList, "1 min for 1 min for 1 min")
        fails(C._parseIntervalList, "1 min for 2 fnords")
        fails(C._parseIntervalList, "0 min for 2 hours")
        fails(C._parseIntervalList, "every 30 minutes")
        fails(C._parseInt, "9.9")
        fails(C._parseInt, "9abc")
        fails(C._parseIP, "256.0.0.1")
        fails(C._parseIP, "192.0.0")
        fails(C._parseIP, "192.0.0.0.0")
        fails(C._parseIP, "A.0.0.0")
        fails(C._parseIP6, "100::200::300")
        fails(C._parseIP6, "10000::")
        fails(C._parseIP6, ":::")
        fails(C._parseIP6, "1:2:3:4:5:6:7:8:9")
        fails(C._parseIP6, "F:E:D:B:C:A:7:1.2.3.4")
        fails(C._parseIP6, ":")
        fails(C._parseIP6, "")
        fails(C._parseHost, ".")
        fails(C._parseHost, "foo..bar")
        fails(pa, "1/1")
        fails(pa, "192.168.0.1 50-40")
        fails(pa, "192.168.0.1 50-9999999")
        fails(C._parseBase64, "Y")
        fails(C._parseHex, "Z")
        fails(C._parseHex, "A")
        fails(C._parseDate, "2000/1/1")
        fails(C._parseDate, "2000/10-10")
        fails(C._parseDate, "2000-10/10")
        fails(C._parseDate, "2000-50-01")
        fails(C._parseDate, "2000/20/01 12:12:12")
        fails(C._parseTime, "2000/50/01 12:12:12")
        fails(C._parseDate, "2000/20/01 12:12:12")
        fails(C._parseDate, "2000-50-01 12:12:12.3")
        fails(C._parseTime, "2000-20-01 12:12:99")
        fails(C._parseNickname, "Mrs Premise")
        fails(C._parseNickname, "-Mrs-Premise")
        fails(C._parseNickname, "../../../AllYourBase")
        fails(C._parseNickname, "Z"*129)
        fails(C._parseNickname, ""*129)
        fails(C._parseNickname, "; rm -f /etc/important;")

        nonexistcmd = '/file/that/does/not/exist'
        if not os.path.exists(nonexistcmd):
            fails(C._parseCommand, nonexistcmd)
        else:
            print 'Whoa. Kurt Go"del would be proud of you.'

        # Nobody would ever have an executable named after my sister's
        # cats, would they?
        nonexistcmd = 'LindenAndPierre -meow'
        try:
            cmd, opts = C._parseCommand(nonexistcmd)
            if os.path.exists(cmd):
                # Ok, I guess they would.
                self.assertEquals(opts, ["-meow"])
            else:
                self.fail("_parseCommand is not working as expected")
        except ConfigError:
            # This is what we expect
            pass

        # Filenames:
        fails(C._parseFilename, " 'ab/cd")

        # IntervalSet validation
        def warns(mixInterval, retryList, self=self):
            ent = mixminion.Config._parseIntervalList(retryList)
            try:
                suspendLog()
                mixminion.server.ServerConfig._validateRetrySchedule(
                    mixInterval, ent, "Section")
            finally:
                r = resumeLog()
            self.assert_(stringContains(r, "[WARN]"))
        warns(30*60, "every .6 hour for 20 hours") # < 1day
        warns(30*60, "every 4 days for 1 month") # > 2 weeks
        warns(30*60, "every 2 days for 4 days") # < twice
        warns(30*60, "every .2 hours for 1 hour, every 1 day for 1 week")#<mix
        warns(30*60, "every 5 days for 1 week") # too few attempts
        warns(30*60, "every 1 hour for 1 week") # too many attempts

        # Fractions
        fails(SC._parseFraction, "90")
        fails(SC._parseFraction, "-.01")
        fails(SC._parseFraction, "101%")
        fails(SC._parseFraction, "1.01")
        fails(SC._parseFraction, "5")
        # Mix algorithms
        fails(SC._parseMixRule, "")
        fails(SC._parseMixRule, "nonesuch")

#----------------------------------------------------------------------
# Server descriptors
SERVER_CONFIG = """
[Server]
EncryptIdentityKey: no
PublicKeyLifetime: 10 days
EncryptPrivateKey: no
BaseDir: %s
Mode: relay
Nickname: The-Server
Contact-Email: a@b.c
Comments: This is a test of the emergency
   broadcast system

[Incoming/MMTP]
Enabled = yes
IP: 192.168.0.1
Hostname: Theserver
Allow: 192.168.0.16 1-1024
Deny: 192.168.0.16
Allow: *

[Outgoing/MMTP]
Enabled = yes
Allow: *
Retry: every 1 hour for 1 day, every 1 day for 1 week
"""

MBOX_SEC = """
[Delivery/MBOX]
Enabled: yes
AddressFile: %s
ReturnAddress: a@b.c
RemoveContact: b@c.d
Retry: every 1 hour for 1 day, every 1 day for 1 week
"""

FRAGMENTED_SEC = """
[Delivery/Fragmented]
Enabled yes
MaximumSize: 100k

"""

SERVER_CONFIG_SHORT = """
[Server]
EncryptIdentityKey: no
PublicKeyLifetime: 10 days
EncryptPrivateKey: no
Homedir: %s
Mode: relay
Nickname: fred-the-bunny
Contact-Email: a@b.c
"""

class ServerInfoTests(TestCase):
    def test_displayServer(self):
        dsbr = mixminion.ServerInfo.displayServerByRouting
        dsba = mixminion.ServerInfo.displayServerByAddress
        eq = self.assertEquals

        # Test without keyid resolver
        eq(dsbr(IPV4Info("1.2.3.4", 48099, "x"*20)),
           "server at 1.2.3.4:48099")
        eq(dsbr(MMTPHostInfo("a.b.com", 48099, "x"*20)),
           "server at a.b.com:48099")
        eq(dsba("1.2.3.4",48099), "host at 1.2.3.4:48099")
        eq(dsba("1.2.3.4",48099, "X.Y"), "host at 1.2.3.4:48099")

        # Test with keyid resolver
        def resolver(keyid):
            if keyid[0] == 'a': return "Kenichi"
            elif keyid[1] == 'b': return "Tima"
            else: return None
        def reverseResolver(addr):
            if addr == "1.2.3.4": return "Each"
            elif addr == "a.b.c.d": return "Peach"
            else: return None
        try:
            mixminion.ServerInfo._keyIDToNicknameFn = resolver
            mixminion.ServerInfo._addressToNicknameFn = reverseResolver
            eq(dsbr(IPV4Info("1.2.3.4", 48099, "b"*20)),
               "'Tima' at 1.2.3.4:48099")
            eq(dsbr(IPV4Info("1.2.3.4", 48099, "x"*20)),
               "server at 1.2.3.4:48099")
            eq(dsbr(MMTPHostInfo("ken.com", 48099, "a"*20)),
               "'Kenichi' at ken.com:48099")
            eq(dsba("1.2.3.4",48099), "host at 1.2.3.4:48099 (Each?)")
            eq(dsba("1.2.3.4",48099,"a.b.c.d"),
               "host at 1.2.3.4:48099 (Peach?)")
            eq(dsba("3.4.5.6",48099,"x.y.z"),
               "host at 3.4.5.6:48099")
        finally:
            mixminion.ServerInfo._keyIDToNicknameFn = None
            mixminion.ServerInfo._addressToNicknameFn = None

    def test_ServerInfo(self):
        # Try generating a serverinfo and see if its values are as expected.
        identity = getRSAKey(1, 2048)
        d = mix_mktemp()
        try:
            suspendLog()
            conf = mixminion.server.ServerConfig.ServerConfig(string=(SERVER_CONFIG % mix_mktemp()))
        finally:
            resumeLog()
        if not os.path.exists(d):
            os.mkdir(d, 0700)

        try:
            overrideDNS({"Theserver": "192.168.0.1"})
            inf = generateServerDescriptorAndKeys(conf,
                                                  identity,
                                                  d,
                                                  "key1",
                                                  d)
        finally:
            undoReplacedAttributes()
        info = mixminion.ServerInfo.ServerInfo(string=inf)
        eq = self.assertEquals
        eq(info['Server']['Descriptor-Version'], "0.2")
        self.assert_(stringContains(inf, "\nIP: 192.168.0.1\n"))
        eq(info['Incoming/MMTP']['Hostname'], "Theserver")
        eq(info['Server']['Nickname'], "The-Server")
        self.failUnless(0 <= time.time()-info['Server']['Published'] <= 120)
        self.failUnless(0 <= time.time()-info['Server']['Valid-After']
                          <= 24*60*60)
        eq(info['Server']['Valid-Until']-info['Server']['Valid-After'],
           10*24*60*60)
        eq(info['Server']['Contact'], "a@b.c")
        eq(info['Server']['Software'], "Mixminion %s"%mixminion.__version__)
        eq(info['Server']['Packet-Versions'], ["0.3"])
        eq(info['Server']['Comments'],
           "This is a test of the emergency broadcast system")

        eq(info['Incoming/MMTP']['Version'], "0.1")
        eq(info['Incoming/MMTP']['Port'], 48099)
        eq(info['Incoming/MMTP']['Protocols'], ["0.3"])
        eq(info['Outgoing/MMTP']['Version'], "0.1")
        eq(info['Outgoing/MMTP']['Protocols'], ["0.3"])
        eq(info['Incoming/MMTP']['Allow'], [("192.168.0.16", "255.255.255.255",
                                            1,1024),
                                           ("0.0.0.0", "0.0.0.0",
                                            48099, 48099)] )
        eq(info['Incoming/MMTP']['Deny'], [("192.168.0.16", "255.255.255.255",
                                            0,65535),
                                           ])
        eq(info['Delivery/MBOX'].get('Version'), None)

        # Now add frag and mbox
        af = mix_mktemp()
        writeFile(af, "")
        try:
            suspendLog()
            conf = mixminion.server.ServerConfig.ServerConfig(
                string=(SERVER_CONFIG % mix_mktemp()+FRAGMENTED_SEC+
                        (MBOX_SEC%af)))
        finally:
            resumeLog()
        if not os.path.exists(d):
            os.mkdir(d, 0700)

        try:
            overrideDNS({"Theserver" : "192.168.0.1"})
            inf = generateServerDescriptorAndKeys(conf,
                                                  identity,
                                                  d,
                                                  "key2",
                                                  d)
        finally:
            undoReplacedAttributes()
        info = mixminion.ServerInfo.ServerInfo(string=inf)
        eq(info['Delivery/MBOX'].get('Version'), '0.1')
        eq(info['Delivery/Fragmented'].get('Version'), '0.1')
        eq(info['Delivery/Fragmented'].get('Maximum-Fragments'), 6)

        # Test features
        rfn = mixminion.Config.resolveFeatureName
        SI = mixminion.ServerInfo.ServerInfo
        self.assertEquals(rfn("softwarE",SI), ("Server", "Software"))
        self.assertEquals(rfn("delivery/mbox:version",SI),
                          ("Delivery/MBOX", "Version"))
        self.assertEquals(rfn("caps",SI), ('-','caps'))
        self.assertRaises(UIError, rfn, "version", SI)
        self.assertRaises(UIError, rfn, "versiojdkasldj", SI)
        self.assertRaises(UIError, rfn, "Server:foob", SI)
        self.assertRaises(UIError, rfn, "Beep:version", SI)
        self.assertEquals(info.getFeature("Server", "Packet-Key"),
                          "<public key>")
        self.assertEquals(info.getFeature("Incoming/MMTP", "Port"), "48099")
        self.assertEquals(info.getFeature("-", "caps"), "mbox relay frag")

        # Check the more complex helpers.
        self.assert_(info.isValidated())
        self.assertEquals(info.getIntervalSet(),
                          IntervalSet([(info['Server']['Valid-After'],
                                        info['Server']['Valid-Until'])]))

        self.assert_(not info.isExpiredAt(time.time()))
        self.assert_(not info.isExpiredAt(time.time()-25*60*60))
        self.assert_(info.isExpiredAt(time.time()+24*60*60*30))

        self.assert_(info.isValidAt(time.time()))
        self.assert_(not info.isValidAt(time.time()-25*60*60))
        self.assert_(not info.isValidAt(time.time()+24*60*60*30))

        self.assert_(info.isValidFrom(time.time(), time.time()+60*60))
        self.assert_(not info.isValidFrom(time.time()-25*60*60,
                                          time.time()+60*60))
        self.assert_(not info.isValidFrom(time.time()-25*60*60,
                                          time.time()+24*60*60*30))
        self.assert_(not info.isValidFrom(time.time(),
                                          time.time()+24*60*60*30))
        self.assert_(not info.isValidFrom(time.time()-25*60*60,
                                          time.time()-23*60*60))
        self.assert_(not info.isValidFrom(time.time()+24*60*60*30,
                                          time.time()+24*60*60*31))

        self.assert_(info.isValidAtPartOf(time.time(), time.time()+60*60))
        self.assert_(info.isValidAtPartOf(time.time()-25*60*60,
                                          time.time()+60*60))
        self.assert_(info.isValidAtPartOf(time.time()-25*60*60,
                                          time.time()+24*60*60*30))
        self.assert_(info.isValidAtPartOf(time.time(),
                                          time.time()+24*60*60*30))
        self.assert_(not info.isValidAtPartOf(time.time()-40*60*60,
                                              time.time()-39*60*60))
        self.assert_(not info.isValidAtPartOf(time.time()+24*60*60*30,
                                              time.time()+24*60*60*31))

        self.assert_(info.isNewerThan(time.time()-60*60))
        self.assert_(not info.isNewerThan(time.time()+60))

        self.assertUnorderedEq(info.getCaps(), ["frag", "relay", "mbox"])

        self.assertEquals(info.getIncomingMMTPProtocols(), ["0.3"])
        self.assertEquals(info.getOutgoingMMTPProtocols(), ["0.3"])

        # Now check whether we still validate the same after some corruption
        self.assertStartsWith(inf, "[Server]\n")
        self.assertEndsWith(inf, "\n")
        self.assert_(stringContains(inf, "b.c\n"))
        inf2 = inf.replace("[Server]\n", "[Server] \r")
        inf2 = inf2.replace("b.c\n", "b.c\r\n")
        inf2 = inf2.replace("0.1\n", "0.1  \n")
        mixminion.ServerInfo.ServerInfo(string=inf2)

        # Make sure we accept an extra key.
        inf2 = inf+"Unexpected-Key: foo\n"
        inf2 = mixminion.ServerInfo.signServerInfo(inf2, identity)
        try:
            suspendLog()
            mixminion.ServerInfo.ServerInfo(string=inf2)
        finally:
            s = resumeLog()
        self.assert_(stringContains(s,"Unrecognized key Unexpected-Key on line"))

        # Now make sure everything was saved properly
        keydir = os.path.join(d, "key_key2")
        eq(inf, readFile(os.path.join(keydir, "ServerDesc")))
        mixminion.server.ServerKeys.ServerKeyset(d, "key2", d) # Can we load?
        packetKey = Crypto.pk_PEM_load(
            os.path.join(keydir, "mix.key"))
        eq(packetKey.get_public_key(),
           info['Server']['Packet-Key'].get_public_key())
#        eq(Crypto.sha1(identity.encode_key(1)),
#           info['Incoming/MMTP']['Key-Digest'])

        # Now check the digest and signature
        identityPK = info['Server']['Identity']
        pat = re.compile(r'^(Digest:|Signature:).*$', re.M)
        x = sha1(pat.sub(r'\1', inf))

        eq(info['Server']['Digest'], x)
        eq(x, Crypto.pk_check_signature(info['Server']['Signature'],
                                                  identityPK))

        # Now check pickleability
        pickled = cPickle.dumps(info, 1)
        loaded = cPickle.loads(pickled)
        eq(info['Server']['Digest'], loaded['Server']['Digest'])
        eq(info['Server']['Identity'].get_public_key(),
           loaded['Server']['Identity'].get_public_key())
        eq(info['Server']['Published'], loaded['Server']['Published'])
        eq(info.isValidated(), loaded.isValidated())

        # Other functionality.
        eq(info.getMMTPHostInfo(), MMTPHostInfo("Theserver", 48099, info.getKeyDigest()))
        eq(info.getMMTPHostInfo(), info.getRoutingInfo())
        self.assert_(info.canStartAt())

        #XXXX this is a workaround to deal with the fact that we've
        #XXXX opened a fragment DB just to configure the server. Not
        #XXXX cool.  Startup should get refactored.
        conf.getModuleManager().close()

        # Now with a shorter configuration
        try:
            suspendLog()
            conf = mixminion.server.ServerConfig.ServerConfig(string=(SERVER_CONFIG_SHORT%mix_mktemp())+
                                           """[Incoming/MMTP]
Enabled: yes
Hostname: Theserver2
IP: 192.168.0.99
""")
        finally:
            resumeLog()
        try:
            overrideDNS({"Theserver2" : "192.168.0.99"})
            generateServerDescriptorAndKeys(conf,
                                            identity,
                                            d,
                                            "key2",
                                            d)
        finally:
            undoReplacedAttributes()
        # Now with a bad signature
        sig2 = Crypto.pk_sign(sha1("Hello"), identity)
        sig2 = base64.encodestring(sig2).replace("\n", "")
        sigpat = re.compile('^Signature:.*$', re.M)
        badSig = sigpat.sub("Signature: %s" % sig2, inf)
        self.failUnlessRaises(ConfigError,
                              mixminion.ServerInfo.ServerInfo,
                              None, badSig)

        # But make sure we don't check the sig on assumeValid
        mixminion.ServerInfo.ServerInfo(None, badSig, assumeValid=1)

        # Now with a bad digest
        badSig = inf.replace("a@b.c", "---")
        self.failUnlessRaises(ConfigError,
                              mixminion.ServerInfo.ServerInfo,
                              None, badSig)

        # Test superseding
        ServerInfo = mixminion.ServerInfo.ServerInfo
        examples = getExampleServerDescriptors()
        bobs = [ ServerInfo(string=s, assumeValid=1) for s in examples["Bob"] ]
        freds= [ ServerInfo(string=s, assumeValid=1) for s in examples["Fred"]]
        self.assert_(bobs[1].isSupersededBy([bobs[3], bobs[4]]))
        self.assert_(not bobs[1].isSupersededBy([bobs[3], bobs[5]]))
        self.assert_(not bobs[1].isSupersededBy([]))
        self.assert_(not bobs[1].isSupersededBy([bobs[1]]))
        self.assert_(not bobs[1].isSupersededBy([freds[2]]))

        # Test whether we ignore server descriptors with unknown versions.
        try:
            inf2 = inf.replace("Descriptor-Version: 0.2\n",
                               "Descriptor-Version: 0.99\n")
            inf2 = inf2.replace("\nIP: ", "\nIP: x.")
            inf2 = inf2.replace("\nNickname: ", "\nNickname-XX: ")
            ServerInfo(string=inf2)
            self.fail("Missing ConfigError")
        except ConfigError, p:
            self.assertEquals(str(p), "Unrecognized descriptor version: 0.99")

        # Try regenerating server descriptor with existing keys.
        key2 = mixminion.server.ServerKeys.ServerKeyset(d, "key2", d)
        key2.load()
        try:
            suspendLog()
            conf = mixminion.server.ServerConfig.ServerConfig(
                string=(SERVER_CONFIG_SHORT%mix_mktemp())+
                                           """[Incoming/MMTP]
Enabled: yes
Hostname: Theserver3
IP: 192.168.100.3
[Delivery/SMTP]
Enabled: yes
ReturnAddress: X@Y.Z
""")
            conf2 =  mixminion.server.ServerConfig.ServerConfig(
                string=(SERVER_CONFIG_SHORT%mix_mktemp())+
                                           """[Incoming/MMTP]
Enabled: yes
Hostname: Theserver4
IP: 192.168.100.4
""")
        finally:
            resumeLog()

        try:
            suspendLog()
            overrideDNS({"Theserver3" : "1.2.3.4"})
            _ = generateServerDescriptorAndKeys(conf,
                                                identity,
                                                d,
                                                "key2",
                                                d,
                                                useServerKeys=1)
        finally:
            s = resumeLog()
            undoReplacedAttributes()

        self.assertEndsWith(s, "Configured hostname 'Theserver3' resolves to"
                      " 1.2.3.4, but we're publishing the IP 192.168.100.3\n")

        key3 = mixminion.server.ServerKeys.ServerKeyset(d, "key2", d)
        key3.load()
        info3 = key3.getServerDescriptor()
        self.assertEquals(key3.getPacketKey().get_public_key(),
                          key2.getPacketKey().get_public_key())
        eq(info3['Incoming/MMTP']['Hostname'], "Theserver3")
        self.assert_('smtp' in info3.getCaps())

        # Check routing info
        self.assert_(info.canRelayTo(info))
        self.assert_(info.getRoutingFor(info), info.getRoutingInfo())
        self.assert_(info.canRelayTo(info3))
        self.assert_(not info3.canRelayTo(info)) # info3 has no outgoing/mmtp
        self.assertEquals(info.getRoutingFor(info3)[1],
                          info3.getRoutingInfo().pack())

        try:
            overrideDNS({"Theserver4": "192.168.100.4"})
            key3.regenerateServerDescriptor(conf2, identity)
        finally:
            undoReplacedAttributes()
        info3 = key3.getServerDescriptor()
        eq(info3['Incoming/MMTP']['Hostname'], "Theserver4")


#----------------------------------------------------------------------
# Directories

class DirectoryServerTests(TestCase):
    def x_testOldDirectory(self):
        #XXXX008 split this into serverlist and serverdirectory tests.
        eq = self.assertEquals
        examples = getExampleServerDescriptors()
        ServerList = mixminion.directory.ServerList.ServerList
        ServerDirectory = mixminion.ServerInfo.ServerDirectory
        baseDir = mix_mktemp()
        dirArchiveDir = os.path.join(baseDir, "dirArchive")
        cfg = {"Directory":{"ClientVersions":["foo"],
                            "ServerVersions":["foo","bar"],
                            } }
        lst = ServerList(baseDir, cfg)

        identity = Crypto.pk_generate(2048)

        now = time.time()
        dayLater = now + 60*60*24
        # Try a couple of simple inserts
        lst.importServerInfo(examples["Fred"][1]) # from day -9 through day 0.
        lst.importServerInfo(examples["Fred"][3]) # from day 11 through day 20
        lst.importServerInfo(examples["Lola"][0]) # from day -2 through day 2
        lst.importServerInfo(examples["Lola"][1]) # From day 0 through day 4.
        # Now, check whether the guts of lst are correct.
        eq(len(lst.servers), 4)
        eq(len(lst.serversByNickname), 2)
        eq(len(lst.serversByNickname['fred']), 2)
        eq(len(lst.serversByNickname['lola']), 2)
        eq(readFile(os.path.join(baseDir, "servers",
                                 lst.serversByNickname['fred'][0])),
           examples["Fred"][1])
        # Now generate a directory...
        lst.generateDirectory(now, dayLater, 0,
                              identity, now)
        # (Fred1, and Lola1 should get included.)
        d = readFile(lst.getDirectoryFilename())
        self.assertStartsWith(d, "[Directory]\n")
        eq(2, d.count("[Server]\n"))
        self.assert_(stringContains(d, examples["Fred"][1]))
        self.assert_(stringContains(d, examples["Lola"][1]))

        # Did a backup directory get made?
        eq(1, len(os.listdir(dirArchiveDir)))
        # Validate the directory, and check that values are as expected.
        sd = ServerDirectory(d)
        eq(len(sd.getServers()), 2)
        eq(sd["Directory"]["Version"], "0.2")
        eq(sd["Directory"]["Published"], int(now))
        eq(sd["Directory"]["Valid-After"], previousMidnight(now))
        eq(sd["Directory"]["Valid-Until"], previousMidnight(dayLater+1))
        eq(sd["Signature"]["DirectoryIdentity"].get_public_key(),
           identity.get_public_key())

        # Try changing the directory, and verify that it doesn't check out
        dBad = d.replace("Fred", "Dref")
        self.failUnlessRaises(ConfigError, ServerDirectory, dBad)
        # Bad digest.
        dBad = re.compile(r"^DirectoryDigest: ........", re.M).sub(
            "DirectoryDigest: ZZZZZZZZ", d)
        self.failUnlessRaises(ConfigError, ServerDirectory, dBad)
        # Bad signature.
        dBad = re.compile(r"^DirectorySignature: ........", re.M).sub(
            "DirectorySignature: ZZZZZZZZ", d)
        self.failUnlessRaises(ConfigError, ServerDirectory, dBad)

        # Can we use messed-up spaces and line-endings?
        ServerDirectory(d.replace("\n", "\r\n"))
        ServerDirectory(d.replace("\n", "\r"))
        ServerDirectory(re.sub(r"Fred$", "Fred  ", d))

        ### Now, try rescanning the directory.
        lst = ServerList(baseDir, cfg)
        eq(len(lst.servers), 4)
        eq(len(lst.serversByNickname), 2)
        eq(len(lst.serversByNickname['fred']), 2)
        eq(len(lst.serversByNickname['lola']), 2)
        lst.generateDirectory(now, dayLater, 0,
                              identity)
        d2 = readFile(lst.getDirectoryFilename())
        sd2 = ServerDirectory(d2)
        self.assertEquals(2, len(sd2.getServers()))

        # Now try cleaning servers.   First, make sure we can't insert
        # an expired server.
        self.failUnlessRaises(MixError,
                              lst.importServerInfo, examples["Fred"][0])
        # Now, make sure we can't insert a superseded server.
        lst.importServerInfo(examples["Bob"][3])
        lst.importServerInfo(examples["Bob"][4])
        self.failUnlessRaises(MixError,
                              lst.importServerInfo, examples["Bob"][1])
        # Now, start with a fresh list, so we can try superseding bob later.
        baseDir = mix_mktemp()
        archiveDir = os.path.join(baseDir, "archive")
        serverDir = os.path.join(baseDir, "servers")
        lst = ServerList(baseDir, cfg)
        # Make sure that when we remove the last of a given server, we
        # still know its ID.
        lst.importServerInfo(examples["Lisa"][1]) # Valid for 2 days.
        lst.clean(now=now+60*60*24*100) # Very far in the future
        eq(0, len(lst.servers))
        eq(1, len(os.listdir(archiveDir)))
        lst.rescan()
        eq(0, len(lst.servers))
        self.assertNotEquals(None, lst.serverIDs.get('lisa'))
        # But we _do_ remove expired servers if others exist.
        lst.importServerInfo(examples["Lisa"][2]) # Valid from 5...7.
        eq(1, len(lst.servers))
        eq(1, len(lst.serversByNickname["lisa"]))
        lst.clean(now=now+60*60*24*100) # Very far in the future.
        eq(0, len(lst.servers))
        eq(0, len(lst.serversByNickname.get("lisa",[])))

        # (Make sure that knownOnly works: failing case.)
        self.failUnlessRaises(MixError, lst.importServerInfo,
                              examples["Bob"][0], 1)

        ### Now test the removal of superseded servers.
        # Clean out archiveDir first so we can see what gets removed.
        for a in os.listdir(archiveDir):
            os.unlink(os.path.join(archiveDir, a))

        # Add a bunch of unconflicting Bobs.
        lst.importServerInfo(examples["Bob"][0]) # From -2 to 1
        # (Make sure that knownOnly works: succeeding case.
        lst.importServerInfo(examples["Bob"][1], 1) # From  2 to 5
        lst.importServerInfo(examples["Bob"][2]) # From  6 to 9
        lst.importServerInfo(examples["Bob"][3]) # Newer, from 0 to 3
        eq(4, len(lst.servers))
        # Right now, nothing is superseded or expired
        lst.clean()
        eq(4, len(os.listdir(serverDir)))
        eq(4, len(lst.serversByNickname["bob"]))
        lst.importServerInfo(examples["Bob"][4]) # Newer, from 4 to 7.
        # Now "Bob1" is superseded.
        lst.clean()
        eq(1, len(os.listdir(archiveDir)))
        eq(4, len(lst.serversByNickname["bob"]))
        eq(4, len(os.listdir(serverDir)))
        eq(4, len(lst.servers))
        eq(readFile(os.path.join(archiveDir, os.listdir(archiveDir)[0])),
           examples["Bob"][1])
        for fn in lst.serversByNickname["bob"]:
            fn = os.path.join(serverDir, fn)
            self.assertNotEquals(readFile(fn), examples["Bob"][1])
        # Now try rescanning...
        lst = ServerList(baseDir, cfg)
        eq(4, len(lst.servers))
        eq(4, len(lst.serversByNickname["bob"]))
        # ... adding a new bob...
        lst.importServerInfo(examples["Bob"][5])
        eq(5, len(lst.servers))
        # ... and watching another old bob get bonked off.
        lst.clean()
        eq(4, len(lst.servers))
        eq(2, len(os.listdir(archiveDir)))

    def testNewDirectoryFormats(self):
        DF = mixminion.directory.DirFormats
        SI = mixminion.ServerInfo
        examples = getExampleServerDescriptors()
        id0 = getRSAKey(1,2048)
        id1 = getRSAKey(3,2048)
        id2 = getRSAKey(5,2048)
        keyid0 = mixminion.Crypto.pk_fingerprint(id0)
        keyid1 = mixminion.Crypto.pk_fingerprint(id1)
        keyid2 = mixminion.Crypto.pk_fingerprint(id2)
        ub0 = "http://foo/"
        ub1 = "http://foo.bar/"
        ub2 = "http://foo.bar/baz/"
        voters = [(keyid0,ub0), (keyid1,ub1), (keyid2,ub2)]

        # Test generating a vote directory.
        va = previousMidnight(time.time())
        s1 = [ examples['Fred'][1], examples['Fred'][2], examples['Lola'][1],
               examples['Joe'][0], examples['Alice'][0] ]
        vd1 = {}
        s_vote1 = DF.generateVoteDirectory(
            id0, s1, [ "Fred", "Lola", "joe" ],
            voters, va, ["0.0.8", "0.0.8.1"], ["0.0.8.1"],
            validatedDigests=vd1)
        self.assert_(s_vote1.startswith('[Signed-Directory]\n'))

        # Test parsing it.
        vote1 = SI.SignedDirectory(string=s_vote1)
        DF.checkVoteDirectory(voters, va, vote1)
        self.assertEquals(vote1['Directory-Info']['Valid-After'],
                          previousMidnight(va))
        self.assertEquals(vote1['Directory-Info']['Valid-Until'],
                          succeedingMidnight(va))
        self.assertEquals(vote1['Directory-Info']['Status'], "vote")
        self.assertEquals(vote1.getSigners(), [(keyid0, ub0)])
        self.assertEquals(vote1['Directory-Info']['Recommended-Servers'],
                          [ "fred", "joe", "lola" ])
        self.assertUnorderedEq(vote1['Directory-Info']['Voting-Server'],
                               [ "%s %s"%(k,u) for k,u in voters ])
        expected = [ SI.ServerInfo(string=s,assumeValid=1).getDigest()
                     for s in s1 ]
        got = [ s.getDigest() for s in vote1.getAllServers() ]
        self.assertUnorderedEq(got, expected)

        # Now make two more, and have them vote.
        s1 = [ examples['Fred'][1], examples['Fred'][2], examples['Lola'][1],
               examples['Joe'][0], examples['Alice'][0] ]
        s2 = [ examples['Fred'][2], examples['Lola'][1],
               examples['Alice'][0], examples['Alice'][1],
               examples['Bob'][1], examples['Bob'][2],
               examples['Lisa'][1] ]
        vd2 = {}
        s_vote2 = DF.generateVoteDirectory(
            id1, s2, [ "Fred", "Lola", "Alice", "Bob", "Lisa"],
            voters, va, [ "0.0.8", "0.0.8.1", "0.0.9.1" ],["0.0.8.1", "0.0.8"],
            validatedDigests=vd2)
        s3 = [ examples['Alice'][1], examples['Bob'][2], examples['Fred'][2] ]
        s_vote3 = DF.generateVoteDirectory(
            id2, s2, [ "Fred", "Lola", "Alice", "Bob", "Lisa"],
            voters, va, [ "0.0.8", "0.0.8.1", "0.0.9.1" ],["0.0.8.1", "0.0.8"],
            validatedDigests=vd2)

        s_voted1 = DF.generateConsensusDirectory(
            id0, voters, va,
            [ ("voter1",s_vote1), ("voter2",s_vote2), ("voter3",s_vote3) ],
            vd1)

    def testVoteFile(self):
        VF = mixminion.directory.Directory.VoteFile
        d = mix_mktemp()
        fn = os.path.join(d, "votes")
        createPrivateDir(d)
        key0 = getRSAKey(0,2048)
        key1 = getRSAKey(1,2048)
        key2 = getRSAKey(2,2048)
        key3 = getRSAKey(3,2048)
        key4 = getRSAKey(4,2048)
        key5 = getRSAKey(5,2048)

        fp0 = mixminion.Crypto.pk_fingerprint(key0)
        fp1 = mixminion.Crypto.pk_fingerprint(key1)
        fp2 = mixminion.Crypto.pk_fingerprint(key2)
        fp3 = mixminion.Crypto.pk_fingerprint(key3)
        fp4 = mixminion.Crypto.pk_fingerprint(key4)
        fp5 = mixminion.Crypto.pk_fingerprint(key5)

        writeFile(fn,
                  ("yes fred %(fp0)s\n"
                   "#foo \n"
                   "no carol %(fp1)s\n"
                   "ignore alice %(fp2)s\n"
                   "abstain joe %(fp3)s\n"
                   "# yes fred %(fp2)s\n"
                   "#no ernesto %(fp5)s\n"
                   "#lemon curry\n")%locals())

        vf = VF(fn)
        self.assertEquals("yes", vf.getStatus(fp0, "fred"))
        self.assertEquals("mismatch", vf.getStatus(fp0, "federico"))
        self.assertEquals("no", vf.getStatus(fp1, "carol"))
        self.assertEquals("no", vf.getStatus(fp1, "carolxx"))
        self.assertEquals("ignore", vf.getStatus(fp2, ""))
        self.assertEquals("abstain", vf.getStatus(fp3, ""))
        self.assertEquals("unknown", vf.getStatus(fp4, ""))
        self.assertEquals("unknown", vf.getStatus(fp5, ""))
        self.assert_("ernesto" in vf.haveComment[fp5])
        now = mixminion.Config._parseTime("2005-06-03 03:26:20")
        vf.appendUnknownServers([("Terrence", fp5),
                                 ("carla", fp5),
                                 ("ernesto", fp5),
                                 ], now=now)
        vf.save()
        self.assertEndsWith(readFile(fn),
                            ("#lemon curry\n"
                             "#   Added 2005-06-03 03:26:20 [GMT]:\n"
                             "#abstain Terrence %(fp5)s\n"
                             "#abstain carla %(fp5)s\n")%locals())

#----------------------------------------------------------------------
# EventStats
class EventStatsTests(TestCase):
    def testNilLog(self):
        import mixminion.server.EventStats as ES
        ES.configureLog({'Server': {'LogStats' : 0}})
        self.failUnless(isinstance(ES.log, ES.NilEventLog))
        ES.log.receivedPacket()
        ES.log.attemptedRelay()
        ES.log.failedRelay()
        ES.log.unretriableRelay()

    def testEventLog(self):
        import mixminion.server.EventStats as ES
        tm = time.time()
        eq = self.assertEquals
        homedir = mix_mktemp()
        class FakeESConfig:
            def __getitem__(self,k):
                items = {'Server':
                {'LogStats' : 1,
                 'Homedir' : self.homedir,
                 'StatsInterval' : mixminion.Config._parseInterval("1 hour")}}
                return items[k]
            def getStatsFile(self):
                return os.path.join(self.homedir, "stats")
            def getWorkDir(self):
                return os.path.join(self.homedir, "work")
        cfg = FakeESConfig()
        cfg.homedir = homedir
##         ES.configureLog({'Server':
##                          {'LogStats' : 1,
##                           'Homedir' : homedir,
##                  'StatsInterval' : mixminion.Config._parseInterval("1 hour")}})
        ES.configureLog(cfg)
        self.failUnless(isinstance(ES.log, ES.EventLog))
        # Test all commands
        ES.log.receivedPacket()
        ES.log.attemptedRelay()
        ES.log.failedRelay()
        ES.log.unretriableRelay()
        ES.log.attemptedDelivery()
        ES.log.attemptedDelivery()
        ES.log.attemptedDelivery("Y")
        ES.log.attemptedDelivery("Y")
        ES.log.successfulDelivery()
        ES.log.failedDelivery()
        ES.log.failedDelivery("Y")
        ES.log.unretriableDelivery("Y")
        ES.log.save(now=tm)
        eq(ES.log.count['UnretriableDelivery']['Y'], 1)
        eq(ES.log.count['AttemptedDelivery'][None], 2)
        # Test reload.
        ES.configureLog(cfg)
##         ES.configureLog({'Server':
##                          {'LogStats' : 1,
##                           'Homedir' : homedir,
##                  'StatsInterval' : mixminion.Config._parseInterval("1 hour")}})
        eq(ES.log.count['AttemptedDelivery']['Y'], 2)
        # Test dump
        buf = cStringIO.StringIO()
        ES.log.dump(buf)
        sOrig = buf.getvalue()
        s = "\n".join(sOrig.split("\n")[1:])
        expected = """\
  ReceivedPacket: 1
  ReceivedConnection: 0
  AttemptedConnect: 0
  SuccessfulConnect: 0
  FailedConnect: 0
  AttemptedRelay: 1
  SuccessfulRelay: 0
  FailedRelay: 1
  UnretriableRelay: 1
  AttemptedDelivery:
     {Unknown}: 2
             Y: 2
         Total: 4
  SuccessfulDelivery: 1
  FailedDelivery:
     {Unknown}: 1
             Y: 1
         Total: 2
  UnretriableDelivery:
             Y: 1
         Total: 1
"""
        eq(s, expected)
        # Test time accumulation.
        ES.log.save(now=tm+1800)
        self.assert_(abs(1800-ES.log.accumulatedTime) < 10)
        self.assertEquals(ES.log.lastSave, tm+1800)
        ES.log.lastSave = tm+1900
        ES.log.save(now=tm+2000)
        self.assert_(abs(1900-ES.log.accumulatedTime) < 10)
        # Test rotate
        ES.log.rotate(now=tm+3600*24)
        s2 = readFile(os.path.join(homedir, "stats"))
        eq([ l for l in s2.split("\n") if l == '' or l[0] not in "#="],
           sOrig.split("\n")[1:])
        eq(ES.log.count['UnretriableDelivery'], {})
        eq(ES.log.lastSave, tm+3600*24)
        eq(ES.log.accumulatedTime, 0)
        self.assert_((ES.log.nextRotation - (tm+3600*25)) < 3600)
        ES.configureLog(cfg)
##         ES.configureLog({'Server':
##                          {'LogStats' : 1,
##                           'Homedir' : homedir,
##                  'StatsInterval' : mixminion.Config._parseInterval("1 hour")}})
        eq(ES.log.count['UnretriableDelivery'], {})
        self.assertFloatEq(ES.log.lastSave, time.time())

        # Test time configured properly.
        # 1) Rotation interval is a multiple of hours.
        pm = previousMidnight(tm)
        ES.log.rotateInterval = 2*60*60
        # 1A) Accumulated time << interval.  We rotated at 23:00.  We were
        #   only up for a few minutes (10), and have accumulated no more time
        #   since the last rotation.  We just reloaded; now it's 2:00.  We
        #   should next rotate at 4 -- since it will take us until 3:20 to
        #   be up for 75% of an interval, and we round up.
        ES.log.lastRotation = pm - 60*60
        ES.log.accumulatedTime = 10*60
        ES.log.lastSave = pm+7200
        ES.log._setNextRotation(now=pm+7200)
        eq(ES.log.getNextRotation(), pm+4*60*60)
        # But suppose that it's 2:45. We should next rotate at 5 (rounding up).
        ES.log.lastSave = pm+7200+45*60
        ES.log._setNextRotation(now=pm+7200+45*60)
        eq(ES.log.getNextRotation(), pm+5*60*60)

        # 1B) Accumulated time ~~ interval
        # Now suppose the same situation as above, but we've been up 100
        #  minutes total.  We rotate *now*.
        ES.log.accumulatedTime = 100*60
        ES.log.lastSave = pm+7200
        ES.log._setNextRotation(now=pm+7200)
        eq(ES.log.getNextRotation(), pm+7200)

        # 1C) Suppose all went well, and it's 23:30: Rotate at 1.
        ES.log.accumulatedTime = 1800
        ES.log.lastSave = pm-1800
        ES.log._setNextRotation(now=pm-1800)
        eq(ES.log.getNextRotation(), pm+3600)

        # 2) Rotation interval is not a multiple of hours: We don't round
        # 2A) Accumulated time << interval
        ES.log.rotateInterval = 40*60
        ES.log.lastSave = pm+7200
        ES.log.accumulatedTime = 10*60
        ES.log._setNextRotation(now=pm+7200)
        eq(ES.log.getNextRotation(), pm+7200+20*60)
        # 2B) Accumulated time ~~ interval
        ES.log.accumulatedTime = 35*60
        ES.log.lastSave = pm+7200
        ES.log._setNextRotation(now=pm+7200)
        eq(ES.log.getNextRotation(), pm+7200)

#----------------------------------------------------------------------
# Modules and ModuleManager

# Text of an example module that we load dynamically.
EXAMPLE_MODULE_TEXT = \
"""
import mixminion.server.Modules
from mixminion.Config import ConfigError

class TestModule(mixminion.server.Modules.DeliveryModule):
    def __init__(self):
        self.processedMessages = []
        self.processedAll = []
    def getName(self):
        return "TestModule"
    def getConfigSyntax(self):
        return { "Example" : { "Foo" : ("REQUIRE", "int", None) } }
    def validateConfig(self, cfg, lines, contents):
        if cfg['Example'] is not None:
            if cfg['Example'].get('Foo',1) % 2 == 0:
                raise ConfigError("Foo was even")
    def getRetrySchedule(self):
        return [.1] *100
    def configure(self,cfg, manager):
        if cfg['Example']:
            self.enabled = 1
            self.foo = cfg['Example'].get('Foo',1)
            manager.enableModule(self)
        else:
            self.foo = None
            self.enabled = 0
    def getServerInfoBlock(self):
        if self.enabled:
            return "[Example]\\nFoo: %s\\n" % self.foo
        else:
            return None
    def getExitTypes(self):
        return (1234,)
    def processMessage(self, packet):
        self.processedMessages.append(packet.getContents())
        self.processedAll.append( packet )
        exitInfo = packet.getAddress()
        if exitInfo == 'fail?':
            return mixminion.server.Modules.DELIVER_FAIL_RETRY
        elif exitInfo == 'fail!':
            return mixminion.server.Modules.DELIVER_FAIL_NORETRY
        else:
            return mixminion.server.Modules.DELIVER_OK
"""

class ModuleManagerTests(TestCase):
    def testModuleManager(self):
        FDP = FakeDeliveryPacket
        mod_dir = mix_mktemp()
        home_dir = mix_mktemp()

        # Create an example module, and try to load it.
        os.mkdir(mod_dir, 0700)
        writeFile(os.path.join(mod_dir, "ExampleMod.py"),
                  EXAMPLE_MODULE_TEXT)

        cfg_test = (SERVER_CONFIG_SHORT%home_dir) + """
ModulePath = %s
Module ExampleMod.TestModule
[Example]
Foo: 99
[Incoming/MMTP]
Enabled: yes
Hostname: Theserver5
IP: 1.0.0.1
""" % (mod_dir)

        try:
            suspendLog()
            conf = mixminion.server.ServerConfig.ServerConfig(string=cfg_test)
        finally:
            resumeLog()
        manager = conf.getModuleManager()
        exampleMod = None
        for m in manager.modules:
            if m.getName() == "TestModule":
                exampleMod = m
        self.failUnless(exampleMod is not None)
        # Configure the new module, and make sure it recognizes its own config.
        manager.configure(conf)

        # Make sure the module enables itself.
        self.failUnless(exampleMod is manager.typeToModule[1234])

        # Try sending a few messages to the module.
        t = "ZZZZ"*5
        manager.queueDecodedMessage(FDP('plain',1234,'fail!',"Hello 1", t))
        manager.queueDecodedMessage(FDP('plain',1234,'fail?',"Hello 2", t))
        manager.queueDecodedMessage(FDP('plain',1234,'good',"Hello 3", t))
        manager.queueDecodedMessage(FDP('plain',
                                        mixminion.Packet.DROP_TYPE, "",
                                        "Drop very much", t))
        queue = manager.queues['TestModule']
        # Did the test module's delivery queue get the messages?
        self.failUnless(isinstance(queue,
                           mixminion.server.Modules.SimpleModuleDeliveryQueue))
        self.assertEquals(3, queue.count())
        # Has it processed any yet? (No.)
        self.assertEquals(exampleMod.processedMessages, [])
        # Now, tell the module to deliver the messages.
        try:
            suspendLog()
            manager.sendReadyMessages()
        finally:
            resumeLog()
        # There should be one message (the retriable one) left in the queue.
        self.assertEquals(1, queue.count())
        # It should have processed all three.
        self.assertEquals(3, len(exampleMod.processedMessages))
        # If we try to send again, the second message should get re-sent.
        time.sleep(.15) # so we retry.
        manager.sendReadyMessages()
        self.assertEquals(1, queue.count())
        self.assertEquals(4, len(exampleMod.processedMessages))
        self.assertEquals("Hello 2", exampleMod.processedMessages[-1])

        self.assert_(exampleMod.processedAll[0].isPlaintext())

        # Check serverinfo generation.
        try:
            suspendLog()
            overrideDNS({"TheServer5" : '1.0.0.1'})
            info = generateServerDescriptorAndKeys(
                conf, getRSAKey(0,2048), home_dir, "key11", home_dir)
            self.failUnless(stringContains(info,"\n[Example]\nFoo: 99\n"))
        finally:
            resumeLog()
            undoReplacedAttributes()
        manager.close()

        # Try again, this time with the test module disabled.
        #
        cfg_test = (SERVER_CONFIG_SHORT%home_dir) + """
ModulePath = %s
Module ExampleMod.TestModule
""" % (mod_dir)

        try:
            suspendLog()
            conf = mixminion.server.ServerConfig.ServerConfig(string=cfg_test)
        finally:
            resumeLog()
        manager = conf.getModuleManager()
        exampleMod = None
        for m in manager.modules:
            if m.getName() == "TestModule":
                exampleMod = m
        self.failUnless(exampleMod is not None)
        manager.configure(conf)

        self.failIf(exampleMod is manager.typeToModule.get(1234))

        # Failing validation
        cfg_test = SERVER_CONFIG_SHORT%home_dir + """
ModulePath = %s
Module ExampleMod.TestModule
[Example]
Foo: 100
""" % (mod_dir)

        # FFFF Add tests for catching exceptions from buggy modules
        manager.close()

    def testDecoding(self):
        'test decoding and test encapsulation.'
        eme = mixminion.server.Modules._escapeMessageForEmail

        message = "Somebody set up us the module!\n\n(What you say?)\n"
        binmessage = hexread("00ADD1EDC0FFEED00DAD")*40
        tag = ".!..!....!........!."

        def FDPFast(type,message,tag="xyzzyxyzzyxyzzyxyzzy"):
            return FakeDeliveryPacket(type,0xFFFE,"addr",message,tag)

        ####
        # Tests escapeMessageForEmail
        self.assert_(stringContains(eme(FDPFast('plain',message)), message))
        expect = "-----\nMessage-type: binary\n\n"+\
                 encodeBase64(binmessage)+"-----"
        self.assert_(stringContains(eme(FDPFast('plain',binmessage)), expect))
        expect = "-----\nMessage-type: encrypted\nDecoding-handle: "+\
                 base64.encodestring(tag)+"\n"+\
                 encodeBase64(binmessage)+"-----"
        self.assert_(stringContains(eme(FDPFast('enc',binmessage,tag)),
                                    expect))

# Sample address file for testing MBOX
MBOX_ADDRESS_SAMPLE = """\
# This is a sample address file
mix-minion: mixminion@thishost

mixdaddy: mixminion@thathost
mixdiddy=mixminion@theotherhost
"""

# The message we expect MBOX to deliver.
MBOX_EXPECTED_MESSAGE = """\
To: mixminion@theotherhost
From: returnaddress@x
Subject: Type III Anonymous Message
X-Anonymous: yes

This message is not in plaintext.  It's either 1) a reply; 2) a forward
message encrypted to you; or 3) junk.

-----BEGIN TYPE III ANONYMOUS MESSAGE-----
Message-type: encrypted
Decoding-handle: eHh4eHh4eHh4eHh4eHh4eHh4eHg=

7/rOqx76yt7v+s6rHvrK3u/6zqse+sre7/rOqx76yt7v+s6rHvrK3u/6zqse+sre
7/rOqx76yt7v+s6rHvrK3u/6zqse+sre7/rOqx76yt7v+s6rHvrK3u/6zqse+sre
7/rOqx76yt7v+s6rHvrK3u/6zqse+sre7/rOqx76yt7v+s6rHvrK3u/6zqse+sre
7/rOqx76yt7v+s6rHvrK3g==
-----END TYPE III ANONYMOUS MESSAGE-----
"""

EXAMPLE_ADDRESS_SET = """
deny User freD
Deny User  mr-ed
# Comment
Deny onehost sally
DENY onehost  HOGWARTS.k12
# Another comment, followed by a blank line
dENY allhosts  deathstar.gov
deny allhosts selva-Oscura.it
deny onehost selva-Oscura.it

deny Address  jim@sMith.com
deny pattern    /nyet.*Nyet/

"""

class FakeDeliveryPacket(mixminion.server.PacketHandler.DeliveryPacket):
    """Stub version of DeliveryPacket used for testing modules"""
    def __init__(self, type, exitType, exitAddress, contents, tag="",
                 headers = {}):
        if tag == "" and exitType not in (DROP_TYPE, FRAGMENT_TYPE):
            tag = "-="*10
        mixminion.server.PacketHandler.DeliveryPacket.__init__(self,
                        exitType, tag+exitAddress, "Z"*16, "Q"*(28*1024))
        self.type = type
        self.headers = headers
        self.payload = None
        self.contents = contents
        if tag:
            self.setTagged(1)

class ModuleTests(TestCase):
    def testEmailAddressSet(self):
        EmailAddressSet = mixminion.server.Modules.EmailAddressSet
        def has(set, item, self=self):
            self.assert_(isSMTPMailbox(item), "Invalid address "+item)
            self.failUnless(set.contains(item), "Set should contain "+item)
        def hasNo(set, item, self=self):
            self.assert_(isSMTPMailbox(item), "Invalid address "+item)
            self.failIf(set.contains(item), "Set should not contain "+item)

        # Basic functionality: Match what we're supposed to match
        set = EmailAddressSet(string=EXAMPLE_ADDRESS_SET)
        for _ in 1,2:
            has(set,"jim@smith.com")
            has(set,"freD@fred.com") #(by user....)
            has(set,"fred@x")
            has(set,"fred@boba-fred")
            has(set,"MR-ED@wilburs-barn.com")
            has(set,"Fred@Fred")
            has(set,"Fred@Sally") #(by domains and subdomains...)
            has(set,"joe@SALLY")
            has(set,"h.potter@hogwarts.K12")
            has(set,"nobody@sally")
            has(set,"dante@selva-oscura.it")
            has(set,"dante@camin.selva-oscura.it")
            has(set,"dante@nel.camin.selva-oscura.it")
            has(set,"cushing@deathstar.gov")
            has(set,"cushing@operational.deathstar.gov")
            has(set,"cushing@fully.operational.deathstar.gov")
            has(set,"nyet.jones@nyet.net")
            has(set,"octavio.nyet.jones@nyet.net")
            has(set,"octavio.jones@nyet.nyet.net")

            # Basic functionality: Don't match anything else.
            hasNo(set,"mojo@jojo.com")
            hasNo(set,"mr-fred@wilburs-barn.com") # almost by user
            hasNo(set,"joe@sally.com") # almost by domain...
            hasNo(set,"joe@bob.sally.com")
            hasNo(set,"joe@bob.sally")
            hasNo(set,"dante@other.it")
            hasNo(set,"cushing@gov")
            hasNo(set,"cushing@deathstar.gov.mit.edu")
            hasNo(set,"nyet.jones@net")
            hasNo(set,"jones@nyet.net")

            # Load from file, then try again!
            fn = mix_mktemp()
            writeFile(fn, EXAMPLE_ADDRESS_SET)
            set = EmailAddressSet(fname=fn)

        # Failing cases: invalid addresses needn't give a right answer, but
        # we need to do something reasonable for invalid files.
        def bad(s,self=self):
            self.assertRaises(ConfigError,
                              mixminion.server.Modules.EmailAddressSet,
                              string=s)

        bad("Address foo@bar.baz")
        bad("deny Address foo@bar@baz")
        bad("deny Rumplestiltskin")
        bad("deny bob@bob.com")
        bad("deny user fred@bob")
        bad("deny address foo")
        bad("deny onehost foo@")
        bad("deny onehost @foo")
        bad("deny allhosts foo@bar")
        bad("deny pattern a.*b")
        bad("deny pattern /a.*b")
        bad("deny pattern a.*b/")
        bad("deny onehost")
        bad("deny user")
        bad("deny pattern")
        bad("deny allhosts")

    def testMixmasterSMTP(self):
        """Check out the SMTP-Via-Mixmaster module.  (We temporarily replace
           os.spawnl with a stub function so that we don't actually send
           anything."""
        manager = self.getManager()
        FDP = FakeDeliveryPacket

        # Configure the module.
        module = mixminion.server.Modules.MixmasterSMTPModule()
        module.configure({"Delivery/SMTP-Via-Mixmaster" :
                          {"Enabled":1, "Server": "nonesuch",
                           "Retry": [0,0,0,0],
                           "SubjectLine":'foobar',
                           "FromTag" : '[NotReally]',
                           'MixCommand' : ('ls', ['-z']),
                           "MaximumSize" : 32*1024,
                           "AllowFromAddress" : 1,
                           }},
                         manager)
        queue = manager.queues['SMTP_MIX2']
        replaceFunction(os, "spawnl")
        try:
            # Send a message...
            queue.queueDeliveryMessage(FDP('plain', SMTP_TYPE, "foo@bar",
                                           "This is the message"))
            queue.sendReadyMessages()
            # And make sure that Mixmaster was invoked correctly.
            calls = getReplacedFunctionCallLog()
            self.assertEquals('spawnl', calls[0][0])
            mixfn, mixargs = calls[0][1][2], calls[0][1][3:]
            self.assertEquals("ls", mixfn)
            self.assertEquals(mixargs[:-1],
                              ('-z', '-l', 'nonesuch'))
            # ...and, if the temporary file it used hasn't been removed yet,
            # that it contains the correct data.
            fn = mixargs[-1]
            fn = os.path.join(os.path.split(fn)[0],
                              "rmv_"+os.path.split(fn)[1][4:])
            m = readFile(fn)
            self.assertStartsWith(m,
                "To: foo@bar\nFrom: nobody\n"
                "Subject: foobar\nX-Anonymous: yes\n\n")
            self.assert_(stringContains(m, "This is the message"))

            ## What about the flush command?
            self.assertEquals("spawnl", calls[-1][0])
            sendfn, sendargs = calls[-1][1][2], calls[-1][1][3:]
            self.assertEquals("ls", sendfn)
            self.assertEquals(sendargs, ('-S',))

            clearReplacedFunctionCallLog()

            # Send another message, with headers.
            queue.queueDeliveryMessage(
                FDP('plain', SMTP_TYPE, "foo@bar",
                    "This is the message",
                    headers={"FROM":"Mal", "SUBJECT":"Fnord"}))
            queue.sendReadyMessages()
            calls = getReplacedFunctionCallLog()
            mixfn, mixargs = calls[0][1][2], calls[0][1][3:]
            fn = mixargs[-1]
            fn = os.path.join(os.path.split(fn)[0],
                              "rmv_"+os.path.split(fn)[1][4:])
            m = readFile(fn)
            self.assertEquals(m, '''\
To: foo@bar
From: "[NotReally] Mal" <nobody>
Subject: Fnord
X-Anonymous: yes

-----BEGIN TYPE III ANONYMOUS MESSAGE-----
Message-type: plaintext

This is the message
-----END TYPE III ANONYMOUS MESSAGE-----
''')
        finally:
            undoReplacedAttributes()
            clearReplacedFunctionCallLog()
        manager.close()

    def testDirectSMTP(self):
        """Check out the SMTP module.  (We temporarily replace sendSMTPMessage
           with a stub function so that we don't actually send anything.)"""
        FDP = FakeDeliveryPacket

        blacklistFile = mix_mktemp()
        writeFile(blacklistFile, "Deny onehost wangafu.net\nDeny user fred\n")

        manager = self.getManager("""[Delivery/SMTP]
Enabled: yes
SMTPServer: nowhere
BlacklistFile: %s
Message: Avast ye mateys!  Prepare to be anonymized!
ReturnAddress: yo.ho.ho@bottle.of.rum
SubjectLine: Arr! This be a Type III Anonymous Message
MaximumSize: 35K
        """ % blacklistFile)

        module = manager.nameToModule["SMTP"]
        queue = manager.queues["SMTP"]
        queueMessage = queue.queueDeliveryMessage

        # Make sure blacklist got read.
        self.assert_(module.blacklist.contains("nobody@wangafu.net"))

        # Stub out sendSMTPMessage.
        replaceFunction(mixminion.server.Modules, 'sendSMTPMessage',
                        lambda *args: mixminion.server.Modules.DELIVER_OK)

        try:
            haiku = ("Hidden, we are free\n"+
                     "Free to speak, to free ourselves\n"+
                     "Free to hide no more.")

            # Try queueing a valid message and sending it.
            queueMessage(FDP('plain', SMTP_TYPE, "users@everywhere", haiku))
            self.assertEquals(getReplacedFunctionCallLog(), [])
            queue.sendReadyMessages()
            # Was sendSMTPMessage invoked correctly?
            calls = getReplacedFunctionCallLog()
            self.assertEquals(1, len(calls))
            fn, args, _ = calls[0]
            self.assertEquals("sendSMTPMessage", fn)
            #server, toList, fromAddr, message
            self.assertEquals((['users@everywhere'],
                               'yo.ho.ho@bottle.of.rum'),
                              args[1:3])
            EXPECTED_SMTP_PACKET = """\
To: users@everywhere
From: yo.ho.ho@bottle.of.rum
Subject: Arr! This be a Type III Anonymous Message
X-Anonymous: yes

Avast ye mateys!  Prepare to be anonymized!

-----BEGIN TYPE III ANONYMOUS MESSAGE-----
Message-type: plaintext

Hidden, we are free
Free to speak, to free ourselves
Free to hide no more.
-----END TYPE III ANONYMOUS MESSAGE-----\n"""
            self.assertLongStringEq(EXPECTED_SMTP_PACKET, args[3])
            clearReplacedFunctionCallLog()

            # Now, with headers.
            queueMessage(FDP('plain', SMTP_TYPE, "users@everywhere", haiku,
                             headers={"FROM":"Captain Nick",
                                      "SUBJECT":"Gold Doubloons",
                                      "IN-REPLY-TO":"aaaaa@b.com",
                                      "REFERENCES":"cccccc@d.com"}))
            queue.sendReadyMessages()
            # Was sendSMTPMessage invoked correctly?
            calls = getReplacedFunctionCallLog()
            self.assertEquals(1, len(calls))
            fn, args, _ = calls[0]
            self.assertEquals("sendSMTPMessage", fn)
            #server, toList, fromAddr, message
            self.assertEquals((['users@everywhere'],
                               'yo.ho.ho@bottle.of.rum'),
                              args[1:3])
            EXPECTED_SMTP_PACKET = '''\
To: users@everywhere
From: "[Anon] Captain Nick" <yo.ho.ho@bottle.of.rum>
Subject: Gold Doubloons
In-Reply-To: aaaaa@b.com
References: cccccc@d.com
X-Anonymous: yes

Avast ye mateys!  Prepare to be anonymized!

-----BEGIN TYPE III ANONYMOUS MESSAGE-----
Message-type: plaintext

Hidden, we are free
Free to speak, to free ourselves
Free to hide no more.
-----END TYPE III ANONYMOUS MESSAGE-----\n'''
            self.assertLongStringEq(EXPECTED_SMTP_PACKET, args[3])
            clearReplacedFunctionCallLog()

            # Now, try a bunch of messages that won't be delivered: one with
            # an invalid address, and one with a blocked address.
            try:
                suspendLog()
                queueMessage(FDP('plain',SMTP_TYPE, "not.an.addr", haiku))
                queueMessage(FDP('plain',SMTP_TYPE,
                                 "blocked@wangafu.net",haiku))
                queue.sendReadyMessages()
            finally:
                s = resumeLog()
            self.assertEquals(1,s.count(
              "Dropping message to blacklisted address 'blocked@wangafu.net'"))
            self.assertEquals(1,s.count(
                "Dropping SMTP message to invalid address 'not.an.addr'"))
            self.assertEquals([], getReplacedFunctionCallLog())


            # Message over 35K should be blocked.
            m = Crypto.getCommonPRNG().getBytes(6*1024)*6
            self.assert_(len(BuildMessage.compressData(m))<28*1024)
            try:
                suspendLog()
                queueMessage(FDP('plain',SMTP_TYPE,"foo@bar.bax",m))
                queue.sendReadyMessages()
            finally:
                s = resumeLog()
            self.assertEquals(1, s.count("Dropping over-long message"))
            self.assertEquals([], getReplacedFunctionCallLog())
        finally:
            undoReplacedAttributes()
            clearReplacedFunctionCallLog()

    def testMBOX(self):
        """Check out the MBOX module. (We temporarily replace sendSMTPMessage
           with a stub function so that we don't actually send anything.)"""
        FDP = FakeDeliveryPacket
        # Configure the module
        manager = self.getManager()
        module = mixminion.server.Modules.MBoxModule()
        addrfile = mix_mktemp()
        writeFile(addrfile, MBOX_ADDRESS_SAMPLE)
        module.configure({'Server':{'Nickname': "nickname"},
                          'Incoming/MMTP':{},
                          "Delivery/MBOX" :
                          {"Enabled": 1,
                           "AddressFile": addrfile,
                           "ReturnAddress": "returnaddress@x",
                           "RemoveContact": "removeaddress@x",
                           "Retry": [0,0,0,3],
                           "SMTPServer" : "foo.bar.baz",
                           "MaximumSize" : 32*1024,
                           "AllowFromAddress": 1,
                           "SubjectLine" : "Type III Anonymous Message"
                           }}, manager)
        # Check that the address file was read correctly.
        self.assertEquals({'mix-minion': 'mixminion@thishost',
                           'mixdaddy':   'mixminion@thathost',
                           'mixdiddy':   'mixminion@theotherhost'},
                           module.addresses)
        queue = manager.queues['MBOX']
        # Stub out sendSMTPMessage.
        replaceFunction(mixminion.server.Modules, 'sendSMTPMessage',
                 lambda *args: mixminion.server.Modules.DELIVER_OK)
        self.assertEquals(queue.retrySchedule, [0,0,0,3])
        try:
            # Try queueing a message...
            queue.queueDeliveryMessage(FDP('enc', MBOX_TYPE, 'mixdiddy',
                                   hexread("EFFACEAB1EFACADE")*20, "x"*20))
            self.assertEquals(getReplacedFunctionCallLog(), [])
            # ...and sending it.
            queue.sendReadyMessages()
            try:
                # Also, try sending a message to an unknown address
                suspendLog()
                queue.queueDeliveryMessage(
                    FDP('env', MBOX_TYPE, 'mixmuffin',
                        hexread("EFFACEAB1EFACADE")*20,
                            'x'*20))
                queue.sendReadyMessages()
            finally:
                m = resumeLog()
            self.assert_(stringContains(m,"Unknown MBOX user 'mixmuffin'"))
            self.assert_(stringContains(m,"Unable to deliver message"))

            # Check that sendSMTPMessage was called correctly.
            self.assertEquals(1, len(getReplacedFunctionCallLog()))
            fn, args, _ = getReplacedFunctionCallLog()[0]
            self.assertEquals('sendSMTPMessage', fn)
            self.assertEquals((['mixminion@theotherhost'],
                               'returnaddress@x'),
                              args[1:3])
            self.assertLongStringEq(MBOX_EXPECTED_MESSAGE, args[3])
        finally:
            undoReplacedAttributes()
            clearReplacedFunctionCallLog()

    def testDirectoryDump(self):
        """Check out the DirectoryStoreModule that we use for testing on
           machines with unreliable/nonexistent SMTP."""
        FDP = FakeDeliveryPacket
        eme = mixminion.server.Modules._escapeMessageForEmail
        dir = mix_mktemp()
        manager = self.getManager()
        # Configure the module: disabled and enabled (queueless mode)
        module = mixminion.testSupport.DirectoryStoreModule()
        module.configure({'Testing/DirectoryDump' : {}}, manager)
        self.assert_(not manager.queues.has_key('Testing_DirectoryDump'))
        module.configure({'Testing/DirectoryDump' :
                          {'Location': dir, 'UseQueue' : 0,
                           'Retry': [60]}}, manager)
        # Try sending a couple of messages.
        queue = manager.queues['Testing_DirectoryDump']
        p1 = FDP('plain',0xFFFE, "addr1","this is the message","t"*20)
        queue.queueDeliveryMessage(p1)
        self.assert_(os.path.exists(os.path.join(dir, "0")))
        p2 = FDP('plain',0xFFFE, "addr2", "This is message 2", "x"*20)
        queue.queueDeliveryMessage(p2)
        self.assert_(os.path.exists(os.path.join(dir, "1")))
        self.assertEquals(eme(p2),
                          readFile(os.path.join(dir, "1")))
        # test failure.
        try:
            suspendLog()
            queue.queueDeliveryMessage(
               FDP('plain', 0xFFFE, "FAIL!",
                   "This is message X which won't be delivered", "x"*20))
            self.assert_(not os.path.exists(os.path.join(dir, "2")))
        finally:
            m = resumeLog()
        self.assertEndsWith(m, "Unable to deliver message\n")

        try:
            suspendLog()
            queue.queueDeliveryMessage(
                FDP('plain', 0xFFFE, "fail",
                    "This is message X which won't be delivered", "z"*20))
            self.assert_(not os.path.exists(os.path.join(dir, "2")))
        finally:
            m = resumeLog()
        self.assertEndsWith(m, "Unable to retry delivery for message\n")

        queue.sendReadyMessages()

        # Check sane behavior on missing files, and on restart.
        writeFile(os.path.join(dir, "90"), 'zed')
        module = mixminion.testSupport.DirectoryStoreModule()
        # This time, use a queue.
        module.configure({'Testing/DirectoryDump' :
                          {'Location': dir, 'UseQueue' : 1,
                           'Retry': [60]}}, manager)
        # Do we skip over the missing messages?
        self.assertEquals(module.next, 91)
        self.assertEquals(len(os.listdir(dir)), 3)
        queue = manager.queues['Testing_DirectoryDump']
        queue.queueDeliveryMessage(
                FDP('plain',0xFFFE, "addr91", "This is message 91"))
        queue.queueDeliveryMessage(
                FDP('plain',0xFFFE, "addr92", "This is message 92"))
        _ = queue.queueDeliveryMessage(
                FDP('plain',0xFFFE, "fail", "This is message 93"))
        queue.queueDeliveryMessage(
                FDP('plain',0xFFFE, "FAIL!", "This is message 94"))

        # All 4 messages go into the queue...
        self.assertEquals(4, queue.count())
        self.assertEquals(3, len(os.listdir(dir)))
        try:
            suspendLog()
            queue.sendReadyMessages()
        finally:
            m = resumeLog()
        self.assert_(stringContains(m,"[ERROR] Unable to deliver message"))
        # After delivery: 91 and 92 go through, 93 stays, and 94 gets dropped.
        self.assertEquals(1, queue.count())
        self.assertEquals(5, len(os.listdir(dir)))

    def testFragmented(self):

        manager = self.getManager("""[Delivery/SMTP]
Enabled: yes
SMTPServer: nowhere
Message: Avast ye mateys!  Prepare to be anonymized!
ReturnAddress: yo.ho.ho@bottle.of.rum
SubjectLine: Arr! This be a Type III Anonymous Message
MaximumSize: 1M
[Delivery/Fragmented]
Enabled: yes
MaximumSize: 1M
""")
        message = Crypto.getCommonPRNG().getBytes(200*1024)
        hm = mixminion.Packet.encodeMailHeaders(subject="Hello")
        payloads = BuildMessage.encodeMessage(hm+message, 0,
               mixminion.Packet.ServerSideFragmentedMessage(SMTP_TYPE,
                                  "pirates@sea","").pack())
        deliv = [ mixminion.server.PacketHandler.DeliveryPacket(
            routingType=FRAGMENT_TYPE, routingInfo="", applicationKey="X"*16,
            payload=p) for p in payloads ]
        self.assertEquals(len(deliv), 11)

        replaceFunction(mixminion.server.Modules, 'sendSMTPMessage',
                        lambda *args: mixminion.server.Modules.DELIVER_OK)
        try:
            # 6 packets; not enough to reconstruct.
            for p in deliv[:6]:
                manager.queueDecodedMessage(p)
            manager.sendReadyMessages()
            self.assertEquals(getReplacedFunctionCallLog(), [])
            manager.queueDecodedMessage(deliv[9])
            manager.queueDecodedMessage(deliv[10])
            manager.sendReadyMessages()
            # 8 packets; should be enough to reconstruct.
            calls = getReplacedFunctionCallLog()
            self.assertEquals(1, len(calls))
            fn, args, _ = calls[0]
            self.assertEquals(fn, "sendSMTPMessage")
            self.assertEquals((["pirates@sea"],
                               "yo.ho.ho@bottle.of.rum"), args[1:3])
            self.assertLongStringEq(args[3], """\
To: pirates@sea
From: yo.ho.ho@bottle.of.rum
Subject: Hello
X-Anonymous: yes

Avast ye mateys!  Prepare to be anonymized!

This message contains nonprinting characters, so I encoded it with Base64
before sending it to you.

-----BEGIN TYPE III ANONYMOUS MESSAGE-----
Message-type: binary

%s-----END TYPE III ANONYMOUS MESSAGE-----
""" % encodeBase64(message))

            pool = manager.queues["FRAGMENT"].pool
            self.assertEquals(pool.store.count(), 0)

            manager.queueDecodedMessage(deliv[3])
            self.assertEquals(pool.store.count(), 0)
        finally:
            undoReplacedAttributes()
            clearReplacedFunctionCallLog()
        manager.close()

    def getManager(self, extraConfig=None):
        d = mix_mktemp()
        c = SERVER_CONFIG_SHORT % d
        if extraConfig:
            c += extraConfig
        try:
            suspendLog()
            conf = mixminion.server.ServerConfig.ServerConfig(string=c)
            m = conf.getModuleManager()
            m.configure(conf)
            return m
        finally:
            resumeLog()

#----------------------------------------------------------------------

# Sample server configuration to test ServerKeys
SERVERCFG = """
[Server]
BaseDir: %(home)s
Mode: local
EncryptIdentityKey: No
PublicKeyLifetime: 10 days
IdentityKeyBits: 2048
EncryptPrivateKey: no
Nickname: mac-the-knife
Contact-Email: a@b.c
[Incoming/MMTP]
Enabled: yes
Hostname: Theserver5
IP: 10.0.0.1
"""

_FAKE_HOME = None
def _getServerKeyring():
    global _FAKE_HOME
    _FAKE_HOME = mix_mktemp()
    cfg = SERVERCFG % { 'home' : _FAKE_HOME }
    try:
        suspendLog()
        conf = mixminion.server.ServerConfig.ServerConfig(string=cfg)
    finally:
        resumeLog()
    return mixminion.server.ServerKeys.ServerKeyring(conf)

class ServerKeysTests(TestCase):
    def testServerKeyring(self):
        keyring = _getServerKeyring()
        home = _FAKE_HOME

        # Test creating identity key
        #identity = getRSAKey(0,2048)
        identity = keyring.getIdentityKey()
        fn = os.path.join(home, "keys", "identity.key")
        identity2 = Crypto.pk_PEM_load(fn)
        self.assertEquals(Crypto.pk_get_modulus(identity),
                          Crypto.pk_get_modulus(identity2))
        # (Make sure warning case can occur.)
        pk = getRSAKey(0,128)
        Crypto.pk_PEM_save(pk, fn)
        suspendLog()
        keyring.getIdentityKey()
        msg = resumeLog()
        self.failUnless(len(msg))
        Crypto.pk_PEM_save(identity, fn)
        if not ON_WINDOWS:
            self.assertEquals(os.stat(fn)[stat.ST_MODE] & 0777, 0600)

        # Now create a keyset
        now = time.time()
        try:
            overrideDNS({"Theserver5" : '10.0.0.1'})
            keyring.createKeys(1, now)
            # check internal state
            ivals = keyring.keySets
            start = mixminion.Common.previousMidnight(now)
            finish = mixminion.Common.previousMidnight(start+(10*24*60*60)+30)
            self.assertEquals(1, len(ivals))
            self.assertEquals((start,finish), ivals[0][0:2])

            keyring.createKeys(2)
        finally:
            undoReplacedAttributes()

        # Check the first key we created
        va, vu, _ = keyring._getLiveKeys()[0]
        self.assertEquals(va, start)
        self.assertEquals(vu, finish)
        keyset = keyring.getServerKeysets()[0]
        self.assertEquals(keyset.getHashLogFileName(),
                          os.path.join(home, "work", "hashlogs", "hash_0001"))
        self.assertEquals(vu, keyring.getNextKeyRotation())

        # Check the second key we created.
        key2time = vu+3600*48
        live = keyring._getLiveKeys(key2time)
        self.assertEquals(len(live),1)
        va, vu, curKey = live[0]
        self.assertEquals(va, finish)
        self.assertEquals(vu, mixminion.Common.previousMidnight(
            finish+10*24*60*60+60))

        # Does published-marking work right?
        self.assert_(not curKey.isPublished())
        curKey.markAsPublished()
        self.assert_(curKey.isPublished())
        keyring.checkKeys()
        va, vu, curKey = keyring._getLiveKeys(key2time)[0]
        self.assert_(curKey.isPublished())
        curKey.markAsUnpublished()
        self.assert_(not curKey.isPublished())
        keyring.checkKeys()
        va, vu, curKey = keyring._getLiveKeys(key2time)[0]
        self.assert_(not curKey.isPublished())
        curKey.markAsPublished()

        # Does 'regenerate' work?
        cfg2 = SERVERCFG%{'home':_FAKE_HOME}
        cfg2 = cfg2.replace("Theserver5", "Theserver999")
        config2 = mixminion.server.ServerConfig.ServerConfig(string=cfg2)

        self.assert_(curKey.isPublished())
        curKey.load()
        key0 = curKey.getPacketKeyID()
        try:
            overrideDNS({"Theserver999" : '1.0.0.1'})
            curKey.regenerateServerDescriptor(config2, keyring.getIdentityKey())
        finally:
            undoReplacedAttributes()
        self.assert_(not curKey.isPublished())
        inf = curKey.getServerDescriptor()
        self.assertEquals(inf['Incoming/MMTP']['Hostname'], "Theserver999")
        curKey.load()
        self.assertEquals(key0, curKey.getPacketKeyID())

        # Make a key in the past, to see if it gets scrubbed.
        keyring.createKeys(1, mixminion.Common.previousMidnight(
            start - 10*24*60*60 + 1))
        self.assertEquals(4, len(keyring.keySets))
        waitForChildren() # make sure keys are really gone before we remove

        # In case we started very close to midnight, remove keys as if it
        # were a little in the future; otherwise, we won't remove the
        # just-expired keys.
        keyring.removeDeadKeys(now+24*60*60)
        self.assertEquals(3, len(keyring.keySets))

        if USE_SLOW_MODE:
            # These are slow, since they regenerate the DH params.
            # Test getDHFile
            f = keyring._getDHFile()
            f2 = keyring._getDHFile()
            self.assertEquals(f, f2)

            # Test getTLSContext
            keyring._getTLSContext()

#----------------------------------------------------------------------
class DNSFarmTests(TestCase):
    def testDNSCache(self):
        import mixminion.server.DNSFarm
        cache = mixminion.server.DNSFarm.DNSCache()
        receiveDict = {}
        lock = threading.RLock()
        def callback(name,val,receiveDict=receiveDict,lock=lock):
            lock.acquire()
            receiveDict[name]=val
            lock.release()
        try:
            DELAY = 0.2
            LATENCY = 1.0
            overrideDNS({'foo'    : '10.2.4.11',
                         'bar'    : '18:0FFF::4:1',
                         'baz.com': '10.99.22.8'},
                        delay=DELAY)
            # Override getProtocolSupport so we don't convert IPv6 addrs to
            # an error if we're running on a host with no IPv6 support.
            mixminion.NetUtils._PROTOCOL_SUPPORT = (1,1)

            self.assertEquals(None, cache.getNonblocking("foo"))
            start = time.time()
            cache.lookup('foo',callback)
            cache.lookup('bar',callback)
            self.assertEquals(cache.getNonblocking("bar"),
                              mixminion.server.DNSFarm.PENDING)
            time.sleep(DELAY/4)
            cache.lookup('baz.com',callback)
            cache.lookup('nowhere.noplace',callback)
            cache.lookup('1.2.3.4', callback)

            while len(receiveDict)<5:
                time.sleep(DELAY/4)
            self.assertEquals(receiveDict['foo'][:2],
                              (socket.AF_INET, '10.2.4.11'))
            self.assertEquals(receiveDict['bar'][:2],
                              (mixminion.NetUtils.AF_INET6, '18:0FFF::4:1'))
            # We allow a little wiggle on DELAY, since many OS's don't
            # stricly guarantee that
            #     t1=time();sleep(x);t2=time();assert t2>=t1+x
            # will pass.
            self.assert_(DELAY*.8 <= receiveDict['foo'][2]-start
                                  <= DELAY+LATENCY)
            self.assert_(DELAY*.8 <= receiveDict['bar'][2]-start
                                  <= DELAY+LATENCY)
            self.assertEquals(receiveDict['nowhere.noplace'][0], "NOENT")
            self.assertEquals(cache.getNonblocking("foo"),
                              receiveDict['foo'])
            self.assertEquals(cache.getNonblocking("baz.com")[:2],
                              (socket.AF_INET, '10.99.22.8'))
            self.assert_(DELAY*1.20 <= receiveDict['baz.com'][2]-start
                                    <= DELAY*1.25 + LATENCY)

            # Change foo's expiration time to be well before nowhere's,
            # then expire foo but not nowhere.
            t2 = receiveDict['nowhere.noplace'][2]
            cache.cache['foo'] = cache.cache['foo'][:2]+((t2-5),)
            cache.cleanCache(t2-1+
                             mixminion.server.DNSFarm.MAX_ENTRY_TTL)

            self.assertEquals(cache.getNonblocking('foo'), None)
            self.assertEquals(cache.getNonblocking('nowhere.noplace'),
                              receiveDict['nowhere.noplace'])

            self.assertEquals(receiveDict['1.2.3.4'][:2],
                              (socket.AF_INET, '1.2.3.4'))

            cache.shutdown(wait=1)
            self.assertEquals(5, len(receiveDict))
        finally:
            undoReplacedAttributes()
            mixminion.NetUtils._PROTOCOL_SUPPORT = None

#----------------------------------------------------------------------

class ServerMainTests(TestCase):
    def testScheduler(self):
        #XXXX008 move this test. it isn'tr a server test any more.
        _Scheduler = mixminion.ScheduleUtils.Scheduler
        lst=[]
        def a(lst=lst): lst.append('a')
        def b(lst=lst): lst.append('b')
        def c(lst=lst): lst.append('c')
        def d(lst=lst): lst.append('d')
        s = _Scheduler()
        self.assertEquals(s.firstEventTime(), -1)
        tm = time.time()

        s.scheduleOnce(tm+1, "A", a)
        s.scheduleOnce(tm+2, "B", b)
        self.assertEquals(s.firstEventTime(), tm+1)
        s.processEvents(tm+1)
        self.assertEquals(['a'],lst)
        del lst[:]

        s.scheduleRecurring(tm+1.5, 1, "C", c)
        s.scheduleOnce(tm+1.9, "D", d)
        self.assertEquals(s.firstEventTime(), tm+1.5)
        s.processEvents(tm+1.5)
        self.assertEquals(["c"], lst)
        diff = abs(s.firstEventTime()-(tm+1.9))
        self.assert_(diff < 0.1)

        s.processEvents(tm+5)
        self.assertEquals(["c", "d", "b", "c" ], lst)

    def testMixPool(self):
        ServerConfig = mixminion.server.ServerConfig.ServerConfig
        MixPool = mixminion.server.ServerMain.MixPool
        baseDir = mix_mktemp()
        mixDir = mix_mktemp()
        cfg = SERVER_CONFIG_SHORT % baseDir

        configTimed = ServerConfig(string=(cfg+
               "MixAlgorithm: timed\nMixInterval: 2 hours\n"))
        configCottrell = ServerConfig(string=(cfg+
               "MixAlgorithm: Mixmaster\nMixInterval: .5 days\n"+
               "MixPoolMinSize: 10\nMixPoolRate: 40%\n"))
        configBCottrell = ServerConfig(string=(cfg+
               "MixAlgorithm: BinomialCottrell\nMixInterval: .25 days\n"+
               "MixPoolMinSize: 10\nMixPoolRate: 40%\n"))

        # Test pool configuration
        pool = MixPool(configTimed, mixDir)
        self.assert_(isinstance(pool.queue,
                                TimedMixPool))
        self.assertEquals(pool.getNextMixTime(100), 100+2*60*60)

        pool = MixPool(configCottrell, mixDir)
        self.assert_(isinstance(pool.queue,
                                CottrellMixPool))
        self.assertEquals(pool.getNextMixTime(100), 100+12*60*60)
        self.assertEquals(pool.queue.minPool, 10)
        self.assertEquals(pool.queue.minSend, 1)
        self.assertFloatEq(pool.queue.sendRate, .4)

        pool = MixPool(configBCottrell, mixDir)
        self.assert_(isinstance(pool.queue,
                                BinomialCottrellMixPool))
        self.assertEquals(pool.getNextMixTime(100), 100+6*60*60)
        self.assertEquals(pool.queue.minPool, 10)
        self.assertEquals(pool.queue.minSend, 1)
        self.assertFloatEq(pool.queue.sendRate, .4)

        # FFFF test other mix pool behavior

#----------------------------------------------------------------------

_EXAMPLE_DESCRIPTORS = {} # name->list of str
EX_SERVER_CONF_TEMPLATE = """
[Server]
Mode: relay
Homedir: %(homedir)s
EncryptIdentityKey: No
PublicKeyLifetime: %(lifetime)s
IdentityKeyBits: 2048
EncryptPrivateKey: no
Nickname: %(nickname)s
Contact-Email: a@b.c
[Incoming/MMTP]
Enabled: yes
Hostname: %(nickname)s
IP: %(ip)s
[Outgoing/MMTP]
Enabled: yes
"""

_EXAMPLE_DESCRIPTORS_INP = [
    # name        days         ip?        validAt
    [ "Fred",     "10 days", "10.0.0.6", (-19,-9,1,11), () ],
    [ "Lola",     "5 days",  "10.0.0.7", (-2,0,5),      (MBOX_TYPE,) ],
    [ "Joe",      "20 days", "10.0.0.8", (-15,5,25),    (SMTP_TYPE,) ],
    [ "Alice",    "8 days",  "10.0.0.9", (-3,5,13),     () ],
    [ "Bob",      "4 days",  "10.0.0.10", (-2, 2, 6, 'X', 0, 4, -3), () ],
    [ "Lisa",     "3 days",  "10.0.0.11", (-10,-1,5),   () ],
]

def getExampleServerDescriptors():
    """Helper function: generate a map of list of ServerInfo objects based
       on the values of _EXAMPLE_DESCRIPTORS_INP"""
    if _EXAMPLE_DESCRIPTORS:
        return _EXAMPLE_DESCRIPTORS

    gen = generateServerDescriptorAndKeys
    tmpkeydir = mix_mktemp()
    now = time.time()

    sys.stdout.flush()

    # For each server...
    serveridx = 0
    for (nickname, lifetime, ip, starting, types) in _EXAMPLE_DESCRIPTORS_INP:
        # Generate a config file
        homedir = mix_mktemp()
        conf = EX_SERVER_CONF_TEMPLATE % locals()
        identity = getRSAKey(serveridx%6,2048)
        serveridx += 1
        for t in types:
            if t == MBOX_TYPE:
                addrf = mix_mktemp()
                writeFile(addrf,"")
                conf += ("[Delivery/MBOX]\nEnabled: yes\nAddressFile: %s\n"+
                         "ReturnAddress: a@b.c\nRemoveContact: b@c.d\n"+
                         "Retry: every 2 hours for 1 week\n") %(
                    addrf)
            elif t == SMTP_TYPE:
                conf += ("[Delivery/SMTP]\nEnabled: yes\n"+
                         "ReturnAddress: a@b.c\n"+
                         "Retry: every 2 hours for 1 week\n")
            else:
                raise MixFatalError("Unrecognized type: %04x"%t)
        try:
            suspendLog()
            conf = mixminion.server.ServerConfig.ServerConfig(string=conf)
            conf.getModuleManager().configure(conf)
        finally:
            resumeLog()

        # Now, for each starting time, generate a server descriptor.
        _EXAMPLE_DESCRIPTORS[nickname] = []
        publishing = now
        for n in xrange(len(starting)):
            if starting[n] == 'X':
                publishing += 60
                continue
            k = "tst%d"%n
            validAt = previousMidnight(now + 24*60*60*starting[n])
            try:
                overrideDNS({nickname: ip})
                gen(config=conf, identityKey=identity, keyname=k,
                    keydir=tmpkeydir, hashdir=tmpkeydir, validAt=validAt,
                    now=publishing)
            finally:
                undoReplacedAttributes()

            sd = os.path.join(tmpkeydir,"key_"+k,"ServerDesc")
            _EXAMPLE_DESCRIPTORS[nickname].append(readFile(sd))

            # (print some dots here; this step can take a while)
            sys.stdout.write('.')
            sys.stdout.flush()
    sys.stdout.flush()
    return _EXAMPLE_DESCRIPTORS

def getDirectory(servers, identity):
    """Return the filename of a newly created server directory, containing
       the server descriptors provided as literal strings in <servers>,
       signed with the RSA key <identity>"""
    cfg = {"Directory":{"ClientVersions":[mixminion.__version__],
                        "ServerVersions":["1.0",mixminion.__version__],
                        } }
    active = IntervalSet()
    sis = []
    nicknames = {}
    for s in servers:
        si = mixminion.ServerInfo.ServerInfo(fname=s, assumeValid=1,
                                             _keepContents=1)
        active += si.getIntervalSet()
        sis.append(si)
        nicknames[si.getNickname()]=1
    nicknames = nicknames.keys()
    nicknames.sort()
    start, end = active.start(), active.end()
    val = mixminion.directory.DirFormats._generateDirectory(
        identity, "consensus", sis, nicknames,
        [(mixminion.Crypto.pk_fingerprint(identity), "nil")], time.time(),
        [mixminion.__version__], [mixminion.__version__])
    tmp = mix_mktemp()
    writeFile(tmp, val)
    return tmp

# variable to hold the latest instance of FakeBCC.
BCC_INSTANCE = None

class ClientUtilTests(TestCase):
    def testEncryptedFiles(self):
        CU = mixminion.ClientUtils
        d = mix_mktemp()
        createPrivateDir(d)
        f1 = os.path.join(d, "foo")
        magic1 = "ABCDEFGH"
        # Test reading and writing.
        CU._writeEncryptedFile(f1,password="x",magic=magic1,data="xyzzyxyzzy")
        contents = readFile(f1)
        self.assert_(contents.startswith(
            "-----BEGIN TYPE III KEYRING-----\n"
            "Version: 0.1\n\n"))
        self.assert_(contents.endswith(
            "\n-----END TYPE III KEYRING-----\n"))
        idx1 = contents.index("\n\n")+2
        idx2 = contents.rindex("\n-----END TYPE")
        contents = base64.decodestring(contents[idx1:idx2])

        self.assertEquals(contents[:8], magic1)
        self.assertEquals(contents[8], '\x00')
        salt = contents[9:17]
        key = mixminion.Crypto.sha1(salt+"x"+salt)[:16]
        decrypted = mixminion.Crypto.ctr_crypt(contents[17:], key)
        self.assertEquals(len(decrypted), 4+1024+20)
        self.assertEquals(decrypted[:4], "\x00\x00\x00\x0A")
        self.assertEquals(decrypted[4:14], "xyzzyxyzzy")
        self.assertEquals(decrypted[-20:],
                          mixminion.Crypto.sha1(decrypted[:-20]+salt+magic1))

        self.assertEquals((magic1,"xyzzyxyzzy"),
                          CU._readEncryptedFile(f1, "x", [magic1, "BLIZNERT"]))

        # Try reading with wrong password.
        self.assertRaises(CU.BadPassword, CU._readEncryptedFile,
                          f1, "nobodaddy", [magic1])

        # Try reading with wrong magic.
        self.assertRaises(ValueError, CU._readEncryptedFile,
                          f1, "x", ["XXXXYYYY"])

        # Try empty data.
        CU._writeEncryptedFile(f1, password="x", magic=magic1, data="")
        self.assertEquals((magic1,""),CU._readEncryptedFile(f1, "x", [magic1]))

        # Test LazyEncryptedStore
        class DummyPasswordManager(CU.PasswordManager):
            def __init__(self,d):
                mixminion.ClientUtils.PasswordManager.__init__(self)
                self.d = d
            def _getPassword(self,name,prompt):
                return self.d.get(name)
            def _getNewPassword(self,name,prompt):
                return self.d.get(name)

        f3 = os.path.join(d, "Baz")
        magic0 = "MAGIC000"
        dpm = DummyPasswordManager({"Password1" : "p1"})
        lep = CU._LazyEncryptedStore(f3, dpm, "Password1", "Q:", "N:",
                                     magic0, lambda: "x"*3)
        # Don't create.
        self.assert_(not lep.isLoaded())
        lep.load(create=0)
        self.assert_(not lep.isLoaded())
        lep.load(create=1)
        self.assert_(lep.isLoaded())
        self.assertEquals("x"*3, lep.get())
        m,v = CU._readEncryptedFile(f3,"p1",[magic0])
        self.assertEquals((magic0,"x"*3), (m,cPickle.loads(v)))

        lep = CU._LazyEncryptedStore(f3, dpm, "Password1", "Q:", "N:",
                                     magic0, lambda: "x"*3)
        lep.load()
        self.assertEquals("x"*3, lep.get())
        dpm.d = {}
        self.assertEquals("x"*3, lep.get())

        # Finally, test keyring.
        f4 = os.path.join(d, "keyring")
        dpm.d = {"ClientKeyring" : "ckr" }
        kr = CU.Keyring(f4, dpm)
        kr.load(create=1) # We should already have the password cached.
        # Test empty keyring.
        self.assertEquals(kr.getAllSURBKeys(), [])
        self.assertEquals(kr.getNewestSURBKey(""), None)
        self.assert_(not kr.isDirty())
        kr.save()
        self.assertEquals(("KEYRING2", ""),
                          CU._readEncryptedFile(f4, "ckr", ["KEYRING2"]))
        kr = CU.Keyring(f4, dpm)
        kr.load()
        self.assertEquals(kr.getAllSURBKeys(), [])

        # Now generate a few keys.
        now = time.time()
        month1 = now+30*24*60*60
        month2 = now+60*24*60*60
        month1d = succeedingMidnight(month1)
        month2d = succeedingMidnight(month2)
        alice1 = kr.newSURBKey("Alice", month1)
        alice2 = kr.newSURBKey("Alice", month2)
        bob = kr.newSURBKey("Bobby",month1)
        self.assertEquals(len(alice1), 20)
        self.assertEquals(len(bob), 20)
        self.assertNotEquals(alice1, alice2)
        self.assert_(kr.isDirty())
        for i in (1,2):
            self.assertEquals(alice2, kr.getNewestSURBKey("alice"))
            self.assertUnorderedEq([("alice",alice1), ("alice",alice2),
                                    ("bobby",bob)], kr.getAllSURBKeys())
            self.assertEquals(None, kr.getNewestSURBKey("alice",
                                               minLifetime=70*24*60*60))
            if i == 1:
                kr.save()
                kr = CU.Keyring(f4, dpm)
                kr.load()

        # Check that contents are correctly encoded.
        _, contents = CU._readEncryptedFile(f4, "ckr", ["KEYRING2"])
        ln = 33
        self.assertEquals(len(contents), ln*3)
        k1, k2, k3 = contents[0:ln], contents[ln:ln*2], contents[ln*2:]
        ex1 = ("\x00" +  # It's a SURB key
               "\x00\x1E" +  # The rest is 30 bytes long
               struct.pack("!L",month1d) + "alice\x00" + alice1)
        ex2 = ("\x00" +  # It's a SURB key
               "\x00\x1E" +  # The rest is 30 bytes long
               struct.pack("!L",month2d) + "alice\x00" + alice2)
        ex3 = ("\x00" +  # It's a SURB key
               "\x00\x1E" +  # The rest is 30 bytes long
               struct.pack("!L",month1d) + "bobby\x00" + bob)
        self.assertUnorderedEq([ex1,ex2,ex3],[k1,k2,k3])

        # Now get fancier: Add an unrecognized type, but expire Alice1 and Bob.
        dummy = "\xFF\x00\x1E"+("X"*30)
        CU._writeEncryptedFile(f4, "ckr", "KEYRING2", contents+dummy)
        kr = CU.Keyring(f4, dpm)
        kr.load(now=now+33*24*60*60)
        self.assert_(kr.isDirty())
        self.assertEquals(kr.getAllSURBKeys(), [('alice', alice2)])
        self.assertEquals(kr.object.unrecognized, [(0xFF, "X"*30)])
        kr.save()
        self.assert_(not kr.isDirty())
        _, contents = CU._readEncryptedFile(f4, "ckr", ["KEYRING2"])
        self.assertEquals(len(contents), 66)
        k1,k2 = contents[0:ln],contents[ln:]
        self.assertUnorderedEq([dummy,ex2], [k1,k2])

    def testSURBLog(self):
        brb = BuildMessage.buildReplyBlock
        SURBLog = mixminion.ClientUtils.SURBLog
        ServerInfo = mixminion.ServerInfo.ServerInfo
        dirname = mix_mktemp()
        fname = os.path.join(dirname, "surblog")

        # generate 3 SURBs.
        examples = getExampleServerDescriptors()
        alice = ServerInfo(string=examples["Alice"][0])
        lola = ServerInfo(string=examples["Lola"][0])
        joe = ServerInfo(string=examples["Joe"][0])
        surbs = [brb([alice,lola,joe], SMTP_TYPE, "bjork@iceland", "x",
                     time.time()+24*60*60)
                 for _ in range(3)]

        #FFFF check for skipping expired and shortlived SURBs.

        s = SURBLog(fname)
        try:
            self.assert_(not s.isSURBUsed(surbs[0]))
            self.assert_(not s.isSURBUsed(surbs[1]))
            s.markSURBUsed(surbs[0])
            self.assert_(s.isSURBUsed(surbs[0]))
            s.close()
            s = SURBLog(fname)
            self.assert_(s.isSURBUsed(surbs[0]))
            self.assert_(not s.isSURBUsed(surbs[1]))
            self.assert_(s.findUnusedSURBs(surbs)[0] is surbs[1])
            one = s.findUnusedSURBs(surbs,1)
            self.assertEquals(len(one),1)
            two = s.findUnusedSURBs(surbs,2)
            self.assert_(two[0] is surbs[1])
            self.assert_(two[1] is surbs[2])
            s.markSURBUsed(surbs[1])
            self.assert_(s.findUnusedSURBs(surbs)[0] is surbs[2])
            s.markSURBUsed(surbs[2])
            self.assert_(s.findUnusedSURBs(surbs) == [])
        finally:
            s.close()

    def testClientQueue(self):
        CQ = mixminion.ClientUtils.ClientQueue
        d = mix_mktemp()
        now = time.time()
        cq = CQ(d)
        p1 = "Z"*(32*1024)
        p2 = mixminion.Crypto.getCommonPRNG().getBytes(32*1024)
        ipv4 = mixminion.Packet.IPV4Info("10.20.30.40",48099,"KZ"*10)
        host = mixminion.Packet.MMTPHostInfo("bliznerty.potrzebie",48099,
                                             "KL"*10)
        self.assertEquals(cq.getHandles(), [])
        self.assert_(not cq.packetExists("Z"))
        h1 = cq.queuePacket(p1, ipv4, now)
        h2 = cq.queuePacket(p2, host, now-24*60*60*10)
        self.assertEquals(ipv4, cq.getRouting(h1))
        self.assert_(cq.packetExists(h1))
        self.assertEquals(host, cq.getRouting(h2))

        cq = CQ(d)
        self.assertUnorderedEq(cq.getHandles(),[h1,h2])
        self.assertEquals(host, cq.getRouting(h2))
        v = cq.getPacket(h2)
        self.assertEquals((host,previousMidnight(now-24*60*60*10)), v[1:])
        self.assertLongStringEq(v[0], p2)

        class FakeDir:
            def getKeyIDByNickname(self, n):
                if n == 'nerty': return "KL"*10
                return None
        D = FakeDir()
        self.assertEquals([h1],
                       cq.getHandlesByDestAndAge(["10.20.30.40"], None, None))
        self.assertEquals([h1],
                       cq.getHandlesByDestAndAge(["10.20.30.40"], D, None))
        self.assertEquals([h2],
                cq.getHandlesByDestAndAge(["bliznerty.potrzebie"], D, None))
        self.assertEquals([h2],
                cq.getHandlesByDestAndAge(["nerty"], D, None))
        self.assertUnorderedEq([h1,h2],
                cq.getHandlesByDestAndAge(["nerty", "10.20.30.40"], D, None))
        self.assertEquals([h2], cq.getHandlesByAge(now-24*60*60))
        cq.removePacket(h2)

        cq.cleanQueue()
        self.assertEquals([h1], cq.getHandles())
        v = cq.getPacket(h1)
        self.assertEquals((ipv4,previousMidnight(now)), v[1:])
        self.assertLongStringEq(v[0], p1)
        cq.removePacket(h1)

class ClientDirectoryTests(TestCase):
    def testClientDirectory(self):
        eq = self.assertEquals
        neq = self.assertNotEquals
        ServerInfo = mixminion.ServerInfo.ServerInfo
        dirname = mix_mktemp()
        config = mixminion.Config.ClientConfig(
            string="[User]\nUserDir: %s\n"%dirname)

        ks = mixminion.ClientDirectory.ClientDirectory(config)

        ## Write the descriptors to disk.
        impdirname = self.writeDescriptorsToDisk()
        edesc = getExampleServerDescriptors()

        ## Test empty keystore
        eq(None, ks.getServerInfo("Fred"))
        self.assertRaises(MixError, ks.getServerInfo, "Fred", strict=1)
        fred = ks.getServerInfo(os.path.join(impdirname, "Fred2"))
        self.assertEquals("Fred", fred.getNickname())
        self.assertSameSD(edesc["Fred"][2],fred)

        ## Test importing.
        ks.importFromFile(os.path.join(impdirname, "Joe0"))
        ks.importFromFile(os.path.join(impdirname, "Joe1.gz"))
        ks.importFromFile(os.path.join(impdirname, "Lola1"))

        now = time.time()
        oneDay=24*60*60
        for i in 0,1,2,3:
            self.assertSameSD(edesc["Joe"][0], ks.getServerInfo("Joe"))
            self.assertSameSD(edesc["Lola"][1], ks.getServerInfo("Lola"))
            self.assertSameSD(edesc["Joe"][1],
                              ks.getServerInfo("Joe", startAt=now+10*oneDay))
            self.assertRaises(MixError, ks.getServerInfo, "Joe",
                              startAt=now+30*oneDay, strict=1)
            self.assertRaises(MixError, ks.getServerInfo, "Joe", startAt=now,
                              endAt=now+6*oneDay, strict=1)
            if i in (0,1,2):
                ks = mixminion.ClientDirectory.ClientDirectory(config)
            if i == 1:
                ks.rescan()
            if i == 2:
                ks.rescan(force=1)

        # Refuse to import a server with a modified identity
        writeFile(os.path.join(impdirname, "Notjoe"),
                  edesc["Lola"][0].replace("Nickname: Lola", "Nickname: Joe"))
        self.assertRaises(MixError,
                          ks.importFromFile,
                          os.path.join(impdirname, "Notjoe"))

        # Try getServerInfo(ServerInfo)
        si = ServerInfo(string=edesc['Lisa'][1],assumeValid=1)
        self.assert_(ks.getServerInfo(si) is si)
        try:
            suspendLog()
            si = ServerInfo(string=edesc['Lisa'][0],assumeValid=1)
            self.assert_(ks.getServerInfo(si) is None)
        finally:
            s = resumeLog()
        self.assert_(stringContains(s, "Server is not currently"))

        ##
        # Now try out the directory.  This is tricky; we add the other
        # descriptors here.
        identity = getRSAKey(0,2048)
        fingerprint = Crypto.pk_fingerprint(identity)
        fname = getDirectory(
            [os.path.join(impdirname, s) for s in
             ("Fred1", "Fred2", "Lola2", "Alice0", "Alice1",
              "Bob3", "Bob4", "Lisa1") ], identity)

        # Replace the real URL and fingerprint with the ones we have; for
        # unit testing purposes, we can't rely on an http server.
        mixminion.ClientDirectory.MIXMINION_DIRECTORY_URL = fileURL(fname)
        mixminion.ClientDirectory.MIXMINION_DIRECTORY_FINGERPRINT = fingerprint

        # Reload the directory.
        ks.update(now=now,force=1)

        for i in 0,1,2,3:
            self.assertSameSD(ks.getServerInfo("Alice"), edesc["Alice"][0])
            self.assertSameSD(ks.getServerInfo("Bob"), edesc["Bob"][3])
            self.assertSameSD(ks.getServerInfo("Bob", startAt=now+oneDay*5),
                              edesc["Bob"][4])


            if i in (0,1,2):
                ks = mixminion.ClientDirectory.ClientDirectory(config)
            if i == 1:
                ks.rescan()
            if i == 2:
                ks.rescan(force=1)

        replaceFunction(ks.store.bases[0], 'downloadDirectory')

        # Now make sure that update is properly zealous.
        ks.update(now=now)
        self.assertEquals([], getReplacedFunctionCallLog())
        ks.update(now=now, force=1)
        self.assertEquals(1, len(getReplacedFunctionCallLog()))
        ks.update(now=now+oneDay+60)
        self.assertEquals(2, len(getReplacedFunctionCallLog()))
        undoReplacedAttributes()
        clearReplacedFunctionCallLog()

        ## Now make sure we can really update the directory.
        # (this is the same as before, but with 'Lisa2'.)
        fname = getDirectory(
            [os.path.join(impdirname, s) for s in
             ("Fred1", "Fred2", "Lola2", "Alice0", "Alice1",
              "Bob3", "Bob4", "Lisa1", "Lisa2") ], identity)
        mixminion.ClientDirectory.MIXMINION_DIRECTORY_URL = fileURL(fname)
        ks.update(force=1)
        # Previous entries.
        self.assertSameSD(ks.getServerInfo("Alice"), edesc["Alice"][0])
        self.assertSameSD(ks.getServerInfo("Bob"), edesc["Bob"][3])
        # New entry
        self.assertSameSD(ks.getServerInfo("Lisa",startAt=now+6*oneDay),
                          edesc["Lisa"][2])
        # Entry from server info
        self.assertSameSD(edesc["Joe"][0], ks.getServerInfo("Joe"))

        def nRuns(lst):
            n = 0
            for idx in xrange(len(lst)-1):
                if lst[idx] == lst[idx+1]:
                    n += 1
            return n

        # Override ks.DEFAULT_REQUIRED_LIFETIME so we don't need to
        # explicitly specify a really early endAt all the time.
        ks.DEFAULT_REQUIRED_LIFETIME = 1

        suspendLog()
        joe = edesc["Joe"]
        alice = edesc["Alice"]
        lola = edesc["Lola"]
        fred = edesc["Fred"]
        bob = edesc["Bob"]
        try:
            ### Try out getPath.
            # 1. Fully-specified paths.
            p = ks.getPath(['Joe', 'Lisa', 'Alice', 'Joe'])

            # 2. Partly-specified paths...
            # 2a. With plenty of servers
            p = ks.getPath([None, None])
            eq(2, len(p))
            neq(p[0].getNickname(), p[1].getNickname())

            p = ks.getPath(["Joe", None, None])
            eq(3, len(p))
            self.assertSameSD(p[0], joe[0])
            neq(p[1].getNickname(), "Joe")
            neq(p[1].getNickname(), p[2].getNickname())

            p = ks.getPath([None, None, "Joe"])
            eq(3, len(p))
            self.assertSameSD(joe[0], p[2])

            p = ks.getPath(["Alice", None, None, "Joe"])
            eq(4, len(p))
            self.assertSameSD(alice[0], p[0])
            self.assertSameSD(joe[0], p[3])
            nicks = [ s.getNickname() for s in p ]
            self.assert_(nicks.count("Alice")>=1)
            self.assert_(nicks.count("Joe")>=1)
            neq(nicks[1],nicks[2])

            p = ks.getPath(["Joe", None, "Alice", "Joe"])
            eq(4, len(p))
            self.assertSameSD(alice[0], p[2])
            self.assertSameSD(joe[0], p[0])
            self.assertSameSD(joe[0], p[3])
            neq(p[1].getNickname(), "Alice")
            neq(p[1].getNickname(), "Joe")

            # 2a.1. (Blocking some servers)
            configBlock = mixminion.Config.ClientConfig(string=(
                "[User]\nUserDir: %s\n[Security]\nBlockServers: Joe\n"
                "BlockEntries: Alice\nBlockExits: Bob\n" % dirname))
            ks.configure(configBlock)

            for _ in xrange(100):
                p = ks.getPath([None]*4)
                eq(4, len(p))
                p = [ s.getNickname() for s in p ]
                self.assertNotEquals("Alice", p[0])
                self.assertNotEquals("Bob", p[3])
                self.assertNotIn("Joe", p)

            ks.configure(config)
            # 2b. With 3 <= servers < length
            dirname2 = mix_mktemp()
            config2 = mixminion.Config.ClientConfig(
                string="[User]\nUserDir: %s\n"%dirname2)

            ks2 = mixminion.ClientDirectory.ClientDirectory(config2)
            ks2.importFromFile(os.path.join(impdirname, "Joe0"))
            ks2.importFromFile(os.path.join(impdirname, "Alice0"))
            ks2.importFromFile(os.path.join(impdirname, "Lisa1"))
            ks2.importFromFile(os.path.join(impdirname, "Bob0"))

            p = ks2.getPath([None]*9)
            eq(9, len(p))
            self.failIf(nRuns([s.getNickname() for s in p]))

            p = ks2.getPath(["Joe"]+[None]*6+["Joe"])
            self.failIf(nRuns([s.getNickname() for s in p]))
            eq(8, len(p))
            self.assertSameSD(joe[0], p[0])
            self.assertSameSD(joe[0], p[-1])

            p = ks2.getPath(["Joe"]+[None]*6)
            self.failIf(nRuns([s.getNickname() for s in p]))
            eq(7, len(p))
            self.assertSameSD(joe[0], p[0])

            p = ks2.getPath([None]*6+["Joe"])
            self.failIf(nRuns([s.getNickname() for s in p]))
            eq(7, len(p))
            self.assertSameSD(joe[0], p[-1])

            # 2c. With 2 servers
            ks2.expungeByNickname("Alice")
            ks2.expungeByNickname("Bob")
            p = ks2.getPath([None]*4)
            self.failIf(nRuns([s.getNickname() for s in p]) > 1)

            p = ks2.getPath(["Joe",None,None,None])

            self.failIf(nRuns([s.getNickname() for s in p]) > 2)
            p = ks2.getPath([None, None, None, "Joe"])
            self.failIf(nRuns([s.getNickname() for s in p]) > 1)

            p = ks2.getPath([None,None,None,None,None, "Joe"])
            self.failIf(nRuns([s.getNickname() for s in p]) > 1)

            # 2d. With only 1.
            ks2.expungeByNickname("Lisa")
            p = ks2.getPath([None]*4)
            eq(len(p), 4)
            p = ks2.getPath(["Joe",None,None,None])
            eq(len(p), 4)
            p = ks2.getPath([None,None,None,"Joe"])
            eq(len(p), 4)

            # 2e. With 0
            self.assertRaises(MixError, ks.getPath,
                              [None]*4, startAt=now+100*oneDay)
        finally:
            s = resumeLog()
        self.assertEquals(4, s.count("to avoid same-server hops"))
        self.assertEquals(3, s.count("Only one relay known"))

        # wrap path parsing and verification and generation.
        def ppath(dir, cfg, path, addr, startAt=None, endAt=None,
                  halfPath=0, nPaths=1):
            isReply = halfPath and (addr is None)
            isSURB = halfPath and (addr is not None)
            pathSpec = mixminion.ClientDirectory.parsePath(
                cfg, path, isReply=isReply,
                isSURB=isSURB)
            dir.validatePath(pathSpec, addr, startAt=startAt, endAt=endAt)
            paths = dir.generatePaths(nPaths, pathSpec, addr, startAt, endAt)
            if nPaths == 1:
                assert len(paths) == 1
                return paths[0]
            else:
                return paths

        paddr = mixminion.ClientDirectory.parseAddress
        email = paddr("smtp:lloyd@dobler.com")
        mboxWithServer = paddr("mbox:Granola@Lola")
        mboxWithoutServer = paddr("mbox:Granola")

        alice = ks.getServerInfo("Alice")
        fred = ks.getServerInfo("Fred")
        bob = ks.getServerInfo("Bob")
        joe = ks.getServerInfo("Joe")
        lola = ks.getServerInfo("Lola")

        def pathIs(p, exp, self=self):
            if isinstance(p[0],mixminion.ServerInfo.ServerInfo):
                p1, p2 = p, ()
                exp1, exp2 = exp, ()
            else:
                p1, p2 = p
                exp1, exp2 = exp
            try:
                self.assertEquals(len(p1),len(exp1))
                self.assertEquals(len(p2),len(exp2))
                for a, b in zip(p1, exp1):
                    self.assertSameSD(a,b)
                for a, b in zip(p2, exp2):
                    self.assertSameSD(a,b)
            except:
                print [s.getNickname() for s in p1], \
                      [s.getNickname() for s in p2]
                raise

        # 1. Successful cases
        # 1a. No colon, no star
        fredfile = os.path.join(impdirname, "Fred1")
        p1,p2 = ppath(ks, None, "Alice,Fred,Bob,Joe", email)
        pathIs((p1,p2), ((alice,fred),(bob,joe)))
        fredfile = os.path.join(impdirname, "Fred1")
        p1,p2 = ppath(ks, None, "Alice,%r,Bob,Joe"%fredfile, email)
        pathIs((p1,p2), ((alice,fred),(bob,joe)))
        p1,p2 = ppath(ks, None, "Alice,Fred,Bob,Lola,Joe", email)
        pathIs((p1,p2), ((alice,fred,bob),(lola,joe)))
        p1,p2 = ppath(ks, None, "Alice,Fred,Bob", mboxWithServer)
        pathIs((p1,p2), ((alice,fred),(bob,lola)))
        p1,p2 = ppath(ks, None, "Alice,Fred,Bob,Lola", mboxWithoutServer)
        pathIs((p1,p2), ((alice,fred),(bob,lola)))
        p1,p2 = ppath(ks, None, "Alice,Fred,Bob,Lola", mboxWithServer)
        pathIs((p1,p2), ((alice,fred),(bob,lola)))
        p1,p2 = ppath(ks, None, "Alice,?,?,Bob", mboxWithServer)
        eq((len(p1),len(p2)), (3,2))
        pathIs((p1[:1],p2[-2:]), ((alice,),(bob,lola)))

        # 1a'. Filename with internal commas and colons, where permitted.
        dirname3 = mix_mktemp()
        os.mkdir(dirname3, 0700)
        fredfile2 = os.path.join(dirname3, "a:b,c")
        try:
            writeFile(fredfile2,edesc["Fred"][1])
        except OSError, e:
            pass
        else:
            p1,p2 = ppath(ks, None, "Alice,%r,Bob,Joe"%fredfile2, email)
            pathIs((p1,p2), ((alice,fred),(bob,joe)))
            p1,p2 = ppath(ks, None, "%r,Alice,Bob,Joe"%fredfile2, email)
            pathIs((p1,p2), ((fred,alice),(bob,joe)))

        # 1b. Colon, no star
        p1,p2 = ppath(ks, None, "Alice:Fred,Joe", email)
        pathIs((p1,p2), ((alice,),(fred,joe)))
        p1,p2 = ppath(ks, None, "Alice:Bob,Fred,Joe", email)
        pathIs((p1,p2), ((alice,),(bob,fred,joe)))
        p1,p2 = ppath(ks, None, "Alice,Bob,Fred:Joe", email)
        pathIs((p1,p2), ((alice,bob,fred),(joe,)))
        p1,p2 = ppath(ks, None, "Alice,Bob,?:Joe", email)
        eq((len(p1),len(p2)), (3,1))
        pathIs((p1[:-1],p2), ((alice,bob),(joe,)))
        p1,p2 = ppath(ks, None, "Alice,Bob,Fred:Joe", email)
        pathIs((p1,p2), ((alice,bob,fred),(joe,)))
        p1,p2 = ppath(ks, None, "Alice,Bob,Fred:Joe", mboxWithServer)
        pathIs((p1,p2), ((alice,bob,fred),(joe,lola)))
        p1,p2 = ppath(ks, None, "Alice,Bob,Fred:Lola", mboxWithoutServer)
        pathIs((p1,p2), ((alice,bob,fred),(lola,)))

        # 1c, 1c':  Stars, no longer used.

        # 1d'. Tilde
        p1,p2 = ppath(ks, None, '~2', email)
        self.assert_(p1 and p2)
        p1,p2 = ppath(ks, None, '?,~4,Bob,Joe', email)
        p = p1+p2
        pathIs((p2[-1], p2[-2],), (joe, bob))
        total = 0
        for _ in xrange(1000):
            p1,p2 = ppath(ks, None, '~2,Bob,Joe', email)
            total += len(p1+p2)
        self.assert_(3.4 <= total/1000.0 <= 4.6)

        # 1e. Complex.
        try:
            suspendLog()
            p1,p2 = ppath(ks, None, '?,Bob,*3:Joe,*2,Joe', email)
        finally:
            resumeLog()
        pathIs((p1[1],p2[0],p2[-1]), (bob, joe, joe))
        eq((len(p1),len(p2)), (5,4))

        # 1f. Half-path
        p1,p2 = ppath(ks, None, "*3", email, halfPath=1)
        eq((len(p1),len(p2)), (0,3))

        # 1g. Multiple paths
        (p1,p2),(p3,p4) = ppath(ks,None,"?:*3",email,nPaths=2)
        self.assertEquals(map(len,[p1,p2,p3,p4]),[1,3,1,3])
        fragEmail2 = paddr("foo@bar.com")
        fragEmail2.setFragmented(1,3)
        (p1,p2),(p3,p4),(p5,p6) = ppath(ks,None,"?:*3",email,nPaths=3)
        self.assertEquals(map(len,[p1,p2,p3,p4,p5,p6]),[1,3,1,3,1,3])
        self.assertEquals(p2[-1].getNickname(), p4[-1].getNickname())
        self.assertEquals(p2[-1].getNickname(), p6[-1].getNickname())

        # 2. Failing cases
        raises = self.assertRaises
        # Nonexistent server
        raises(MixError, ppath, ks, None, "Pierre:Alice,*2", email)
        # Two swap points
        raises(MixError, ppath, ks, None, "Alice:Bob:Joe", email)
        # Last hop doesn't support exit type
        raises(MixError, ppath, ks, None, "Alice:Bob,Fred", email)
        raises(MixError, ppath, ks, None, "Alice:Bob,Fred", mboxWithoutServer)
        # Old star syntax.
        raises(MixError, ppath, ks, None, "Alice,*,Bob,Joe", email)
        # Nonexistent file
        raises(MixError, ppath, ks, None, "./Pierre:Alice,*2", email)

        ## Try 'expungeByNickname'.
        # Zapping 'Lisa' does nothing, since she's in the directory...
        ks.expungeByNickname("Lisa")
        self.assertSameSD(ks.getServerInfo("Lisa",startAt=now+6*oneDay),
                          edesc["Lisa"][2])
        # But Joe can be removed.
        ks.expungeByNickname("Joe")
        eq(None, ks.getServerInfo("Joe"))

        ## Now try clean()
        ks.clean() # Should do nothing.
        ks = mixminion.ClientDirectory.ClientDirectory(config)
        ks.clean(now=now+oneDay*500) # Should zap all of imported servers.
        raises(MixError, ks.getServerInfo, "Lola", strict=1)

    def testFeatureMaps(self):
        from mixminion.ClientDirectory import compressFeatureMap
        from mixminion.ClientDirectory import formatFeatureMap
        d = self.writeDescriptorsToDisk()
        dirname = mix_mktemp()
        config = mixminion.Config.ClientConfig(
            string="[User]\nUserDir: %s\n"%dirname)
        direc = mixminion.ClientDirectory.ClientDirectory(config)
        self.loadDirectory(direc, d)
        edesc = getExampleServerDescriptors()

        bob3 = mixminion.ServerInfo.ServerInfo(string=edesc['Bob'][3])
        bob4 = mixminion.ServerInfo.ServerInfo(string=edesc['Bob'][4])
        features1 = [ 'software' , 'caps', 'status' ]
        # Simple tests for getFeatureMap
        fm = direc.getFeatureMap(features1)
        self.assertUnorderedEq(fm.keys(), ['Fred','Lola','Alice','Bob','Lisa'])
        self.assertEquals(len(fm['Bob']), 2)
        self.assertUnorderedEq(fm['Bob'].keys(),
                               [(bob3['Server']['Valid-After'],
                                 bob3['Server']['Valid-Until']),
                                (bob4['Server']['Valid-After'],
                                 bob4['Server']['Valid-Until'])])
        self.assertEquals(fm['Bob'].values()[0],
                          { 'software' : 'Mixminion %s' %mixminion.__version__,
                            'caps' : 'relay',
                            'status' : '(ok)' })

        # Now let's try compressing.
        fm = { 'Alice' : { (100,200) : { "A" : "xxx", "B" : "yy" },
                           (200,300) : { "A" : "xxx", "B" : "yy" },
                           (350,400) : { "A" : "xxx", "B" : "yy" } },
               'Bob'   : { (100,200) : { "A" : "zz", "B" : "ww" },
                           (200,300) : { "A" : "zz", "B" : "kk" } } }
        fm2 = compressFeatureMap(fm, ignoreGaps=0, terse=0)
        self.assertEquals(fm2,
            { 'Alice' : { (100,300) : { "A" : "xxx", "B" : "yy" },
                          (350,400) : { "A" : "xxx", "B" : "yy" } },
              'Bob'   : { (100,200) : { "A" : "zz", "B" : "ww" },
                          (200,300) : { "A" : "zz", "B" : "kk" } } })
        fm3 = compressFeatureMap(fm, ignoreGaps=1, terse=0)
        self.assertEquals(fm3,
            { 'Alice' : { (100,400) :  { "A" : "xxx", "B" : "yy" } },
              'Bob'   : { (100,200) : { "A" : "zz", "B" : "ww" },
                          (200,300) : { "A" : "zz", "B" : "kk" } } })

        fm4 = compressFeatureMap(fm, terse=1)
        self.assertEquals(fm4,
            { 'Alice' : { (100,400) : { "A" : "xxx", "B" : "yy" } },
              'Bob'   : { (100,300) : { "A" : "zz", "B" : "ww / kk" } } })

        # Test formatFeatureMap.
        self.assertEquals(formatFeatureMap(["A","B"],fm4,showTime=0,cascade=0),
          [ "Alice:xxx yy", "Bob:zz ww / kk" ])
        self.assertEquals(formatFeatureMap(["A","B"],fm4,showTime=0,cascade=0,
                                           just=1,sep="!"),
          [ "Alice:xxx!yy     ",
            "Bob  :zz !ww / kk" ])
        self.assertEquals(formatFeatureMap(["A","B"],fm3,showTime=1,cascade=0),
          [ "Alice:1970-01-01 to 1970-01-01:xxx yy",
            "Bob:1970-01-01 to 1970-01-01:zz ww",
            "Bob:1970-01-01 to 1970-01-01:zz kk" ])

        day1 = 24*60*60
        day2 = 2*24*60*60
        day3 = 3*24*60*60
        fmx = { 'Alice' : { (day1,day3)  : { "A" : "aa", "B" : "bb" } },
                'Bob' : { (day1,day2) : { "A" : "a1", "B" : "b1" },
                          (day2,day3) : { "A" : "a2", "B" : "b2" } } }
        self.assertEquals(formatFeatureMap(["A","B"],fmx,cascade=1,sep="##"),
          [ "Alice:", "  [1970-01-02 to 1970-01-04] aa##bb",
            "Bob:",
            "  [1970-01-02 to 1970-01-03] a1##b1",
            "  [1970-01-03 to 1970-01-04] a2##b2" ])

        self.assertEquals(formatFeatureMap(["A","B"],fmx,showTime=1,cascade=2),
          [ "Alice:", "  [1970-01-02 to 1970-01-04]", "    A:aa", "    B:bb",
            "Bob:",
            "  [1970-01-02 to 1970-01-03]", "    A:a1", "    B:b1",
            "  [1970-01-03 to 1970-01-04]", "    A:a2", "    B:b2" ])

    def writeDescriptorsToDisk(self):
        edesc = getExampleServerDescriptors()
        d = mix_mktemp()
        createPrivateDir(d)
        for server, descs in edesc.items():
            for idx in xrange(len(descs)):
                fname = os.path.join(d, "%s%s"%(server,idx))
                writeFile(fname, descs[idx])
                f = gzip.GzipFile(fname+".gz", 'wb')
                f.write(descs[idx])
                f.close()
        return d

    def loadDirectory(self, direc, d, now=None):
        identity = getRSAKey(0,2048)
        fingerprint = Crypto.pk_fingerprint(identity)
        fname = getDirectory(
            [os.path.join(d, s) for s in
             ("Fred1", "Fred2", "Lola2", "Alice0", "Alice1",
              "Bob3", "Bob4", "Lisa1", "Lisa2") ], identity)
        mixminion.ClientDirectory.MIXMINION_DIRECTORY_URL = fileURL(fname)
        mixminion.ClientDirectory.MIXMINION_DIRECTORY_FINGERPRINT = fingerprint
        direc.update(now=now)

    def assertSameSD(self, s1, s2):
        self.assert_(self.isSameServerDesc(s1,s2))

    def isSameServerDesc(self, s1, s2):
        """s1 and s2 are either ServerInfo objects or strings containing server
           descriptors. Returns 1 iff their digest fields match"""
        ds = []
        for s in s1, s2:
            if type(s) == types.StringType:
                m = re.search(r"^Digest: (\S+)\n", s, re.M)
                assert m
                ds.append(base64.decodestring(m.group(1)))
            elif isinstance(s, mixminion.ServerInfo.ServerInfo):
                ds.append(s.getDigest())
            else:
                return 0
        return ds[0] == ds[1]

class ClientMainTests(TestCase):
    def testAddress(self):
        def parseEq(s, tp, addr, server, eq=self.assertEquals):
            "Helper: return true iff parseAddress(s).getRouting() == t,s,a."
            t, a, s = mixminion.ClientDirectory.parseAddress(s).getRouting()
            eq(t, tp)
            eq(s, server)
            eq(a, addr)

        ##
        # Check valid mbox and smtp addresses
        parseEq("mbox:foo", MBOX_TYPE, "foo", None)
        parseEq("mbox:foo@bar", MBOX_TYPE, "foo", "bar")
        parseEq("mbox:foo@bar@baz", MBOX_TYPE, "foo", "bar@baz")
        parseEq("smtp:foo@bar", SMTP_TYPE, "foo@bar", None)
        parseEq("smtp:foo@bar.com", SMTP_TYPE, "foo@bar.com", None)
        parseEq("foo@bar.com", SMTP_TYPE, "foo@bar.com", None)
        ##
        # Check other address formats.
        parseEq("drop", DROP_TYPE, "", None)
        parseEq("test:foobar", 0xFFFE, "foobar", None)
        parseEq("test", 0xFFFE, "", None)
        parseEq("0x0999:zymurgy", 0x999, "zymurgy", None)
        parseEq("0x0999", 0x999, "", None)
        parseEq("0x0999:", 0x999, "", None)

        def parseFails(s, f=self.failUnlessRaises):
            f(ParseError, mixminion.ClientDirectory.parseAddress, s)

        # Check failing cases
        parseFails("sxtp:foo@bar.com") # unknown module
        parseFails("mbox") # missing mbox address
        parseFails("mbox:") # missing mbox address
        parseFails("smtp:Foo") # Invalid mailbox
        parseFails("smtp:foo@bar@baz") # Invalid mailbox
        parseFails("hello-friends") # Invalid mailbox
        parseFails("@bar.baz") # Invalid mailbox
        parseFails("moo:") # Unknown module; no text
        parseFails("moo") # Unknown module; no text
        parseFails(":oom") # Missing module
        parseFails("0xZZ:zymurgy") # Bad hex literal
        parseFails("0xZZ") # Bad hex literal, no data.
        parseFails("0x999")
        parseFails("0x99999")
        parseFails("0x9999z")
        parseFails("0xFEEEF:zymurgy") # Hex literal out of range


    def testClientKeyring(self):
        keydir = mix_mktemp()
        keyring = mixminion.ClientMain.ClientKeyring(keydir)
        # Check for some nonexistent keys.
        self.assertEquals([], keyring.getSURBKeys(password="pwd"))
        self.assertEquals(None, keyring.getSURBKey(create=0))
        # Reload, try again:
        kfirst = None
        for _ in 1, 2:
            keyring = mixminion.ClientMain.ClientKeyring(keydir)
            self.assertEquals(kfirst, keyring.getSURBKey(
                create=0,password="pwd"))
            k1 = keyring.getSURBKey(create=1,password="pwd")
            if kfirst:
                self.assertEquals(k1, kfirst)
            kfirst = k1
            self.assertEquals(20, len(k1))
            k2 = keyring.getSURBKey(name="Bob",create=1)
            self.assertEquals(20, len(k2))
            self.assertNotEquals(k1,k2)
            self.assertUnorderedEq([("",k1),("bob",k2)], keyring.getSURBKeys())

        # Incorrect password
        keyring = mixminion.ClientMain.ClientKeyring(keydir)
        try:
            suspendLog()
            self.assertEquals(None,keyring.getSURBKey(password="incorrect"))
        finally:
            s = resumeLog()
        self.assert_(stringContains(s, "Incorrect password"))

    def testMixminionClient(self):
        # Create and configure a MixminionClient object...
        parseAddress = mixminion.ClientDirectory.parseAddress
        parsePath = mixminion.ClientDirectory.parsePath
        userdir = mix_mktemp()
        usercfgstr = "[User]\nUserDir: %s\n[DirectoryServers]\n"%userdir
        usercfg = mixminion.Config.ClientConfig(string=usercfgstr)
        client = mixminion.ClientMain.MixminionClient(usercfg)

        # Create a directory...
        directory = mixminion.ClientDirectory.ClientDirectory(usercfg)

        edesc = getExampleServerDescriptors()
        fname = mix_mktemp()
        for server, descriptors in edesc.items():
            for d in descriptors:
                writeFile(fname, d)
                try:
                    directory.importFromFile(fname)
                except UIError:
                    pass

        # Now try with some servers...
        edesc = getExampleServerDescriptors()

        # ... and for now, we need to restart the client.
        client = mixminion.ClientMain.MixminionClient(usercfg)

        pathSpec1 = parsePath(usercfg, "lola,joe:alice,joe")

        ##  Test generateForwardPacket.
        # We replace 'buildForwardPacket' to make this easier to test.
        replaceFunction(mixminion.BuildMessage, "buildForwardPacket",
                        lambda *a, **k:"X")
        try:
            getCalls = getReplacedFunctionCallLog
            clearCalls = clearReplacedFunctionCallLog
            # First, two forward messages that end with 'joe' and go via
            # SMTP
            payload = "Hey Joe, where you goin' with that gun in your hand?"
            client.generateForwardPackets(
                directory,
                parseAddress("joe@cledonism.net"),
                pathSpec1,
                payload, 0, time.time(), time.time()+200)
            client.generateForwardPackets(
                directory,
                parseAddress("smtp:joe@cledonism.net"),
                pathSpec1,
                "Hey Joe, where you goin' with that gun in your hand?",
                0, time.time(), time.time()+200)

            for fn, args, kwargs in getCalls():
                self.assertEquals(fn, "buildForwardPacket")
                self.assertEquals(args[1:3],
                                  (SMTP_TYPE, "joe@cledonism.net"))
                self.assert_(len(args[3]) == len(args[4]) == 2)
                self.assertEquals(["Lola", "Joe", "Alice", "Joe"],
                     [x['Server']['Nickname'] for x in args[3]+args[4]])
            clearCalls()

            # Now try an mbox message, with an explicit last hop.
            payload = "Hey, Lo', where you goin' with that pun in your hand?"
            client.generateForwardPackets(
                directory,
                parseAddress("mbox:granola"),
                parsePath(usercfg, "lola,joe:alice,lola"),
                payload, 0, time.time(), time.time()+200)
            # And an mbox message with a last hop implicit in the address
            client.generateForwardPackets(
                directory,
                parseAddress("mbox:granola@Lola"),
                parsePath(usercfg, "Lola,Joe:Alice"),
                payload, 0, time.time(), time.time()+200)

            for fn, args, kwargs in getCalls():
                self.assertEquals(fn, "buildForwardPacket")
                self.assertEquals(args[1:3],
                                  (MBOX_TYPE, "granola"))
                self.assertEquals(payload,
                    BuildMessage.decodePayload(args[0],"Z"*20).getUncompressedContents())
                self.assert_(len(args[3]) == len(args[4]) == 2)
                self.assertEquals(["Lola", "Joe", "Alice", "Lola"],
                     [x.getNickname() for x in args[3]+args[4]])
            clearCalls()
        finally:
            undoReplacedAttributes()
            clearCalls()

        ### Now try some failing cases for generateForwardPackets

        # Temporarily replace BlockingClientConnection so we can try the client
        # without hitting the network.
        args = []
        def fakeSendPackets(routing,packetList,timeout=300,callback=None,
                            args=args):
            args.append((routing,packetList,timeout,callback))
            for i in xrange(len(packetList)):
                callback(i)

        replaceAttribute(mixminion.MMTPClient, "sendPackets", fakeSendPackets)
        overrideDNS({'alice' : "10.0.0.100"})
        try:
            client.sendForwardMessage(
                directory,
                parseAddress("mbox:granola@Lola"),
                parsePath(usercfg,"alice,lola,joe,alice:joe,alice"),
                "You only give me your information.",
                time.time(), time.time()+300)

            r,p,t,c = args[0]
            # first hop is alice
            self.assertEquals(r.hostname, "alice")
            self.assertEquals(r.port, 48099)
            self.assertEquals(1, len(p))
            self.assertEquals(32*1024, len(p[0]))

        finally:
            undoReplacedAttributes()
            clearCalls()

#----------------------------------------------------------------------
class FragmentTests(TestCase):
    def testFragmentParams(self):
        K28 = 28*1024
        FP = mixminion.Fragments.FragmentationParams
        # 1 chunk, small
        fp1 = FP(K28 + 1, 0)
        self.assertEquals((fp1.length, fp1.fragCapacity, fp1.k, fp1.n,
                           fp1.nChunks, fp1.chunkSize, fp1.paddedLen,
                           fp1.paddingLen),
                          (K28+1, K28 - 47, 2, 3,
                           1, 2*(K28-47), 2*(K28-47),
                           fp1.paddedLen-(K28+1)))
        # 1 chunk, a bit larger, with enc-fwd overhead
        fp1 = FP(150*1024, 38)
        self.assertEquals((fp1.length, fp1.fragCapacity, fp1.k, fp1.n,
                           fp1.nChunks, fp1.chunkSize, fp1.paddedLen,
                           fp1.paddingLen),
                          (150*1024, K28 - 47 - 38, 8, 11,
                           1, 8*(K28-47-38), 8*(K28-47-38),
                           fp1.paddedLen-(150*1024)))
        # 2 chunks.
        fp1 = FP(K28 * 20, 0)
        self.assertEquals((fp1.length, fp1.fragCapacity, fp1.k, fp1.n,
                           fp1.nChunks, fp1.chunkSize, fp1.paddedLen,
                           fp1.paddingLen),
                          (K28*20, K28 - 47, 16, 22,
                           2, 16*(K28-47), 32*(K28-47),
                           fp1.paddedLen-(K28*20)))


        # 3 chunks.
        fp1 = FP(K28 * 32 + 101, 0)
        self.assertEquals((fp1.length, fp1.fragCapacity, fp1.k, fp1.n,
                           fp1.nChunks, fp1.chunkSize, fp1.paddedLen,
                           fp1.paddingLen),
                          (K28*32+101, K28 - 47, 16, 22,
                           3, 16*(K28-47), 48*(K28-47),
                           fp1.paddedLen-(K28*32+101)))


    def testFragmentation(self):
        FP = mixminion.Fragments.FragmentationParams

        # One chunk.
        msg = Crypto.getCommonPRNG().getBytes(150*1024)
        fp1 = FP(len(msg), 38)
        fec = mixminion.Fragments._getFEC(8, 11)
        blocks = fp1.getFragments(msg)
        self.assertEquals(len(blocks), 11)
        for b in blocks:
            self.assertEquals(len(b), 28*1024 - 47 - 38)
        m2 = "".join(fec.decode(zip(range(3,11),blocks[3:])))
        self.assertLongStringEq(msg, unwhiten(m2[:150*1024]))

        # Three chunks.
        msg = Crypto.getCommonPRNG().getBytes(28*32*1024 + 101)
        fp1 = FP(len(msg), 0)
        fec = mixminion.Fragments._getFEC(16, 22)
        blocks = fp1.getFragments(msg)
        self.assertEquals(len(blocks), 66)
        for b in blocks: self.assertEquals(len(b), 28*1024 - 47)
        chunks = []
        for chunkno in 0,1,2:
            blocksInChunk = zip(range(22), blocks[chunkno*22:(chunkno+1)*22])
            receivedBlocks = Crypto.getCommonPRNG().shuffle(blocksInChunk, 16)
            self.assertEquals(16, len(receivedBlocks))
            chunks.append("".join(fec.decode(receivedBlocks)))
        self.assertLongStringEq(msg, unwhiten(("".join(chunks))[:len(msg)]))

    def testFragmentPool(self):
        em = mixminion.BuildMessage.encodeMessage
        pp = mixminion.Packet.parsePayload
        M1 = Crypto.getCommonPRNG().getBytes(1024*30)
        M2 = Crypto.getCommonPRNG().getBytes(1024*900)
        pkts1 = [ pp(x) for x in em(M1,0) ]
        pkts2 = [ pp(x) for x in em(M2,0) ]
        self.assertEquals(map(len, [pkts1,pkts2]),
                          [3, 66])

        loc = mix_mktemp()
        pool = mixminion.Fragments.FragmentPool(loc)
        self.assertEquals([], pool.listReadyMessages())
        pool.unchunkMessages()

        ####
        # Reconstruct: simple case.
        ####
        pool.addFragment(pkts1[2]) # Add a single packet.
        self.assertEquals(1, pool.store.count()) # Is it there?
        self.assertEquals([], pool.listReadyMessages()) # Message isn't ready.
        pool.addFragment(pkts1[0]) # Add another packet.
        self.assertEquals(2, pool.store.count()) # Two packets waiting.
        pool.addFragment(pkts1[1]) # duplicate; should be 'unnneeded'
        self.assertEquals(2, pool.store.count()) # Still two waiting.
        self.assertEquals([], pool.listReadyMessages()) # Still not ready.
        pool.unchunkMessages() # Now try to unchunk the message.
        self.assertEquals([pkts1[0].msgID], pool.listReadyMessages()) # Ready!
        mid = pool.listReadyMessages()[0] # Get the message; is it right?
        self.assertLongStringEq(M1, uncompressData(pool.getReadyMessage(mid)))
        pool.markMessageCompleted(mid) # Mark it as done.
        self.assertEquals([], pool.listReadyMessages()) # Not there any more.
        pool.addFragment(pkts1[1]) # Adding more fragments to it doesn't work.
        self.assertEquals([], pool.store.getAllMessages())

        ####
        # Reconstruct: large message, stop half-way, reload, finish.
        ####
        # enough for chunk1 -- 17 messages
        for p in pkts2[5:22]: pool.addFragment(p)
        # enough for half of chunk2: 8 messages
        for p in pkts2[22:30]: pool.addFragment(p)
        pool.unchunkMessages()
        # close and re-open messages
        pool.close()
        pool = mixminion.Fragments.FragmentPool(loc)
        # Enough for the rest of message 4...  8 from 2, 17 from 3.
        for p in pkts2[36:44]+pkts2[49:66]:
            pool.addFragment(p)
            pool.unchunkMessages()
        self.assertEquals(len(pool.listReadyMessages()), 1)
        mid = pool.listReadyMessages()[0]
        self.assertLongStringEq(M2, uncompressData(pool.getReadyMessage(mid)))
        pool.markMessageCompleted(mid)
        pool.close()
        pool = mixminion.Fragments.FragmentPool(loc)
        pool.addFragment(pkts2[48])
        self.assertEquals([], pool.store.getAllMessages())

        # free some RAM
        M1 = M2 = None
        pkts1 = pkts2 = None

        ####
        # Try an interleaved case with two smallish msgs and one big one.
        # Provoke an error in the big one part way through.
        ####
        M1 = Crypto.getCommonPRNG().getBytes(1024*150)
        M2 = Crypto.getCommonPRNG().getBytes(1024*200)
        M3 = Crypto.getCommonPRNG().getBytes(1024*900)
        pkts1 = [ pp(x) for x in em(M1,38) ]
        pkts2 = [ pp(x) for x in em(M2,0) ]
        pkts3 = [ pp(x) for x in em(M3,0) ]
        self.assertEquals(map(len,[pkts1,pkts2,pkts3]),
                          [11,11,66])
        for i in xrange(7,11):
            pool.addFragment(pkts1[i])
            pool.addFragment(pkts2[11-i])
        for i in xrange(26):
            pool.addFragment(pkts3[i])
        pool.unchunkMessages()
        self.assertEquals(pool.store.count(), 4+4+1+4)
        # Change index of message to impossible value, make sure m3 gets
        # dropped.
        pkts3[60].index = 66
        try:
            suspendLog()
            pool.addFragment(pkts3[60])
        finally:
            s = resumeLog()

        self.assert_(stringContains(s, "Found inconsistent fragment"))
        self.assertEquals(pool.store.count(), 4+4)
        for i in xrange(40,55):
            pool.addFragment(pkts3[i])
        self.assertEquals(pool.store.count(), 4+4)

        for i in xrange(1,5):
            pool.addFragment(pkts1[i])
            pool.addFragment(pkts2[11-i])
        pool.unchunkMessages()
        msgids = pool.listReadyMessages()
        self.assertUnorderedEq([pkts1[0].msgID, pkts2[0].msgID],
                               msgids)
        self.assertLongStringEq(M1,
                 uncompressData(pool.getReadyMessage(pkts1[0].msgID)))
        pool.markMessageCompleted(pkts1[0].msgID)

        # Force pkts2 to rejected by expiring it.
        pool.expireMessages(time.time()+48*60*60)
        self.assertEquals(pool.listReadyMessages(), [])
        pool.close()

#----------------------------------------------------------------------

class PingerTests(TestCase):

    def testPinglog(self):
        #XXXX We need way better tests here. 'now' needs to be an arg to
        #all the new log functions.
        P = mixminion.server.Pinger
        if not P.canRunPinger():
            print """
[Skipping ping tests; your Python is too old to support the pinger code, or
you haven't installed pysqlite.  This is nothing to worry about, unless you
wanted to play with the new experimental built-in pinger code.]"""
            return

        id0 = "Premature optimizati"
        id1 = "on is the root of al"
        id2 = "l evil in programmin"
        id3 = "g. -- Prof Don Knuth"
        assert len(id0)==len(id1)==len(id2)==len(id3)==20

        d = mix_mktemp()
        os.mkdir(d,0700)
        loc = os.path.join(d, "db")
        t = previousMidnight(time.time())+3600
        log = P.openPingLog(None,location=loc)
        log.startup(now=t)
        log.heartbeat(now=t+1)
        log.heartbeat(now=t+20)
        log.connected(id0,now=t+20)
        log.connectFailed(id1,now=t+20.2)
        log.connected(id0,now=t+30)
        log.connected(id1,now=t+30.2)
        log.connected(id2,now=t+31)
        log.queuedPing("\x00Z"*10, [id0], now=t+31)
        log.queuedPing("BN"*10, [id1], now=t+31)
        log.queuedPing("!!"*10, [id0,id1], now=t+31)
        log.queuedPing("''"*10, [id2], now=t+32)
        log.queuedPing("<>"*10, [id0], now=t+32)
        log.connected(id0,now=t+60)
        log.connected(id1,now=t+60.2)
        log.connectFailed(id1,now=t+70)
        log.connected(id0,now=t+90)
        log.gotPing("\x00Z"*10, now=t+130)
        log.gotPing("BN"*10, now=t+150)
        suspendLog()
        try:
            log.gotPing("BL"*10, now=t+160) #Never sent.
        finally:
            s = resumeLog()
        self.assertEndsWith(s, "Received ping with no record of its hash\n")
        log.gotPing("''"*10, now=t+161.1)
        log.rotate(t-15*24*60*60,t-15*24*60*60)
        log.heartbeat(t+200)
        log.calculateUptimes(t,t+200,now=t+200)
        ups = log.getUptimes(t,t+200)
        interval = (previousMidnight(t), succeedingMidnight(t))

        # I've been up 200 seconds this day, of which not all has passed.
        self.assertFloatEq(ups[interval]['<self>'],
                           200./(24*60*60))
        # Of the time I've been up, server with id0 has never been down.
        self.assertFloatEq(ups[interval][id0], 1.0)
        # id1 was down at 20, up at 30, up at 60, down at 70.  So we'll
        # assume it was down from 20...25, up from 25..65, down from 65..70.
        # That makes 10 sec down, 40 sec up.
        self.assertFloatEq(ups[interval][id1], 40/50.)
        # id2 was only down once in the interval; we refuse to extrapolate.
        self.assert_(not ups[interval].has_key(id2))

        log.calculateChainStatus(now=t+200)
        log.calculateAll(now=t+200)
        log.shutdown()
        #log.calculateDailyResults( ) #XXXX TEST
        log.close()
        log = P.openPingLog(None,location=loc)
        t += 3600
        log.startup(now=t)
        log.calculateUptimes(t-3600, t+100, now=t+100)
        log.calculateOneHopResult(t)
        log.calculateChainStatus(t)

        statusFile = os.path.join(d, "status")
        log.calculateAll(statusFile,now=t+200)
        env = {}
        execfile(statusFile,env)

        log.rotate(t+15*24*60*60,t+30*24*60*60)
        log.close()

#----------------------------------------------------------------------

def initializeGlobals():
    init_crypto()

    mixminion.ClientMain.configureClientLock(mix_mktemp())

    # Suppress 'files-can't-be-securely-deleted' message while testing
    LOG.setMinSeverity("FATAL")
    #LOG.setMinSeverity("TRACE")
    mixminion.Common.secureDelete([],1)

    # Don't complain about owner on /tmp, no matter who it is.
    mixminion.Common._VALID_DIRECTORIES["/tmp"] = 1
    mixminion.Common._VALID_DIRECTORIES["/var/tmp"] = 1

    # Disable TRACE and DEBUG log messages, unless somebody overrides from
    # the environment.
    LOG.setMinSeverity(os.environ.get('MM_TEST_LOGLEVEL', "WARN"))
    #LOG.setMinSeverity(os.environ.get('MM_TEST_LOGLEVEL', "TRACE"))

    # Suppress DNS resolution on fake hostnames.
    klh = mixminion.server.ServerKeys._KNOWN_LOCAL_HOSTNAMES
    for name in [ 'Theserver', 'Theserver2', 'Theserver3', 'Theserver4',
                  'Theserver5', 'Alice', 'Bob', 'Fred', 'Lola', 'Joe',
                  'Lisa' ]:
        klh[name]=1

def testSuite():
    """Return a PyUnit test suite containing all the unit test cases."""
    suite = unittest.TestSuite()
    loader = unittest.TestLoader()
    tc = loader.loadTestsFromTestCase

    if 0:
        suite.addTest(tc(PingerTests))
        return suite
    testClasses = [MiscTests,
                   MinionlibCryptoTests,
                   MinionlibFECTests,
                   CryptoTests,
                   PacketTests,
                   LogTests,
                   FileParanoiaTests,
                   ConfigFileTests,
                   HashLogTests,
                   BuildMessageTests,
                   PacketHandlerTests,
                   FilestoreTests,
                   FragmentTests,
                   QueueTests,
                   EventStatsTests,
                   NetUtilTests,
                   DNSFarmTests,
                   ClientUtilTests,

                   DirectoryServerTests,

                   ModuleTests,
                   ClientDirectoryTests,
                   ClientMainTests,
                   ServerKeysTests,
                   ServerMainTests,
                   PingerTests,

                   # These tests are slowest, so we do them last.
                   ModuleManagerTests,
                   ServerInfoTests,
                   MMTPTests
                  ]

    for cl in testClasses:
        suite.addTest(tc(cl))

    return suite

def testAll(name, args):
    initializeGlobals()
    unittest.TextTestRunner(verbosity=1).run(testSuite())

    # If we're doing coverage analysis, then report on all the modules
    # under the mixminion package.
    if os.environ.get("MM_COVERAGE"):
        allmods = [ mod for name, mod in sys.modules.items()
                    if (mod is not None and
                        name.startswith("mixminion") and
                        name != 'mixminion._minionlib') ]
        coverage.report(allmods)
