Hi,
I've added newsgroups support to mixminion, patch included.
The client now supports "news:group1,group2,group3"; type
destinations, the server now has "news" capability and forwards
incoming newsgroup postings to a configurable mail2news gateway,
e.g. mail2news@xxxxxxxxx -- a template for the configuration is
included below the patch.
Crosspostings to more than three groups are dropped. Additional
headers are not yet allowed, but might be added quite easily.
BTW: why are there two NEWS_TYPE definitions, 102 and 104?
Feedback is welcome.
Especially I'd like somebody else to have a look into the
following, because I probably have missed something with the
(configuration parsing?) part:
* I had to add desc.has_section("Delivery/NEWS") before doing
get("Version"), and this is not needed for e.g. Delivery/MBOX.
* If this check is not included for Delivery/NEWS, there are
errors (see below) with the client, If the check is included,
both errors seem to not occur.
===== Beginn =====
Index: Common.py
===================================================================
--- Common.py (revision 2)
+++ Common.py (working copy)
@@ -1532,6 +1532,19 @@
return m is None
#----------------------------------------------------------------------
+
+# A single newsgroup is a sequence of dot-separated atoms.
+_SINGLE_NEWSGROUP_PAT = r"(?:%s)(?:\.(?:%s))*" % (_ATOM_PAT, _ATOM_PAT)
+# A list of newsgroups is a sequence of comma-separated newsgroups.
+_NEWSGROUPS_PAT = r"\A%s(,%s)*\Z" % (_SINGLE_NEWSGROUP_PAT, _SINGLE_NEWSGROUP_PAT)
+NEWSGROUPS_RE = re.compile(_NEWSGROUPS_PAT)
+
+def isNewsgroups(s):
+ """Return true iff s is a valid sequence of newsgroup names"""
+ n = NEWSGROUPS_RE.match(s)
+ return n is not None
+
+#----------------------------------------------------------------------
# Signal handling
def waitForChildren(onceOnly=0, blocking=1):
Index: ClientDirectory.py
===================================================================
--- ClientDirectory.py (revision 2)
+++ ClientDirectory.py (working copy)
@@ -33,8 +33,8 @@
ceilDiv, createPrivateDir, formatDate, formatFnameTime, openUnique, \
previousMidnight, readPickled, readPossiblyGzippedFile, \
replaceFile, tryUnlink, writePickled, floorDiv, isSMTPMailbox
-from mixminion.Packet import MBOX_TYPE, SMTP_TYPE, DROP_TYPE, FRAGMENT_TYPE, \
- parseMBOXInfo, parseRelayInfoByType, parseSMTPInfo, ParseError, \
+from mixminion.Packet import MBOX_TYPE, NEWS_TYPE, SMTP_TYPE, DROP_TYPE, FRAGMENT_TYPE, \
+ parseMBOXInfo, parseNEWSInfo, parseRelayInfoByType, parseSMTPInfo, ParseError, \
ServerSideFragmentedMessage
from mixminion.ThreadUtils import RWLock, DummyLock
@@ -1659,7 +1659,7 @@
# What exit type names do we know about?
KNOWN_STRING_EXIT_TYPES = [
- "mbox", "smtp", "drop"
+ "mbox", "news", "smtp", "drop"
]
# Map from (type, nickname) to 1 for servers we've already warned about.
@@ -1802,6 +1802,18 @@
if exitKB > msec['Maximum-Size']:
raise UIError("Message to long for server %s to deliver."%
nickname)
+ elif self.exitType in ('news', NEWS_TYPE):
+ if not desc.has_section("Delivery/NEWS"):
+ raise UIError("Server %s doesn't support NEWS"%nickname)
+ nsec = desc['Delivery/NEWS']
+ if not nsec.get("Version"):
+ raise UIError("Server %s doesn't support NEWS"%nickname)
+ if needFrom and not nsec['Allow-From']:
+ raise UIError("Server %s doesn't support user-supplied From"%
+ nickname)
+ if exitKB > nsec['Maximum-Size']:
+ raise UIError("Message to long for server %s to deliver."%
+ nickname)
elif self.exitType in ('drop', DROP_TYPE):
# everybody supports 'drop'.
pass
@@ -1852,6 +1864,8 @@
rt = DROP_TYPE
elif self.exitType == 'mbox':
rt = MBOX_TYPE
+ elif self.exitType == 'news':
+ rt = NEWS_TYPE
else:
assert type(self.exitType) == types.IntType
rt = self.exitType
@@ -1908,6 +1922,9 @@
return ExitAddress('mbox', parseMBOXInfo(mbox).pack(), server)
else:
return ExitAddress('mbox', parseMBOXInfo(val).pack(), None)
+ elif tp == 'news':
+ # May raise ParseError
+ return ExitAddress('news', parseNEWSInfo(val).pack(), None)
elif tp == 'smtp':
# May raise ParseError
return ExitAddress('smtp', parseSMTPInfo(val).pack(), None)
Index: Packet.py
===================================================================
--- Packet.py (revision 2)
+++ Packet.py (working copy)
@@ -24,7 +24,7 @@
'SWAP_FWD_HOST_TYPE', 'SingletonPayload',
'Subheader', 'TAG_LEN', 'TextEncodedMessage',
'parseHeader', 'parseIPV4Info', 'parseMMTPHostInfo',
- 'parseMBOXInfo', 'parsePacket', 'parseMessageAndHeaders',
+ 'parseMBOXInfo', 'parseNEWSInfo', 'parsePacket', 'parseMessageAndHeaders',
'parsePayload', 'parseRelayInfoByType', 'parseReplyBlock',
'parseReplyBlocks', 'parseSMTPInfo', 'parseSubheader',
'parseTextEncodedMessages', 'parseTextReplyBlocks',
@@ -39,8 +39,8 @@
import zlib
from socket import inet_ntoa, inet_aton
from mixminion.Common import MixError, MixFatalError, encodeBase64, \
- floorDiv, formatBase64, formatTime, isSMTPMailbox, LOG, armorText, \
- unarmorText, isPlausibleHostname
+ floorDiv, formatBase64, formatTime, isSMTPMailbox, isNewsgroups, \
+ LOG, armorText, unarmorText, isPlausibleHostname
from mixminion.Crypto import sha1
if sys.version_info[:3] < (2,2,0):
@@ -728,6 +728,23 @@
"""Return the external representation of this routing info."""
return self.user
+def parseNEWSInfo(s):
+ """Convert the encoding of an News exitinfo into an NEWSInfo address"""
+ if not isNewsgroups(s):
+ raise ParseError("Invalid newsgroups %r" % s)
+ return NEWSInfo(s)
+
+class NEWSInfo:
+ """Represents the exitinfo for a NEWS hop.
+
+ Fields: user (a user identifier)"""
+ def __init__(self, newsgroups):
+ self.newsgroups = newsgroups
+
+ def pack(self):
+ """Return the external representation of this routing info."""
+ return self.newsgroups
+
#----------------------------------------------------------------------
# Ascii-encoded packets
#
Index: ClientAPI.py
===================================================================
--- ClientAPI.py (revision 2)
+++ ClientAPI.py (working copy)
@@ -663,8 +663,8 @@
class MsgDest(_Encodeable):
"""A MsgDest is the address for a mssage. It may be either a SURBMsgDest,
to indate a message that we will send with one or more SURBs, or an
- AddrMsgDest, to indicate an email address, mbox address, or other
- Type III exit address.
+ AddrMsgDest, to indicate an email address, mbox address, newsgroups,
+ or other Type III exit address.
"""
def __init__(self):
pass
@@ -678,6 +678,7 @@
of the format:
mbox:<mailboxname>@<server>
OR smtp:<email address>
+ OR news:<group1>,<group2>,<group3> (up to three groups)
OR <email address> (smtp is implicit)
OR drop
OR 0x<routing type>:<routing info>
Index: server/Modules.py
===================================================================
--- server/Modules.py (revision 2)
+++ server/Modules.py (working copy)
@@ -38,7 +38,7 @@
import mixminion.server.PacketHandler
from mixminion.Config import ConfigError
from mixminion.Common import LOG, MixError, ceilDiv, createPrivateDir, \
- encodeBase64, floorDiv, isPrintingAscii, isSMTPMailbox, previousMidnight,\
+ encodeBase64, floorDiv, isPrintingAscii, isSMTPMailbox, isNewsgroups, previousMidnight,\
readFile, waitForChildren
from mixminion.Packet import ParseError, CompressedDataTooLong, uncompressData
@@ -47,6 +47,9 @@
DELIVER_FAIL_RETRY = 2
DELIVER_FAIL_NORETRY = 3
+# Maximum number of newsgroups per message
+MAX_NEWSGROUPS = 3
+
class DeliveryModule:
"""Abstract base for modules; delivery modules should implement
the methods in this class.
@@ -318,6 +321,7 @@
self.queues = {}
self.registerModule(MBoxModule())
+ self.registerModule(NewsModule())
self.registerModule(DropModule())
self.registerModule(DirectSMTPModule())
self.registerModule(MixmasterSMTPModule())
@@ -574,7 +578,7 @@
# There are two concerns with making fragment config match up with
# the other delivery modules: first, we need to make sure that if
# we defragment, we have some way to reassemble fragmented messages.
- deliverySecs = [ 'Delivery/MBOX', 'Delivery/SMTP',
+ deliverySecs = [ 'Delivery/MBOX', 'Delivery/NEWS', 'Delivery/SMTP',
'Delivery/SMTP-Via-Mixmaster' ]
enabled = [ config.get(s,{}).get("Enabled") for s in deliverySecs ]
@@ -1000,6 +1004,15 @@
fromAddr = self.returnAddress
morelines = []
+ if mixminion.Packet.NEWS_TYPE in self.getExitTypes():
+ newsgroups = packet.getAddress()
+ if isNewsgroups(newsgroups):
+ if newsgroups.count(',') > MAX_NEWSGROUPS-1:
+ LOG.warn("Dropping excessive crossposting: %s" % newsgroups)
+ return None
+ morelines.append("Newsgroups: %s\n" % newsgroups)
+ else:
+ LOG.warn("Invalid newsgroups: %s" % newsgroups)
if headers.has_key("IN-REPLY-TO"):
morelines.append("In-Reply-To: %s\n" % headers['IN-REPLY-TO'])
if headers.has_key("REFERENCES"):
@@ -1042,6 +1055,122 @@
self.header = "".join(header)
#----------------------------------------------------------------------
+class NewsModule(DeliveryModule, MailBase):
+ """Implementation for newsgroup delivery: sends messages, via SMTP,
+ to a mail2news gateway, setting the newsgroups header.
+ """
+ ##
+ # Fields:
+ # gateway: mail2news gateway
+ # server: the name of our SMTP server
+ # returnAddress: the address we use in our 'From' line
+ # contact: the contact address we mention in our boilerplate
+ # nickname: our server nickname; for use in our boilerplate
+ # addr: our IP address, or "<Unknown IP>": for use in our boilerplate.
+ def __init__(self):
+ DeliveryModule.__init__(self)
+ self.maxMessageSize = None
+
+ def getRetrySchedule(self):
+ return self.retrySchedule
+
+ def getConfigSyntax(self):
+ # FFFF There should be some way to say that fields are required
+ # FFFF if the module is enabled.
+ cfg = { 'Enabled' : ('REQUIRE', "boolean", "no"),
+ 'Retry': ('ALLOW', "intervalList",
+ "7 hours for 6 days"),
+ 'Gateway' : ('ALLOW', None, None),
+ 'RemoveContact' : ('ALLOW', None, None),
+ 'SMTPServer' : ('ALLOW', None, None),
+ 'SendmailCommand' : ('ALLOW', "command", None),
+ 'Advertise' : ('ALLOW', "boolean", "yes")
+ }
+ cfg.update(MailBase.COMMON_OPTIONS)
+ return { "Delivery/NEWS" : cfg }
+
+ def validateConfig(self, config, lines, contents):
+ sec = config['Delivery/NEWS']
+ if not sec.get('Enabled'):
+ return
+ for field in ['Gateway', 'ReturnAddress', 'RemoveContact']:
+ if not sec.get(field):
+ raise ConfigError("Missing field %s in [Delivery/NEWS]"%field)
+ for field in ['ReturnAddress', 'RemoveContact']:
+ if not isSMTPMailbox(sec[field]):
+ LOG.warn("Value of %s (%s) doesn't look like an email address",
+ field, sec[field])
+ if (sec['SMTPServer'] is not None and
+ sec['SendmailCommand'] is not None):
+ raise ConfigError("Cannot specify both SMTPServer and SendmailCommand")
+
+ config.validateRetrySchedule("Delivery/NEWS")
+
+ def configure(self, config, moduleManager):
+ if not config['Delivery/NEWS'].get("Enabled", 0):
+ moduleManager.disableModule(self)
+ return
+
+ sec = config['Delivery/NEWS']
+ self.advertise = sec.get('Advertise') #DOCDOC
+ self.cfgSection = sec.copy() #DOCDOC
+ self.gateway = sec['Gateway']
+ self.returnAddress = sec['ReturnAddress']
+ self.contact = sec['RemoveContact']
+ self.retrySchedule = sec['Retry']
+ self.allowFromAddr = sec['AllowFromAddress']
+ # validate should have caught these.
+ assert (self.returnAddress and self.contact)
+
+ self.nickname = config['Server']['Nickname']
+ if not self.nickname:
+ self.nickname = socket.gethostname()
+ self.addr = config['Incoming/MMTP'].get('IP', "<Unknown IP>")
+ self.maxMessageSize = _cleanMaxSize(sec['MaximumSize'],
+ "Delivery/NEWS")
+
+ # These fields are needed by MailBase
+ self.initializeHeaders(sec)
+ self.fromTag = "[Anon]"
+
+ # Enable module
+ moduleManager.enableModule(self)
+
+ def getServerInfoBlock(self):
+ if not self.advertise:
+ return ""
+
+ if self.allowFromAddr:
+ allowFrom = "yes"
+ else:
+ allowFrom = "no"
+ return """\
+ [Delivery/NEWS]
+ Version: 0.1
+ Maximum-Size: %s
+ Allow-From: %s
+ """ % (floorDiv(self.maxMessageSize,1024), allowFrom)
+
+ def getName(self):
+ return "NEWS"
+
+ def getExitTypes(self):
+ return [ mixminion.Packet.NEWS_TYPE ]
+
+ def processMessage(self, packet): #message, tag, exitType, address):
+ # Determine that message's address;
+ assert packet.getExitType() == mixminion.Packet.NEWS_TYPE
+ LOG.debug("Received NEWS message")
+
+ # Generate the boilerplate (FFFF Make this more configurable)
+ msg = self._formatEmailMessage(self.gateway, packet)
+ if not msg:
+ return DELIVER_FAIL_NORETRY
+
+ # Deliver the message
+ return sendSMTPMessage(self.cfgSection, [self.gateway], self.returnAddress, msg)
+
+#----------------------------------------------------------------------
class MBoxModule(DeliveryModule, MailBase):
"""Implementation for MBOX delivery: sends messages, via SMTP, to
addresses from a local file. The file must have the format
Index: server/ServerMain.py
===================================================================
--- server/ServerMain.py (revision 2)
+++ server/ServerMain.py (working copy)
@@ -844,11 +844,12 @@
mixminion.server.Pinger.HEARTBEAT_INTERVAL))
# FFFF if we aren't using a LOCKING_IS_COURSE database, we will
# FFFF still want this to happen in another thread.
+ recomputeInterval = self.config['Pinging']['RecomputeInterval'].getSeconds();
self.scheduleEvent(RecurringEvent(
- now+60,
+ now+recomputeInterval,
lambda self=self: self.pingLog.calculateAll(
os.path.join(self.config.getWorkDir(), "pinger", "status")),
- self.config['Pinging']['RecomputeInterval'].getSeconds()))
+ recomputeInterval))
# Makes next update get scheduled.
nextUpdate = self.updateDirectoryClient(reschedulePings=0)
Index: server/ServerKeys.py
===================================================================
--- server/ServerKeys.py (revision 2)
+++ server/ServerKeys.py (working copy)
@@ -926,7 +926,7 @@
elif not config_im['Enabled'] and info_im.get('Version'):
warn("Incoming MMTP published but not enabled.")
- for section in ('Outgoing/MMTP', 'Delivery/MBOX', 'Delivery/SMTP'):
+ for section in ('Outgoing/MMTP', 'Delivery/MBOX', 'Delivery/NEWS', 'Delivery/SMTP'):
info_out = info[section].get('Version')
config_out = (config[section].get('Enabled') and
config[section].get('Advertise',1))
@@ -934,7 +934,7 @@
config_out = (config['Delivery/SMTP-Via-Mixmaster'].get("Enabled")
and config['Delivery/SMTP-Via-Mixmaster'].get("Advertise", 1))
if info_out and not config_out:
- warn("%s published, but not enabled.", section)
+ warn("%s published, but not enabled/advertised.", section)
if config_out and not info_out:
warn("%s enabled, but not published.", section)
Index: ServerInfo.py
===================================================================
--- ServerInfo.py (revision 2)
+++ ServerInfo.py (working copy)
@@ -157,6 +157,11 @@
"Maximum-Size": ("REQUIRE", "int", "32"),
"Allow-From": ("REQUIRE", "boolean", "yes"),
},
+ "Delivery/NEWS" : {
+ "Version": ("REQUIRE", None, None),
+ "Maximum-Size": ("REQUIRE", "int", "32"),
+ "Allow-From": ("REQUIRE", "boolean", "yes"),
+ },
"Delivery/SMTP" : {
"Version": ("REQUIRE", None, None),
"Maximum-Size": ("REQUIRE", "int", "32"),
@@ -182,6 +187,7 @@
"Outgoing/MMTP" : ("Version", "0.1"),
"Delivery/Fragmented" : ("Version", "0.1"),
"Delivery/MBOX" : ("Version", "0.1"),
+ "Delivery/NEWS" : ("Version", "0.1"),
"Delivery/SMTP" : ("Version", "0.1"),
}
@@ -432,6 +438,9 @@
return caps
if self['Delivery/MBOX'].get('Version'):
caps.append('mbox')
+ if self.has_section("Delivery/NEWS"):
+ if self['Delivery/NEWS'].get('Version'):
+ caps.append('news')
if self['Delivery/SMTP'].get('Version'):
caps.append('smtp')
# XXXX This next check is highly bogus.
Index: ClientMain.py
===================================================================
--- ClientMain.py (revision 2)
+++ ClientMain.py (working copy)
@@ -29,7 +29,7 @@
isSMTPMailbox, readFile, stringContains, succeedingMidnight, writeFile, \
previousMidnight
from mixminion.Packet import encodeMailHeaders, ParseError, parseMBOXInfo, \
- parseReplyBlocks, parseSMTPInfo, parseTextEncodedMessages, \
+ parseReplyBlocks, parseSMTPInfo, parseNEWSInfo, parseTextEncodedMessages, \
parseTextReplyBlocks, ReplyBlock, parseMessageAndHeaders, \
CompressedDataTooLong
===== Ende =====
Configuration template:
===== Beginn =====
[Delivery/NEWS]
Enabled: yes
Gateway: mail2news@xxxxxxxxx
ReturnAddress: <"From:" address to use>
RemoveContact: <Address to use as a contact>
#SendmailCommand: sendmail -i -t
SMTPServer: localhost
Retry: every 7 hours for 6 days
MaximumSize: 100K
AllowFromAddress: yes
===== Ende =====
Error when leaving out desc.has_section("Delivery/NEWS")
from ClientDirectory.py:
===== Beginn =====
$ mixminion send -v -t news:group
Mixminion version 0.0.8alpha3
This software is for testing purposes only. Anonymity is not guaranteed.
Nov 01 22:54:35 [INFO] Setting entropy source to '/dev/urandom'
Nov 01 22:54:35 [DEBUG] Configuring client
Nov 01 22:54:35 [DEBUG] Configuring server list
Nov 01 22:54:35 [DEBUG] Directory is up to date.
Nov 01 22:54:35 [WARN] This software is newer than any version on the recommended list.
Traceback (most recent call last):
File "/usr/bin/mixminion", line 9, in ?
mixminion.Main.main(sys.argv)
File "/usr/lib/python2.4/site-packages/mixminion/Main.py", line 335, in main
func(commandStr, args[2:])
File "/usr/lib/python2.4/site-packages/mixminion/ClientMain.py", line 1249, in runClient
parser.parsePath()
File "/usr/lib/python2.4/site-packages/mixminion/ClientMain.py", line 1060, in parsePath
self.startAt, self.endAt)
File "/usr/lib/python2.4/site-packages/mixminion/ClientDirectory.py", line 1414, in validatePath
warnUnrecommended)
File "/usr/lib/python2.4/site-packages/mixminion/ClientDirectory.py", line 1466, in _validatePath
if exitAddress.isSupportedByServer(desc):
File "/usr/lib/python2.4/site-packages/mixminion/ClientDirectory.py", line 1746, in isSupportedByServer
self.checkSupportedByServer(desc,verbose=0)
File "/usr/lib/python2.4/site-packages/mixminion/ClientDirectory.py", line 1808, in checkSupportedByServer
nsec = desc['Delivery/NEWS']
File "/usr/lib/python2.4/site-packages/mixminion/Config.py", line 932, in __getitem__
return self._sections[sec]
KeyError: 'Delivery/NEWS'
===== Ende =====
Error when leaving out desc.has_section("Delivery/NEWS")
from ServerInfo.py:
===== Beginn =====
$ mixminion list-server -v -F caps
Mixminion version 0.0.8alpha3
This software is for testing purposes only. Anonymity is not guaranteed.
Nov 01 22:56:21 [INFO] Setting entropy source to '/dev/urandom'
Nov 01 22:56:21 [DEBUG] Configuring server list
Nov 01 22:56:21 [DEBUG] Directory is up to date.
Nov 01 22:56:21 [WARN] This software is newer than any version on the recommended list.
Traceback (most recent call last):
File "/usr/bin/mixminion", line 9, in ?
mixminion.Main.main(sys.argv)
File "/usr/lib/python2.4/site-packages/mixminion/Main.py", line 335, in main
func(commandStr, args[2:])
File "/usr/lib/python2.4/site-packages/mixminion/ClientMain.py", line 1622, in listServers
featureMap = directory.getFeatureMap(features,goodOnly=goodOnly)
File "/usr/lib/python2.4/site-packages/mixminion/ClientDirectory.py", line 1059, in getFeatureMap
d[feature] = str(sd.getFeature(sec,ent))
File "/usr/lib/python2.4/site-packages/mixminion/ServerInfo.py", line 528, in getFeature
return " ".join(self.getCaps())
File "/usr/lib/python2.4/site-packages/mixminion/ServerInfo.py", line 442, in getCaps
if self['Delivery/NEWS'].get('Version'):
File "/usr/lib/python2.4/site-packages/mixminion/Config.py", line 932, in __getitem__
return self._sections[sec]
KeyError: 'Delivery/NEWS'
===== Ende =====
Ciao
Tobias
--
mbox:admin@tainaron
Attachment:
pgpLwZuQKEMQX.pgp
Description: PGP signature