#!/bin/python # Archive my newsserver into mbox files. # azz@us-lot.org import nntplib, os, pickle, time, string, sys, traceback, gzip mboxpath = "/home/azz/newsarchive" positionsfile = mboxpath + "/newsarchive.positions" interesting = "/var/spool/news/interesting.groups" server = "localhost" n = None prefix = "newsarchive" def setprefix(prefix): global logprefix logprefix = prefix def log(*msg): global logprefix print logprefix + ": " + " ".join(map(str, msg)) def show_exception(): log("Exception:", sys.exc_info()[0]) traceback.print_exc() try: f = open(positionsfile) positions = pickle.load(f) f.close() except: positions = {} nothing = [] try: n = nntplib.NNTP(server) for group in os.listdir(interesting): setprefix(group) try: (response, count, first, last, name) = n.group(group) except: log("Exception when getting group -- skipping") continue (count, first, last) = (int(count), int(first), int(last)) if positions.has_key(group): num = positions[group] if first > num: num = first while num < last: try: n.stat(str(num)) except nntplib.NNTPTemporaryError: # Article doesn't exist. num += 1 else: break statfirst = 0 else: log("New group") num = first statfirst = 1 positions[group] = first - 1 if num >= last: nothing.append(group) else: log("Archiving articles from", num, "to", last) f = gzip.open(mboxpath + "/" + group + ".gz", "ab") first = num while 1: try: if statfirst: statfirst = 0 (response, num, id) = n.stat(str(num)) else: (response, num, id) = n.next() num = int(num) (response, none, id, lines) = n.article(str(num)) except nntplib.NNTPTemporaryError: log("NNTP temporary error") break except: show_exception() break def escape_from(x): if x[:6] == "From ": return ">" + x else: return x text = string.join(map(escape_from, lines), "\n") + "\n" f.write("From newsserver@" + server + " " + time.ctime(time.time()) + "\n" + text + "\n") positions[group] = num if num == last: break f.close() except: show_exception() if nothing != []: setprefix("newsarchive") log("Fetched nothing from", len(nothing), "groups") f = open(positionsfile, "w") pickle.dump(positions, f) f.close()