MacAddict 108

home *** CD-ROM | disk | FTP | other *** search

/ MacAddict 108 / MacAddict108.iso / Software / Internet & Communication / JunkMatcher 1.5.5.dmg / JunkMatcher.app / Contents / Resources / MatcherProxy.py < prev next >

Wrap

Python Source | 2005-06-01 | 54.4 KB | 1,373 lines

# # MatcherProxy.py # JunkMatcher # # Created by Benjamin Han on 2/2/05. # Copyright (c) 2005 Benjamin Han. All rights reserved. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. import copy, cPickle, shutil from Foundation import * from AppKit import * from consts import * from Matcher import * from installFactoryPatterns import * # read-only EmailDB for GUI globalObjects.emailDBReadFlag = True PACKAGE_PLUGIN_PATH = '%sJunkMatcher.mailbundle/Contents/Resources/' % ROOT_PATH _receivedDatePat = re.compile(r'(\d+-\d\d-\d\d \d\d:\d\d:\d\d)(?:\.\d+)?') _ipPat = re.compile(r'[\d.]+') viewsList = [VIEW_SUBJECT, VIEW_SENDER, VIEW_HEADERS, VIEW_BODY, VIEW_FILENAMES, VIEW_CHARSETS, VIEW_RENDERING] viewsDict = dict(zip(viewsList, range(7))) logAttributeNames = ['receivedDate', 'k', 'sender', 'subject', 'junkFlag', 'matchResult'] null = NSNull.null() sys.path.insert(0, PACKAGE_PLUGIN_PATH) try: from pluginConsts import * pluginMissing = False except: # TO-DO: maybe warn users that the plugin is not installed properly? pluginMissing = True jmClientModule = None # to load absolutely minimal # of modules at startup! def _sendCommandToPlugin (cmd, arg = None): """Sends a command via a JMClient; returns the reply string if successful, otherwise returns False.""" global jmClientModule # TO-DO: why do I need this? if pluginMissing: return False if jmClientModule is None: jmClientModule = __import__('JMClient') jmc = jmClientModule.JMClient('%sserver.port' % TMP_PATH, block = False) if jmc.port != -1: return jmc.send(cmd, arg) return False class MatcherProxy (NSObject): def init (self): self = super(MatcherProxy, self).init() if self: self.matcher = Matcher(False) self.metaPatternsInitialized = False self.logReadPosition = 0 self.whitelistInitialized = False self.propertiesChanged = self.patternsChanged = self.metaPatternsChanged = self.testsChanged = False return self def setAttribute_withValue_ (self, name, value): """This is basically for Obj-C side of PyObjC bridge so we can set an instance variable""" setattr(self, name, value) def prepareFeedbackScript (self): jmVersion = NSBundle.mainBundle().objectForInfoDictionaryKey_(u'CFBundleVersion') sysInfo = 'JunkMatcher: %s; %s' % (jmVersion, __import__('sysInfo').sysInfo()) return open('%sfeedbackAppleScript.txt' % ROOT_PATH).read() % sysInfo.replace('"', '\\"') # ---------------------------------------------------------------------- # for the first run # ---------------------------------------------------------------------- def firstRun (self): # back the old JM stuff (1.19c or earlier) try: # remove the old JunkMatcher Central applescript os.remove('%sScripts/Mail Scripts/JunkMatcher Central___ctl-J.scpt' % HOMELIB_PATH) except: pass pluginPath = '%sMail/Bundles/JunkMatcher.mailbundle/' % HOMELIB_PATH oldConfPath = '%sContents/Resources/junkMatcher/conf/' % pluginPath if os.path.exists(oldConfPath): # configurations of 1.19c/earlier exist - migrating them NSLog(u'Backing up old JunkMatcher patterns.') try: __import__('backupOldSettings').backupOldSettings(oldConfPath, os.path.expanduser('~/Desktop/')) except Exception, e: printException('Exception when backing up old JM settings', e) # patch tests file since new versions could have a different set of properties self.matcher.tests.patchPropertiesAgainstDefaults() # these shouldn't be necessary cuz Mail.app should not be running as of now # but it doesn't hurt to do this _sendCommandToPlugin(JM_CMD_LOAD_PROPERTIES) _sendCommandToPlugin(JM_CMD_LOAD_TESTS) # ---------------------------------------------------------------------- # for installing the plugin # ---------------------------------------------------------------------- def enableMailBundleSupport_ (self, isTiger = True): os.system('defaults write com.apple.mail EnableBundles -bool YES') if isTiger: os.system('defaults write com.apple.mail BundleCompatibilityVersion -int 2') else: os.system('defaults write com.apple.mail BundleCompatibilityVersion -int 1') def installPlugin_ (self, isTiger = True): """Installs JunkMatcher.mailbundle; returns True iff the installation is successful.""" targetPath = '%sMail/Bundles' % HOMELIB_PATH # we asked users to quit Mail.app - now we'll nuke all JMServer processes by force pidSet = __import__('nukeJMServer').nukeJMServer() if pidSet: NSLog(u'JMServer.py processes killed: %s' % ', '.join(pidSet)) # create Library/Mail/Bundles if it doesn't exist if not os.path.exists(targetPath): try: os.makedirs(targetPath, 0755) except: return False # remove the old Library/Mail/Bundles/JunkMatcher.mailbundle if it's there targetPath += '/JunkMatcher.mailbundle' if os.path.exists(targetPath): try: NSLog(u'Trying to remove %s' % targetPath) shutil.rmtree(targetPath) except: return False # copy over the stock version of the plugin try: shutil.copytree('%sJunkMatcher.mailbundle' % ROOT_PATH, targetPath, True) except: return False # enable bundle support in Mail.app self.enableMailBundleSupport_(isTiger) return True # ---------------------------------------------------------------------- # for installing the Mail rules # ---------------------------------------------------------------------- def repairMailRules_usingTemplate_inTiger_ (self, mailRules, template, isTiger): """Repairs mailRules using template; returns True if mailRules is modified.""" return __import__('mailRules').repairMailRules(mailRules, template, isTiger) def isMailRunning (self): return len(os.popen('ps axjww | grep Mail\.app | grep ^%s | grep -v grep' % os.environ['USER']).read().strip()) != 0 # ---------------------------------------------------------------------- # for installing the factory version of patterns # ---------------------------------------------------------------------- def installFactoryPatterns (self): """Returns True if change did occur.""" ret = installFactoryPatterns(self.matcher.tests.patterns, globalObjects.metaPatterns, self.matcher.tests) if ret: self.patternsChanged = self.metaPatternsChanged = self.testsChanged = True NSLog(u'Factory versions of meta patterns and patterns are installed.') else: NSLog(u'The current meta patterns and patterns are identical to the factory versions.') return ret # ---------------------------------------------------------------------- # for messages # ---------------------------------------------------------------------- def setMessageFromFile_toDictionary_badTagArray_hiddenUrlArray_vacuousTagArray_ (self, msgFN, messageViewDictionary, badTagArray, hiddenUrlArray, vacuousTagArray): """Returns tuple (receivedDateString, htmlFlag); htmlFlag is '1' iff the message is an HTML-based one.""" self.msg = Message(open(msgFN).read()) return self._loadMessage(messageViewDictionary, badTagArray, hiddenUrlArray, vacuousTagArray) def setMessageFromKey_withReceivedDateString_toDictionary_badTagArray_hiddenUrlArray_vacuousTagArray_ (self, key, receivedDateString, messageViewDictionary, badTagArray, hiddenUrlArray, vacuousTagArray): """Returns tuple (receivedDateString, htmlFlag); htmlFlag is '1' iff the message is an HTML-based one.""" # remember I use a peculiar way to pass in the received date: as a prefix following with '@'! # this is so that I can pass as a single argument from Mail.app to the server if receivedDateString is not None: self.msg = Message('%s%s' % (encodeText(receivedDateString), globalObjects.emailDB.getEntry(key))) else: self.msg = Message(globalObjects.emailDB.getEntry(key)) return self._loadMessage(messageViewDictionary, badTagArray, hiddenUrlArray, vacuousTagArray) def reloadMessageToDictionary_badTagArray_hiddenUrlArray_vacuousTagArray_ (self, messageViewDictionary, badTagArray, hiddenUrlArray, vacuousTagArray): """Reload the message from self.msg. ASSUMPTION: self.msg must have been initialized (via either setMessageFromFile_toDictionary_badTagArray_hiddenUrlArray_vacuousTagArray_() or setMessageFromKey_withReceivedDateString_toDictionary_badTagArray_hiddenUrlArray_vacuousTagArray_().""" return self._loadMessage(messageViewDictionary, badTagArray, hiddenUrlArray, vacuousTagArray) def _loadMessage (self, messageViewDictionary, badTagArray, hiddenUrlArray, vacuousTagArray): messageViewDictionary.clear() del badTagArray[:] del hiddenUrlArray[:] del vacuousTagArray[:] # correct messages must have headers too! if self.msg and self.msg.m and len(self.msg.headers): if self.msg.isHTML: htmlSrc = self.msg.htmlBody.htmlSrc htmlBody = self.msg.htmlBody if len(htmlBody.badTagList): contentWithBadTags = self.msg.htmlBody.contentWithoutEntities badTagArray.extend(htmlBody.badTagList) else: contentWithBadTags = null if len(htmlBody.hiddenURLList): contentWithHiddenURLs = self.msg.htmlBody.contentWithoutEntities hiddenUrlArray.extend(htmlBody.hiddenURLList) else: contentWithHiddenURLs = null if len(htmlBody.vacuousTagList): contentWithVacuousTags = self.msg.htmlBody.contentWithoutBadTags vacuousTagArray.extend(htmlBody.vacuousTagList) else: contentWithVacuousTags = null else: htmlSrc = null contentWithBadTags = null contentWithHiddenURLs = null contentWithVacuousTags = null if self.msg.receivedDate: # get rid of the annoying trailing decimal seconds mo = _receivedDatePat.search(str(self.msg.receivedDate)) d = NSDate.dateWithString_(u'%s +0000' % mo.group(1)) receivedDateString = d.description() else: receivedDateString = u'(date unknown)' if self.msg.isHTML: htmlFlag = '1' else: htmlFlag = '0' messageViewDictionary['subject'] = self.msg.subject messageViewDictionary['sender'] = self.msg.sender messageViewDictionary['headers'] = self.msg.headers messageViewDictionary['body'] = self.msg.body messageViewDictionary['filenames'] = self.msg.filenames messageViewDictionary['charsets'] = self.msg.charsets messageViewDictionary['rendering'] = self.msg.rendering messageViewDictionary['recipients'] = u', '.join(self.msg.decodedRecipients) messageViewDictionary['htmlSrc'] = htmlSrc messageViewDictionary['contentWithBadTags'] = contentWithBadTags messageViewDictionary['contentWithHiddenURLs'] = contentWithHiddenURLs messageViewDictionary['contentWithVacuousTags'] = contentWithVacuousTags else: receivedDateString = None htmlFlag = None return (receivedDateString, htmlFlag) def indexOfView_ (self, view): return viewsDict[view] # ---------------------------------------------------------------------- # for tests (properties and patterns) # ---------------------------------------------------------------------- def _loadTestsToArray (self, tests): # initialize viewsInUse dictionaries for all patterns for p in self.matcher.tests.patterns.values(): if hasattr(p, 'viewsInUse'): p.viewsInUse.clear() else: p.viewsInUse = {} self.getStatsFromPlugin() # get the latest stats for test in self.matcher.tests: i = test.propertyOrPattern d = {} d['kind'] = test.isPattern d['onFlag'] = test.isOn d['hardFlag'] = test.isHard d['matched'] = False d['test'] = test # a Test object if test.isPattern: view = test.view viewIdx = viewsDict[view] d['htmlFlag'] = test.isHTML d['name'] = i.name d['content'] = i.origPattern d['view'] = viewIdx testRecord = i.testRecords.get(view) if not testRecord: # usually this shouldn't happen (inconsistency between patterns # and tests files) testRecord = TestRecord() i.testRecords[view] = testRecord d['pr'] = u'%.1f/%.1f' % (testRecord.precision() * 100.0, testRecord.recall() * 100.0) d['managed'] = i.isManaged i.viewsInUse[viewIdx] = d else: d['name'] = u'Property: %s' % i.name d['content'] = i.contentString() d['pr'] = u'%.1f/%.1f' % (i.testRecord.precision() * 100.0, i.testRecord.recall() * 100.0) d['lastMatchAll'] = u'(N/A)' d['managed'] = True tests.append(d) tests[0]['first'] = True # mark the first property def loadTests (self): """Returns a list for showing all tests in an NSTableView. ASSUMPTION: will be called only ONCE (in the beginning).""" # DON'T TRY TO MERGE loadTests() and reloadTestsToArray_() - I've tried and # drag-n-drop crashed like hell! The difference is that 'ret' is a Python list, # and if you do it just by reloadTestsToArray_(), you need to use an NSMutableArray. ret = [] self._loadTestsToArray(ret) return ret def reloadTestsToArray_ (self, tests): """Load self.matcher.tests into 'tests' for showing the tests in an NSTableView.""" del tests[:] self._loadTestsToArray(tests) def findPatternWithID_andView_inTestArray_ (self, idStr, view, testArray): viewIdx = viewsDict[view] for idx, test in enumerate(testArray): if test['kind'] \ and test['content']== idStr \ and test['view'] == viewIdx: return idx return -1 def updateStatsInTests_ (self, tests): for d in tests: test = d['test'] i = test.propertyOrPattern if test.isPattern: view = test.view d['pr'] = u'%.1f/%.1f' % (i.testRecords[view].precision() * 100.0, i.testRecords[view].recall() * 100.0) else: d['pr'] = u'%.1f/%.1f' % (i.testRecord.precision() * 100.0, i.testRecord.recall() * 100.0) def getStatsFromPlugin (self): """Update the statistics with the fresh stats obtained from the running plugin (if any); returns True if update did occur.""" f1 = self._getPropertiesStatsFromPlugin() f2 = self._getPatternsStatsFromPlugin() return (f1 or f2) def _getPropertiesStatsFromPlugin (self): statsString = _sendCommandToPlugin(JM_CMD_GET_STATS_PROPERTIES) if statsString is not False: self.matcher.tests.properties.setStats(statsString) return True return False def _getPatternsStatsFromPlugin (self): statsString = _sendCommandToPlugin(JM_CMD_GET_STATS_PATTERNS) if statsString is not False: self.matcher.tests.patterns.setStats(statsString) return True return False def _getSiteDBFromPlugin (self): siteDBString = _sendCommandToPlugin(JM_CMD_GET_SITEDB) if siteDBString is not False: globalObjects.siteDB.loadFromString(siteDBString) return True return False def checkPattern_noMetaPattern_ (self, pat, noMetaPattern): """Return None if everything is fine; otherwise returns a string explaining the error occurring in pat.""" if pat.find('\n') != -1: return u'Your pattern contains a newline (\'\\n\') character.' if noMetaPattern: try: p = re.compile(pat) except: return u'Syntax error in the regular expression.' if mpPat.search(pat) is not None: return u'You cannot use meta patterns here to define a pattern.' else: try: pat = Pattern(None, None, pat) # a pattern with no name and no TestRecord p = pat.pattern except Exception, e: if isinstance(e, JMExceptionMetaPattern): return u'Meta pattern "%s" is not defined.' % e.info return u'Syntax error in the regular expression.' if p.search('') is not None: return u'Regular expression you entered matches empty strings.' return None def matchPattern_withText_ (self, pat, txt): """Returns a list of spans (i.e., tuples (start index, end index)).""" return Pattern(None, None, pat).runWithText(txt) def matchAll_ (self, onlyPatterns): if onlyPatterns: self.matcher.setMode(MATCHER_MODE_PATTERNS) else: self.matcher.setMode(MATCHER_MODE_ALL) matchResult = self.matcher.run(self.msg) retList = [] if type(matchResult.verdict) is bool: # otherwise it's whitelisted for m in filter(lambda m:m.isPositive, matchResult): if hasattr(m, 'info'): info = m.info else: info = None if hasattr(m, 'view'): view = m.view else: view = None retList.append((m.testIdx, info, view)) return (matchResult.verdict, retList) def canAddPattern_ (self, pattern): """Returns True if the name is ok to add.""" return self.matcher.tests.patterns.get(pattern) is None def addPattern_withName_forView_withTableContent_ (self, pattern, name, view, d): """Add a new pattern for a specified view; returns the newly added test (the pattern).""" # create a new pattern p = Pattern(name, {view:TestRecord()}, pattern) p.viewsInUse = {viewsDict[view]:d} # only one view is in use for now self.matcher.tests.patterns[pattern] = p # update tests test = Test(p, [view, u'_S', '1']) self.matcher.tests.append(test) # update metaPatterns in p and relevant entries in the meta pattern table if self.metaPatternsInitialized: mps = globalObjects.metaPatterns p.metaPatterns = mps.findAll(pattern) for mpName in p.metaPatterns: mps[mpName][-1][p] = null self.patternsChanged = True self.testsChanged = True return test def updatePatternObject_withPattern_ (self, p, pattern): p.changePattern(pattern) p.isManaged = False # any update makes a pattern "user" (non-managed) # update metaPatterns in p and relevant entries in the meta pattern table if self.metaPatternsInitialized: mps = globalObjects.metaPatterns # for all of the old meta patterns, remove p from their corresponding entries # in the meta pattern table for mpName in p.metaPatterns: del mps[mpName][-1][p] # for all of the new meta patterns, add p to their corresponding entries # in the meta pattern table p.metaPatterns = mps.findAll(pattern) for mpName in p.metaPatterns: mps[mpName][-1][p] = null self.patternsChanged = True self.testsChanged = True # cuz we use the regular expressions as the IDs of these patterns def removeTest_ (self, test): self.matcher.tests.remove(test) p = test.propertyOrPattern # it has to be a pattern del p.viewsInUse[viewsDict[test.view]] del p.testRecords[test.view] if len(p.viewsInUse) == 0: # p is no longer used in any test - delete it # first update p's records in all the meta patterns it uses, if necessary if self.metaPatternsInitialized: mps = globalObjects.metaPatterns # for all of the meta patterns p uses, remove p from their corresponding entries # in the meta pattern table for mpName in p.metaPatterns: del mps[mpName][-1][p] # IMPORTANT: this has to come after the metaPatterns update # cuz this can potentially delete a Pattern instance, which is used as # keys in mps (see the line using mps[mpName][-1][p]) del self.matcher.tests.patterns[p.origPattern] self.patternsChanged = True self.testsChanged = True def spawnRow_ (self, row): """Set up the views and other details for a spawned test. ASSUMPTIONS: (1) it's a pattern; (2) at least an unused view exists""" test = row['test'] p = test.propertyOrPattern # find the first available view viewsInUse = p.viewsInUse for viewIdx in range(7): if viewsInUse.get(viewIdx) is None: break view = viewsList[viewIdx] # create a new TestRecord for the view p.testRecords[view] = TestRecord() newTest = copy.copy(test) # this won't copy test.propertyOrPattern newTest.view = view self.matcher.tests.append(newTest) newRow = dict(row) newRow['matched'] = False newRow['test'] = newTest newRow['view'] = viewIdx newRow['pr'] = u'0.0/0.0' viewsInUse[viewIdx] = newRow self.patternsChanged = True self.testsChanged = True return newRow def moveTestFromIndex_toIndex_ (self, fromIdx, toIdx): """Move a test at fromIdx to toIdx.""" tests = self.matcher.tests test = tests[fromIdx] del tests[fromIdx] tests.insert(toIdx, test) self.testsChanged = True def changeTestRow_toViewIdx_ (self, row, viewIdx): test = row.valueForKey_(u'test') pattern = test.propertyOrPattern viewsInUse = pattern.viewsInUse oldViewIdx = row.valueForKey_(u'view') newView = viewsList[viewIdx] thisRow = viewsInUse[oldViewIdx] # remember in the viewsInUse dictionary the values are "rows" initialized at Python side # change the TestRecords of the pattern testRecords = pattern.testRecords del testRecords[viewsList[oldViewIdx]] testRecords[newView] = TestRecord() # change the attributes of the test test.view = newView # remove the old view and add the new view for GUI del viewsInUse[oldViewIdx] viewsInUse[viewIdx] = thisRow # we can't add 'row' here cuz that's an _NSControllerObjectProxy from objc # change the data shown in the selected row row.setValue_forKey_(viewIdx, u'view') row.setValue_forKey_(u'0.0/0.0', u'pr') self.patternsChanged = True self.testsChanged = True def saveTests (self): if self.propertiesChanged: self.propertiesChanged = False self._getPropertiesStatsFromPlugin() # get the latest stats self.matcher.tests.properties.writeToFile() _sendCommandToPlugin(JM_CMD_LOAD_PROPERTIES) if self.metaPatternsChanged: self.metaPatternsChanged = False self.patternsChanged = True # force to reload patterns globalObjects.metaPatterns.writeToFile() _sendCommandToPlugin(JM_CMD_LOAD_META) if self.patternsChanged: self.patternsChanged = False self._getPatternsStatsFromPlugin() # get the latest stats self.matcher.tests.patterns.writeToFile() _sendCommandToPlugin(JM_CMD_LOAD_PATTERNS) if self.testsChanged: self.testsChanged = False self.matcher.tests.writeToFile() _sendCommandToPlugin(JM_CMD_LOAD_TESTS) # ---------------------------------------------------------------------- # for test inspector # ---------------------------------------------------------------------- def getPropertySettingsFromRow_ (self, testRow): r = testRow.valueForKey_(u'test').propertyOrPattern testRecord = r.testRecord if r.recipientPattern is None: rPat = None else: rPat = r.recipientPattern.pattern return {'name': r.name, 'idStr': r.__class__.__name__, 'rPat': rPat, 'time': testRecord.readTime(), 'numbers': u'%d/%d; %d/%d' % testRecord.readNumbers(), 'pr': u'%.3f/%.3f' % (testRecord.precision() * 100.0, testRecord.recall() * 100.0)} def getPatternSettingsFromRow_toArray_ (self, testRow, patternStats): p = testRow.valueForKey_(u'test').propertyOrPattern testRecords = p.testRecords if p.recipientPattern is None: rPat = None else: rPat = p.recipientPattern.pattern if p.encodingPattern is None: ePat = None else: ePat = p.encodingPattern.pattern del patternStats[:] for view in viewsList: testRecord = testRecords.get(view) if testRecord: n1, n2, n3, n4 = testRecord.readNumbers() patternStats.append({'view': view, 'time': testRecord.readTime(), 'n1': n1, 'n2': n2, 'n3': n3, 'n4': n4}) return {'name': p.name, 'idStr': p.origPattern, 'rPat': rPat, 'ePat': ePat, 'pat': p.origPattern} def updatePatternObject_withName_ (self, p, name): p.name = name p.isManaged = False # any update makes a pattern "user" (non-managed) self.patternsChanged = True def updatePatternObject_withAuxPattern_forKind_ (self, p, pat, kind): if kind == 'rPat': p.changeRecipientPattern(pat) else: p.changeEncodingPattern(pat) self.patternsChanged = True def updatePropertyObject_withRecipientPattern_ (self, r, pat): r.changeRecipientPattern(pat) self.propertiesChanged = True def getPropertyWithName_ (self, name): return self.matcher.tests.properties[name] def setCheckPhishingURLInWhitelistedEmail_ (self, flag): self.matcher.tests.properties[u'PropertyPhishingURL'].checkWhitelistedEmail = flag self.propertiesChanged = True # ---------------------------------------------------------------------- # for meta patterns # ---------------------------------------------------------------------- def loadMetaPatternsToArray_ (self, metaPatterns): """Load meta patterns into NSMutableArray tests for showing them in an NSTableView. NOTE: this also sets up Pattern.metaPatterns (a set of names of the meta patterns used in a Pattern).""" mps = globalObjects.metaPatterns keys = mps.keys() keys.sort() if self.metaPatternsInitialized: # this is a reload metaPatterns.removeAllObjects() for name in keys: d = mps[name][-1] if type(d) is dict: # clear the additional dictionary we added in the last load mps[name][-1].clear() else: # we haven't added a trailing dict (this is probably a new meta pattern) mps[name] = mps[name] + ({}, ) else: self.metaPatternsInitialized = True # add an additional dictionary to each meta pattern tuple (for recording which patterns # use this meta pattern) # TO-DO: I use dictionaries instead of sets here because PyObjC doesn't bridge # sets.Set yet. for name in keys: mps[name] = mps[name] + ({}, ) # update attribute 'metaPatterns' in each Pattern, and the dict in each meta pattern # tuple (recording what patterns use this meta pattern) for p in self.matcher.tests.patterns.values(): p.metaPatterns = mps.findAll(p.origPattern) for mpName in p.metaPatterns: mps[mpName][-1][p] = null for name in keys: mp = mps[name] # n flag is True iff it can be modified (i.e., it's *not* a reserved meta pattern) metaPatterns.addObject_({'name':name, 'pat':mp[0], 'n':not mp[1], 'managed':mp[2], 'inUse':mp[-1]}) def canAddMetaPatternName_ (self, metaPatternName): """Returns True if the name is ok to add.""" return globalObjects.metaPatterns.get(metaPatternName) is None def addMetaPatternName_pattern_ (self, metaPatternName, pat): inUseDict = {} # not a reserved meta pattern, not a managed meta pattern globalObjects.metaPatterns[metaPatternName] = (pat, False, False, inUseDict) self.metaPatternsChanged = True return inUseDict def updateMetaPatternName_pattern_ (self, metaPatternName, pat): mps = globalObjects.metaPatterns mp = mps[metaPatternName] mpList = list(mp) mpList[0] = pat mpList[2] = False # an updated meta pattern becomes a managed (non-user) meta pattern mps[metaPatternName] = tuple(mpList) # we need to reinstantiate patterns that used the changed meta pattern for p in mp[-1].keys(): p.pattern = re.compile(mps.instantiate(p.origPattern)) self.metaPatternsChanged = True def updateMetaPatternName_withUserFlag_ (self, metaPatternName, flag): mps = globalObjects.metaPatterns mp = mps[metaPatternName] mpList = list(mp) mpList[2] = not bool(flag) mps[metaPatternName] = tuple(mpList) self.metaPatternsChanged = True def removeMetaPatternName_ (self, metaPatternName): del globalObjects.metaPatterns[metaPatternName] self.metaPatternsChanged = True # ---------------------------------------------------------------------- # for log # ---------------------------------------------------------------------- def loadLogToArray_ (self, logArray): """Read in all new log entries and apppend logArray with them.""" try: # log file can be missing logFN = '%sjm.log' % CONF_PATH logSize = os.path.getsize(logFN) logF = open(logFN) except: del logArray[:] return if logSize >= self.logReadPosition: logF.seek(self.logReadPosition) else: # reset to the start self.logReadPosition = 0 del logArray[:] logF.seek(0) logStr = logF.read() properties = self.matcher.tests.properties patterns = self.matcher.tests.patterns maxIdx = len(logStr) if maxIdx != 0: # re-open the emailDB cuz new entries have been added globalObjects.emailDB.reOpen() else: if self.logReadPosition == 0: del logArray[:] return # rIdx continues from where we left - BUT remember the latest entry comes # at the top, so it's really an "reversed" index (realIdx = len(logArray) - rIdx - 1) rIdx = len(logArray) sList = [] cList = [] idx = 0 while idx < maxIdx: idx2 = logStr.find('\n', idx) entryLen = int(logStr[idx:idx2]) idx2 += 1 idx = idx2 + entryLen + 1 # '+ 1' is for the trailing \n d = dict(zip(logAttributeNames, cPickle.loads(logStr[idx2:idx - 1]))) d['receivedDate'] = NSDate.dateWithString_(u'%s +0000' % d['receivedDate']) junkFlag = d['junkFlag'] # produce details and comments from matchResult if type(junkFlag) is bool: numPatterns = 0 for m in filter(lambda m: m.isPositive, d['matchResult']): if hasattr(m, 'info'): if m.isProperty: name = properties[m.idStr].name if hasattr(m, 'info'): try: sList.append('- %s: %s.' % (name, m.info)) except: sList.append('- %s.' % name) else: sList.append('- %s.' % name) cList.append(name) else: p = patterns.get(m.idStr) if p is None: sList.append('- Unknown pattern ("%s") matched in view "%s".' % (m.idStr, m.view)) else: sList.append('- Pattern "%s" matched in view "%s".' % (p.name, m.view)) numPatterns += 1 else: name = properties[m.idStr].name sList.append('- %s.' % name) cList.append(name) d['details'] = (d['sender'], d['subject'], '\n'.join(sList), d['k']) if len(cList): if numPatterns: d['comments'] = '%s and %d pattern(s)' % (', '.join(cList), numPatterns) else: d['comments'] = ', '.join(cList) else: if numPatterns: d['comments'] = '%d pattern(s)' % numPatterns else: # junkFlag is a string, which means the message is whitelisted d['details'] = (d['sender'], d['subject'], u'- Whitelisted using email address pattern "%s".' % junkFlag, d['k']) d['comments'] = u'Whitelisted' d['junkFlag'] = False d['cFlag'] = False d['rIdx'] = rIdx rIdx += 1 del sList[:] del cList[:] logArray.insert(0, d) self.logReadPosition += maxIdx def loadCorrections_intoLog_ (self, corrections, log): """Load corrections (indices w.r.t. the log array/list). ASSUMPTION: ONLY call this after loadLogToArray_(), and you only need to call this once (the first time the log is loaded).""" # corrections is an NSMutableSet, log is an NSMutableArray try: f = open('%scorrections' % CONF_PATH) except: corrections.removeAllObjects() return # remember the rIdx is "reversed" (see comment in loadLogToArray_()) logLen = len(log) for rIdx in map(lambda l: int(l), filter(lambda l: len(l), f.read().split(' '))): corrections.addObject_(rIdx) entry = log[logLen - rIdx - 1] entry['cFlag'] = True entry['junkFlag'] = not entry['junkFlag'] # remember it's been corrected! def recycleLog (self): if not _sendCommandToPlugin(JM_CMD_RECYCLE): # server is not running - recycle the log ourselves self.matcher.recycleLog() self.logReadPosition = 0 def setCorrectionForEntry_withCorrectionsSet_ (self, entry, corrections): """Correct (cFlag is True) or remove correction on a message with key 'k', so that the target verdict equals to junkFlag, and dump the changed corrections into its file. ASSUMPTION: Corrections is already updated before this being called. INVARIANCE: the statistics of properties/patterns, siteDB and whitelist of the GUI and the Plugin (if it's running at the time) should be all synched after this call.""" open('%scorrections' % CONF_PATH, 'w').write(' '.join(map(str, corrections.allObjects()))) k = entry.valueForKey_('k') receivedDate = entry.valueForKey_('receivedDate') msg = Message('%s%s' % (encodeText(receivedDate.description()), globalObjects.emailDB.getEntry(k))) if msg.m is None: # message is malformed - nothing can be done return cFlag = entry.valueForKey_('cFlag') junkFlag = entry.valueForKey_('junkFlag') matchResult = entry.valueForKey_('matchResult') properties = self.matcher.tests.properties patterns = self.matcher.tests.patterns if cFlag: # set correction if junkFlag: # change to junk if msg.addSites(): globalObjects.siteDB.writeToFile() _sendCommandToPlugin(JM_CMD_LOAD_SITEDB) else: # change to clean if msg.senderEmail: # whitelist the sender - no check on possible duplicates here! globalObjects.whitelist.add(u'Automatically added %s' % NSCalendarDate.calendarDate(), '%s' % re.escape(msg.senderEmail), True) _sendCommandToPlugin(JM_CMD_WHITELIST) if msg.removeSites(False): globalObjects.siteDB.writeToFile() _sendCommandToPlugin(JM_CMD_LOAD_SITEDB) else: # remove correction if junkFlag: # change to junk if msg.senderEmail: # try to remove the sender from the whitelist # TO-DO: easy but slow implementation! addr = msg.senderEmail removeList = [] for idx, d in enumerate(globalObjects.whitelist.theList): if d['managed']: if re.search(d['pat'], addr): removeList.append(idx) if removeList: globalObjects.whitelist.removeMany(removeList) _sendCommandToPlugin(JM_CMD_WHITELIST) if msg.addSites(): globalObjects.siteDB.writeToFile() _sendCommandToPlugin(JM_CMD_LOAD_SITEDB) else: # change to clean if msg.removeSites(False): globalObjects.siteDB.writeToFile() _sendCommandToPlugin(JM_CMD_LOAD_SITEDB) # update the statistics self.getStatsFromPlugin() for m in matchResult: if m.isPositive != junkFlag: claimed = m.isPositive if m.isProperty: properties[m.idStr].testRecord.changeToFalse(claimed) else: p = patterns.get(m.idStr) # patterns can be already gone by now if p: p.testRecords[m.view].changeToFalse(claimed) else: if m.isProperty: properties[m.idStr].testRecord.changeToTrue(junkFlag) else: p = patterns.get(m.idStr) # patterns can be already gone by now if p: p.testRecords[m.view].changeToTrue(junkFlag) # send the updated stats to a live plugin, if any pluginAWOL = (_sendCommandToPlugin(JM_CMD_SET_STATS_PROPERTIES, properties.getStatsString()) is False) pluginAWOL &= (_sendCommandToPlugin(JM_CMD_SET_STATS_PATTERNS, patterns.getStatsString()) is False) if pluginAWOL: properties.writeToFile() patterns.writeToFile() def getRecycleDays (self): return globalObjects.prefs.recycleDays def setRecycleDays_ (self, numDays): """Returns True iff recycling did happen.""" prefs = globalObjects.prefs if prefs.recycleDays != numDays: prefs.recycleDays = numDays prefs.writeToFile() reply = _sendCommandToPlugin(JM_CMD_PREFS_RECYCLEDAYS, str(numDays)) # note if the plugin is not running we call recycleLogWhenItsDue() ourselves if (reply is False and self.matcher.recycleLogWhenItsDue()) or\ reply == 'RECYCLED': # recycling did happen self.logReadPosition = 0 return True return False # ---------------------------------------------------------------------- # for whitelist # ---------------------------------------------------------------------- def loadWhitelist_ (self, patterns): del patterns[:] if self.whitelistInitialized: globalObjects.whitelist.load() patterns.extend(globalObjects.whitelist.theList) else: self.whitelistInitialized = True patterns.extend(globalObjects.whitelist.theList) for idx, d in enumerate(patterns): d['idx'] = idx def addPattern_toWhitelistWithName_ (self, pat, name): globalObjects.whitelist.add(name, pat) _sendCommandToPlugin(JM_CMD_WHITELIST) def updatePatternOfIndex_toWhitelistwithPattern_andName_ (self, idx, pat, name): globalObjects.whitelist.update(idx, name, pat) _sendCommandToPlugin(JM_CMD_WHITELIST) def removePatternInWhitelistWithIndex_ (self, idx): globalObjects.whitelist.remove(idx) _sendCommandToPlugin(JM_CMD_WHITELIST) # ---------------------------------------------------------------------- # for recipient patterns (very similar to those for the whitelist) # ---------------------------------------------------------------------- def loadRecipientPatterns (self): # ASSUMPTION: only called ONCE patterns = [] patterns.extend(globalObjects.recipientPatterns.theList) for idx, d in enumerate(patterns): d['idx'] = idx return patterns def addPattern_toRecipientPatternsWithName_ (self, pat, name): globalObjects.recipientPatterns.add(name, pat) _sendCommandToPlugin(JM_CMD_RECIPIENTPATTERNS) def updatePatternOfIndex_toRecipientPatternswithPattern_andName_ (self, idx, pat, name): globalObjects.recipientPatterns.update(idx, name, pat) _sendCommandToPlugin(JM_CMD_RECIPIENTPATTERNS) def removePatternInRecipientPatternsWithIndex_ (self, idx): globalObjects.recipientPatterns.remove(idx) _sendCommandToPlugin(JM_CMD_RECIPIENTPATTERNS) def getMyEmailAddressPatterns (self): """Returns a list of dictionary: keys are names and values are regex of the user's email addresses""" myEmailPat = re.compile(r'myEmail(\d+)') ret = [] for name, pat in __import__('MetaPatterns').getReservedMPs(): mo = myEmailPat.match(name) if mo: ret.append({'name': 'My Email %s' % mo.group(1), 'pat':pat, 'managed':True}) globalObjects.recipientPatterns.addMany(ret) _sendCommandToPlugin(JM_CMD_RECIPIENTPATTERNS) return ret # ---------------------------------------------------------------------- # for safe IPs (very similar to those for the recipient patterns) # ---------------------------------------------------------------------- def loadSafeIPs (self): # ASSUMPTION: only called ONCE patterns = [] patterns.extend(globalObjects.safeIPs.theList) for idx, d in enumerate(patterns): d['idx'] = idx return patterns def addPattern_toSafeIPsWithName_ (self, pat, name): globalObjects.safeIPs.add(name, pat) _sendCommandToPlugin(JM_CMD_SAFEIPS) def updatePatternOfIndex_toSafeIPswithPattern_andName_ (self, idx, pat, name): globalObjects.safeIPs.update(idx, name, pat) _sendCommandToPlugin(JM_CMD_SAFEIPS) def removePatternInSafeIPsWithIndex_ (self, idx): globalObjects.safeIPs.remove(idx) _sendCommandToPlugin(JM_CMD_SAFEIPS) # ---------------------------------------------------------------------- # for sites # ---------------------------------------------------------------------- def loadBadSites_ (self, badSites): """Load siteDB into badSites (list of dict).""" # get the latest collection of bad sites self._getSiteDBFromPlugin() globalObjects.siteDB.loadIntoList(badSites) for s in badSites: s['time'] = NSDate.dateWithString_(u'%s +0000' % s['time']) def getSiteDBLimit (self): return globalObjects.prefs.siteLimit def setSiteDBLimit_ (self, siteLimit): """Returns True iff siteDB gets pruned.""" prefs = globalObjects.prefs if prefs.siteLimit != siteLimit: prefs.siteLimit = siteLimit prefs.writeToFile() reply = _sendCommandToPlugin(JM_CMD_PREFS_SITEDBLIMIT, str(siteLimit)) if reply is False: return globalObjects.siteDB.setSizeLimit(siteLimit) else: # plugin is running! self._getSiteDBFromPlugin() # get the latest collection of bad sites return (reply == JM_REPLY_PRUNED) def canAddBadSite_ (self, site): mo = _ipPat.match(site) if mo and mo.end(0) == len(site): site = site.split('.') if len(site) != 4: return False # numerical needs 4 components else: # don't add safe site! if globalObjects.safeSitesPattern.search(site): return False site = site.split('.') if len(site) < 2: return False # domainn names need at least 2 components l, n, b = globalObjects.siteDB.getOne(site) return l is not None and len(l) # don't add existing or bogus sites! def addBadSite_withTime_ (self, badSite, timeSec): """Adds a bad site into the siteDB; returns 1 if a positive change happened, 0 for no change, and -1 for negative change (pruning happened).""" # get the latest collection of bad sites self._getSiteDBFromPlugin() siteDB = globalObjects.siteDB oldCount = siteDB.size() siteDB.addOne(badSite.split('.'), timeSec) newCount = siteDB.size() if oldCount < newCount: ret = 1 elif oldCount == newCount: ret = 0 else: ret = -1 if ret != 0: if _sendCommandToPlugin(JM_CMD_SET_SITEDB, str(siteDB)) is False: # plugin is not running: save the file ourselves siteDB.writeToFile() return ret def removeBadSite_ (self, badSite): # get the latest collection of bad sites self._getSiteDBFromPlugin() siteDB = globalObjects.siteDB oldCount = siteDB.size() siteDB.removeOne(badSite.split('.')) changeHappened = (oldCount != siteDB.size()) if changeHappened: if _sendCommandToPlugin(JM_CMD_SET_SITEDB, str(siteDB)) is False: # plugin is not running: save the file ourselves siteDB.writeToFile() return changeHappened def doWhoisOn_ (self, site): try: whoisInPipe, whoisOutPipe = os.popen4(encodeText('whois %s' % site)) return decodeText(whoisOutPipe.read()) except: return u'Error querying the Whois database (missing whois executable or Internet connection problem?).' def convertBadSiteToSafeSite_ (self, s): return u'(?:^|\.)%s$' % re.escape(s.strip()) # ---------------------------------------------------------------------- # for safe sites # ---------------------------------------------------------------------- def loadSafeSites (self): """Returns a list of safe sites suitable for displaying in a NSTableView. ASSUMPTION: should be only called ONCE.""" safeSiteLines = filter(lambda l: len(l), openFile('%ssafeSites' % CONF_PATH).read().split('\n')) safeSiteLines.sort() self.safeSites = sets.Set(safeSiteLines) # for later removal operations return map(lambda l: {'pat':l}, safeSiteLines) def canAddSafeSite_ (self, safeSite): """Returns True iff safeSite doesn't already exist. ASSUMPTION: loadSafeSites() should've been called earlier than this.""" return safeSite not in self.safeSites def addSafeSite_ (self, safeSite): self.safeSites.add(safeSite) openFile('%ssafeSites' % CONF_PATH, 'w').write('\n'.join(self.safeSites)) _sendCommandToPlugin(JM_CMD_SAFESITES) def removeSafeSite_ (self, safeSite): self.safeSites.discard(safeSite) openFile('%ssafeSites' % CONF_PATH, 'w').write('\n'.join(self.safeSites)) _sendCommandToPlugin(JM_CMD_SAFESITES) # ---------------------------------------------------------------------- # for tags window # ---------------------------------------------------------------------- def loadTags (self): htmlTags = globalObjects.htmlTags tags = htmlTags.keys() tags.sort() ret = [] for tag in tags: attrs = htmlTags[tag] if attrs is None: attrs = [] elif attrs == '*': attrs = ['*'] else: attrs = list(attrs) attrs.sort() ret.append({'tag':tag, 'attrs':' '.join(attrs)}) return ret def canAddTag_ (self, tag): return globalObjects.htmlTags.get(tag) is None def checkHTMLAttributes_ (self, attrs): attrs = attrs.strip().split(' ') try: attrs.index('*') return len(attrs) == 1 except: return True def setTag_withAttributes_ (self, tag, attrs): htmlTags = globalObjects.htmlTags htmlTags.setTag(tag.lower(), attrs.lower()) htmlTags.writeToFile() _sendCommandToPlugin(JM_CMD_HTMLTAGS) def removeTag_ (self, tag): htmlTags = globalObjects.htmlTags htmlTags.removeTag(tag.lower()) htmlTags.writeToFile() _sendCommandToPlugin(JM_CMD_HTMLTAGS) # ---------------------------------------------------------------------- # for preferences window # ---------------------------------------------------------------------- def getMatchingModeAndArguments (self): prefs = globalObjects.prefs return (prefs.mode, prefs.modeArgs) def setMatchingMode_andArguments_ (self, mode, args): """Set the matching mode (int) and args (a list of string).""" prefs = globalObjects.prefs if prefs.mode != mode or prefs.modeArgs != args: prefs.mode = mode prefs.modeArgs = args prefs.writeToFile() # NOTE: this has no effect to the matcher in the GUI it's only to set # the matcher in the plugin _sendCommandToPlugin(JM_CMD_PREFS_MODE, '%s %s' % (mode, ' '.join(args))) def getActionJunkFlag (self): return globalObjects.prefs.junkMessage def setActionJunkFlag_ (self, junkFlag): prefs = globalObjects.prefs if prefs.junkMessage != junkFlag: prefs.junkMessage = junkFlag prefs.writeToFile() _sendCommandToPlugin(JM_CMD_PREFS_JUNK, str(int(junkFlag)))