MessageCatalog.diff0100644000175200001440000002733207741316005014162 0ustar bennettusers--- MessageCatalog.HLD 2003-10-09 08:29:35.000000000 -0900 +++ MessageCatalog.py 2003-10-09 08:28:24.000000000 -0900 @@ -18,10 +18,14 @@ """ This module provides the MessageCatalog base class, which provides message catalogs for the web. + +Warning: This is not the original MessageCatalog.py from Localizer; this + file has been hacked to provide date-stamp information for + individual message catalog entries and to support regular expression + filtering of key entries. Use with extreme caution. """ __version__ = "$Revision: 1.30.2.3 $" - # Python import base64, re, time from types import StringType, UnicodeType @@ -33,6 +37,7 @@ from AccessControl import ClassSecurityInfo from OFS.SimpleItem import SimpleItem from OFS.ObjectManager import ObjectManager +from DateTime import DateTime # Localizer from LocalFiles import LocalDTMLFile @@ -40,6 +45,7 @@ from Utils import charsets, get_language_name, lang_negotiator from LanguageManager import LanguageManager import Gettext +from zLOG import LOG,BLATHER _ = Gettext.translation(globals()) @@ -64,7 +70,6 @@ _marker = [] - class MessageCatalog(LanguageManager, ObjectManager, SimpleItem): """ Stores messages and their translations... @@ -74,18 +79,23 @@ security = ClassSecurityInfo() - def __init__(self, id, title, languages): self.id = id self.title = title + + # A regular expression filter that may be applied to messages + self._regexp = None - # Language Manager data + # Language Manager data self._languages = tuple(languages) self._default_language = languages[0] # Here the message translations are stored self._messages = PersistentMapping() + + # Here is a storage area for date-stamps + self._dates = PersistentMapping() # Data for the PO files headers self._po_headers = PersistentMapping() @@ -131,18 +141,243 @@ """ """ return self._messages.has_key(message) - + # Set a compiled regular expression to be used for filtering + # messages. This is useful when a catalog should contain only + # unique keys conforming to a specified format. + security.declareProtected('Manage messages', 'set_regexp') + def set_regexp(self, regexp): + self._regexp = re.compile(regexp) + + # Unset the regular expression filter. + security.declareProtected('Manage messages', 'unset_regexp') + def unset_regexp(self): + self._regexp = None + + # Set a default text to return instead of the key + security.declareProtected('Manage messages', 'set_default_text') + def set_default_text(self, text): + self._default_text = text + + # Unset a default text + security.declareProtected('Manage messages', 'unset_default_text') + def unset_default_text(self): + self._default_text = _marker + + security.declareProtected('Manage messages', 'message_edit') - def message_edit(self, message, language, translation): - """ """ - self._messages[message][language] = translation - + def message_edit(self, message, language, translation, + origindate=None, + reindex=None): + """For the given MESSAGE key, and the given LANGUAGE code, set the + specified TRANSLATION text, creating a new mapping slot for the entry + if required. If ORIGINDATE is set, the origin date for the updated or + newly created entry is set to that value (a DateTime() return value). + If REINDEX is set, it must be set to the name of a searchable index + that can be expected to contain the MESSAGE key; the index will be + searched, and any times returned will be reindexed. This facilitates + the construction of message-based metadata management scripts that + transparently maintain indexes incorporating multilingual content. + It is perhaps worth noting that no effort is made to create + blank entries in the message catalog or the parallel data map. + Any code that draws on these facilities should be able to cope with + non-existent keys at the 'language' level and below. + """ + if origindate: + if type(origindate) == type(float(0.01)): + origindate = DateTime(origindate) + else: + raise Exception, 'Wrong type for origindate' + # + # Try to initialize the language list + #if not langs: + # langs = [] + # for i in self.unrestrictedTraverse('Localizer').get_supported_languages(): + # langs.append(i) + # + # Initialize message if req. Maybe force origindate to current date. + if not self._messages.has_key(message): + origindate=DateTime() + self._messages[message] = PersistentMapping() + if not self._dates.has_key(message): + self._dates[message] = PersistentMapping() + # + # Initialize language if req. + if not self._dates[message].has_key(language): + self._dates[message][language] = PersistentMapping() + # + # Step through languages, doing the right thing for each + # (set it to an empty string, so that gettext or the + # templates can return an under-construction label, and + # set the origin to an early date, so that the translator + # picks the entry up in their worklist) + #langs.remove(language) + #for i in langs: + # if not self._messages[message].has_key(i): + # self._messages[message][i] = '' + # if not self._dates[message].has_key(i): + # self._dates[message][i] = PersistentMapping() + # self._dates[message][i]['origin'] = DateTime('1970-01-01T00:00:00Z') + # self._dates[message][i]['rev'] = DateTime() + # + # Set origin date if requested or if forced to do so + if origindate: + self._dates[message][language]['origin']= origindate + # + # Do what we came here to do + self._messages[message][language] = translation + # + # Always set the revision date + self._dates[message][language]['rev'] = DateTime() + + # + # If requested, find objects relying on this metadata, and + # reindex them + # + # RH Retreat revision + if reindex and type(reindex) == type('string'): + catalog = self.unrestrictedTraverse('portal_catalog') + searchterms = {} + searchterms[reindex] = message + result = catalog(searchterms) + for i in result: + obj = i.getObject() + if hasattr(obj,'reindex_object'): + obj.reindex_object() + elif hasattr(obj,'reindexObject'): + obj.reindexObject() + + security.declareProtected('Manage messages', 'message_queue') + def message_queue(self, transcap): + """Generates a list of maps containing texts appropriate + to the translation capacity of a given translator. The sole + argument is a single tuple, consisting of a string representing + the translator's native language, and a string or tuple + representing the other language or languages he or she is able + to read. + """ + if not transcap: + return None + nativelang = transcap[0] + otherlangs = transcap[1] + if type(otherlangs) == type(''): + otherlangs = [otherlangs] + rex = re.compile('^.*_.*_[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z_.$') + # + # Give up if we're asking for a language that is not on the agenda + if not nativelang in self._languages: + return None + # + # Prepare a list of conforming message IDs + messages = self._messages.keys() + for i in range(len(messages)-1,-1,-1): + # + # Cull message ID if it is not one of ours + if not rex.match(messages[i]): + messages.pop(i) + messages.sort() + lastid = None + ret = [] + ids = [] + # + # Iterate over the conforming message IDs + for i in messages: + # + # Cull our native language from the inspection list + langkeys = self._messages[i].keys() + if nativelang in langkeys: + langkeys.remove(nativelang) + # + # Cull languages we can't read from the inspection list + for j in range(len(langkeys)-1,-1,-1): + if not langkeys[j] in otherlangs: + langkeys.pop(j) + # + # Cull entries with empty texts + for j in range(len(langkeys)-1,-1,-1): + try: + tmp = self._messages[i][langkeys[j]] + except: + tmp = '' + if not tmp.strip(): + langkeys.pop(j) + # + # Get the origin or our own entry + try: + myorigin = self._dates[i][nativelang]['origin'] + except: + # + # Undated entries get an early date + myorigin = DateTime('1970-01-01T00:00:00Z') + # + # Check for first entry with a date, and evaluate + didit = 0 + for j in langkeys: + if not self._dates[i[:-1]+'t'][j].has_key('origin'): + self._dates[i[:-1]+'t'][j]['origin'] = DateTime('1970-01-01T00:00:00Z') + if not self._dates[i[:-1]+'d'][j].has_key('origin'): + self._dates[i[:-1]+'d'][j]['origin'] = DateTime('1970-01-01T00:00:00Z') + theirorigin = self._dates[i][j]['origin'] + if theirorigin.strftime('%Y%m%d%H%M%S')>myorigin.strftime('%Y%m%d%H%M%S'): + # + # Discard duplicate objects + if lastid and lastid[:-1] == i[:-1]: + continue + # + # Get result object and return to list with source language + # as a tuple + + # The transition is throwing up lots of shit that requires + # ugly error catching. Forgive me, I'm scrambling to + # make a plane. + #if not self._messages[i[:-1]+'t'].has_key(j): + # self._messages[i[:-1]+'t'][j] = '' + #if not self._messages[i[:-1]+'d'].has_key(j): + # self._messages[i[:-1]+'d'][j] = '' + ret.append({'Title': self._messages[i[:-1]+'t'][j], + 'Description': self._messages[i[:-1]+'d'][j], + 'ID': i[:-1], + 'Origin': str(self._dates[i[:-1]+'t'][j]['origin'].timeTime()), + 'Lang': j}) + # + # Remember this ID + lastid = i + didit = 1 + break + # + # If we ourselves are empty, accept anything as a translation + # source. Accept the first entry, and set default timestamp + if not self._messages[i].has_key(nativelang) or \ + not self._messages[i][nativelang].strip(): + iexist=0 + else: + iexist=1 + if langkeys and not iexist and not didit: + # Cut-and-paste code below -- fixme + # + # Discard duplicate objects + if lastid and lastid[:-1] == i[:-1]: + continue + # + # Make my date + first = langkeys[0] + # + # Return our details + ret.append({'Title': self._messages[i[:-1]+'t'][first], + 'Description': self._messages[i[:-1]+'d'][first], + 'ID': i[:-1], + 'Origin': str(self._dates[i[:-1]+'t'][first]['origin'].timeTime()), + 'Lang': first}) + # + # Remember this ID + lastid = i + return ret - security.declareProtected('Manage_messages', 'message_del') + security.declareProtected('Manage messages', 'message_del') def message_del(self, message): """ """ del self._messages[message] - + if self._dates.has_key(message): + del self._dates[message] security.declarePublic('gettext') def gettext(self, message, lang=None, add=1, default=_marker): @@ -153,13 +388,26 @@ as a translation for unknown messages. """ - if type(message) not in (StringType, UnicodeType): + if type(message) not in (StringType, UnicodeType): raise TypeError, 'only strings can be translated.' message = message.strip() - - if default is _marker: - default = message + + # Filter messages if a regexp has been set in this message catalog. + # This allows us to limit the content of a message catalog to + # unique keys conforming to a specified format. + if hasattr(self,'_regexp'): + if self._regexp: + if not self._regexp.match(message): + return message + + # A special text may be passed as an argument to gettext, or it + # may be set in the message catalog. The argument has priority. + if type(default) != type('string'): + if hasattr(self,'_default_text'): + default = self._default_text + else: + default = message # Add it if it's not in the dictionary if add and not self._messages.has_key(message) and message: ZCTextIndex.diff0100644000175200001440000001357307741316110013453 0ustar bennettusers--- ZCTextIndex.HLD 2003-10-09 08:30:17.000000000 -0900 +++ ZCTextIndex.py 2003-10-09 08:30:29.000000000 -0900 @@ -14,6 +14,20 @@ """Plug in text index for ZCatalog with relevance ranking.""" +### +# +# Start of egregious hack 1 +# +# Import re for use in i18n support +# + +import re + +# +# End of egregious hack 1 +# +### + from cgi import escape from types import TupleType @@ -40,6 +54,13 @@ from Products.ZCTextIndex.CosineIndex import CosineIndex from Products.ZCTextIndex.OkapiIndex import OkapiIndex + +from zLOG import LOG, BLATHER + +### NEW +import japanese +decode_euc = japanese.euc_jp.Codec.decode + index_types = {'Okapi BM25 Rank':OkapiIndex, 'Cosine Measure':CosineIndex} @@ -64,16 +85,43 @@ ## Constructor ## def __init__(self, id, extra=None, caller=None, index_factory=None, - field_name=None, lexicon_id=None): + field_name=None, lexicon_id=None, i18n_key_pattern=None): self.id = id # Arguments can be passed directly to the constructor or # via the silly "extra" record. self._fieldname = field_name or getattr(extra, 'doc_attr', '') or id - + lexicon_id = lexicon_id or extra.lexicon_id lexicon = getattr(caller, lexicon_id, None) + ### + # + # Start of egregious hack 2 + # + # Provide for a regular expression that can be used to identify + # entries or entry headers that should be referred to a + # message catalog. We only do this once, so we compile on + # first sight. Keys must not contain spaces. + # + + keypat = i18n_key_pattern or extra.i18n_key_pattern + if not keypat: + self.i18n_key_pattern = None + else: + self.i18n_key_pattern = keypat + try: + self.i18n_rex = re.compile(self.i18n_key_pattern) + except: + self.i18n_rex = None + + # + #i18n_key_pattern = re.compile(i18n_key_pattern) + # + # End of egregious hack 2 + # + ### + if lexicon is None: raise LookupError, 'Lexicon "%s" not found' % escape(lexicon_id) @@ -100,6 +148,15 @@ security.declarePrivate('getLexicon') + def get_i18n_key_pattern(self): + if self.i18n_key_pattern: + ret = self.i18n_key_pattern + else: + ret = '(no regular expression set)' + return ret + + security.declarePrivate('getLexicon') + def getLexicon(self): """Get the lexicon for this index """ @@ -151,10 +208,107 @@ def index_object(self, docid, obj, threshold=None): # XXX We currently ignore subtransaction threshold text = getattr(obj, self._fieldname, None) - if text is None: + + if text is None: return 0 if callable(text): text = text() + ### + # + # Start of egregious hack 3 + # + # Check to see if the leading word in the text for indexing matches + # the i18n key pattern. If it does, attempt to fetch texts from + # the message catalog embedded in the key. If the attempt fails, + # fall back to the literal putative key. If the attempt succeeds, + # loop until matching fails, and then suck in the remainder of the + # text, if any. + # + # If there is no translation service, what comes after is pointless + if self.i18n_rex: + try: + d = self.unrestrictedTraverse('translation_service') + d = d.getDomainInfo()[0][1] + mc = self.unrestrictedTraverse(d) + except: + mc = None + if not mc or not getattr(mc,'meta_type',None) == 'MessageCatalog': + cycle = 0 + # + # Initialize a few variables. + # + header_offset = 0 + count = 0 + metatext = u'' + while mc and self.i18n_rex and count < 50: + # Safety harness + count = count+1 + #LOG('ZCT',BLATHER,'Count is '+str(count)) + # Grab the header if poss, break if not + try: + header_target = text[header_offset:].index(' ') + except: + header_target = len(text[header_offset:]) + if header_target: + header = text[header_offset:header_offset+header_target] + else: + break + # We have a header for inspection. Check it against the pattern. + if self.i18n_rex.match(header): + #LOG('ZCT',BLATHER,'Matched header OK') + if mc._messages.has_key(header): + #LOG('ZCT',BLATHER,'Header key found in catalog') + # The preprocessing pipelines for indexing (including + # the splitter) need to be able to handle all of + # our languages. + + # Note unicode protection. + + tmp = '' + for i in mc._messages[header].keys(): + #LOG('ZCT',BLATHER,'Processing key '+i) + if type(u'unicode') != type(mc._messages[header][i]): + #LOG('ZCT',BLATHER,'Message text is Unicode') + metatext = metatext+u' '+decode_euc(mc._messages[header][i],'replace')[0] + else: + #LOG('ZCT',BLATHER,'Message text NOT Unicode') + metatext = metatext+' '+mc._messages[header][i] + # UNDOME + #metatext = metatext+u' '+mc._messages[header][i] + else: + #LOG('ZCT',BLATHER,'Key NOT found in catalog') + # We ditch things that look like keys but that don't + # return message catalog entries. ZCTextIndex will + # break keys into meaningless little stringlets, so + # this seems like the sensible thing to do. + pass + else: + # If we didn't match, we want to include this word in + # the stream for indexing. + header_target = 0 + break + # If we haven't broken, advance header_offset + header_offset = header_offset+header_target+1 + + # + # Now we should have two strings (text and metatext) and two + # numbers (header_offset and header_target). We can use these + # to derive the correct stir-fry of text for the indexer. + # + + # Note unicode protection. + + if type(u'unicode') != type(text): + text = decode_euc(text,'replace')[0] + + #text = text[header_offset+header_target:] + #text = metatext + text = metatext+u' '+text[header_offset+header_target:] + + # + # End of egregious hack 3 + # + ### count = self.index.index_doc(docid, text) self._p_changed = 1 # XXX return count README.txt0100644000175200001440000001140107741342433012141 0ustar bennettusersMessage-based Metadata for Zope Objects ======================================= 2003.10.09 This software package contains patches that permit Title and Description metadata for Zope content and folder objects to be stored in a Localizer message catalog. The features and possibilities that this introduces are as follows: - Title and Description metadata are stored in a Localizer message catalog. The key used for catalog entries is a unique identifier registered in the object's Title and Description fields. The unique identifier is generated automatically when the object is created, or (for pre-existing objects) upon the first multilingual edit of metadata. - Edits of multilingual metadata trigger reindexing of the underlying object. - When the underlying object is edited, multilingual metadata stored in the message catalog is slurped into the index entry for the object where appropriate. Searches can be made in all languages, all the time. - In contrast to other message catalog schemes, maintenance is multi-centric. The entry for each language is associated with a "currency date" for that language. In typical usage, a translator will request all texts for which there is an entry in a language he can read, and for which there is either no entry for his native language, or for which the "currency date" for his native language entry is older than that for any language he can read. Changes to metadata for one language can therefore be broadcast to other language maintainers by touching the "currency date" field for that entry. In contrast to hub-and-spoke schemes, this offers "full citizenship" in the multilingual environment to participating translators, which is a good thing, both for them and for the site. - The metadata messaging scheme is compatible with Translation Services; a simple i18n:translate="" attribute is the only change needed to invoke the catalog engine within Zope Page Templates. - Translation maintenance code is simplified by placing texts for all languages in a single mapping. BUGS: Before setting out to experiment with this code, please note that IT WILL ALTER THE METADATA FOR EXISTING OBJECTS when they are edited. If the product is later removed from your Zope instance, or if objects are exported for use in another Zope elsewhere, you may find unique identifier strings in place of your metadata. This can be undone, since the metadata are still available in a persistent message catalog within the original Zope instance. However, you would need to write the code to reverse things yourself; it is not provided in this package. Also note that copying objects with MbM metadata will (I think) behave strangely, since both the copy and the original will be coded to look at the same message catalog entry -- I think you will find that changes to metadata in one object will be reflected in the other. This, obviously, will need to be fixed at some point. Finally, please note that there is no packaged mechanism yet for deleting message catalog entries that don't refer to anything. When you delete an object, its message catalog entries will still hang around. This is probably desireable behavior (since the delete might, after all, be undone at some later time). But these orphaned entries should probably be deleted when the database is packed. Feel free to write the code that does that, too. :-) INSTALLATION: It should be pretty simple: 1. Start with an experiemental site, since this is experimental code. The patches are for Zope 2.6, and the latest version of Localizer in Debian sid, whatever that is. 2. Apply the ZCTextIndex.diff patch, AFTER making a backup copy of the target source file. In Debian: cd /usr/lib/zope/lib/python/Products/ZCTextIndex cp ZCTextIndex.py ZCTextIndex.py.HOLD cp /path/to/patch/ZCTextIndex.diff . patch < ./ZCTextIndex.diff 3. Apply the MessageCatalog.diff patch, AFTER making a backup copy of the target source file. In Debian: cd /usr/lib/zope/lib/python/Products/Localizer cp MessageCatalog.py MessageCatalog.py.HOLD cp /path/to/patch/MessageCatalog.diff . patch < ./MessageCatalog.diff 4. Read the source files for clues on how the functionality works. 5. Restart Zope. 6. Enjoy! We are running this code on a small site here, and it is working fine for us so far. There is, however, no guarantee that this will not turn your data into strawberry jam. Like any experimental code, it could mess things up pretty badly for you, so be careful. If you try it, send me a note about your experiences! Frank Bennett bennett@nomolog.nagoya-u.ac.jp