lib/python/ 40755 1007 144 0 7217466751 12326 5ustar petrilliuserslib/python/SearchIndex/ 40755 1007 144 0 7217467051 14515 5ustar petrilliuserslib/python/SearchIndex/GlobbingLexicon.py100644 1007 144 21251 7217463235 20252 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## import string, regex, ts_regex import regsub from Lexicon import Lexicon __doc__=""" Lexicon object that supports """ from Splitter import Splitter from Persistence import Persistent from Acquisition import Implicit import OIBTree, BTree, IOBTree from intSet import intSet OIBTree=OIBTree.BTree OOBTree=BTree.BTree IOBTree=IOBTree.BTree import re from UnTextIndex import Or class GlobbingLexicon(Lexicon): """ Base class to support globbing lexicon object. """ multi_wc = '*' single_wc = '?' eow = '$' def __init__(self): self.initialize() def initialize(self): # Clear the lexicon entries, don't use this unless you know # what you're doing. ;-) self.counter = 0 self._lexicon = OIBTree() self._inverseLex = IOBTree() self._digrams = OOBTree() def set(self, word): """ """ if self._lexicon.has_key(word): return self._lexicon[word] else: word = intern(word) self._lexicon[word] = self.counter self._inverseLex[self.counter] = word ## now, split the word into digrams and insert references ## to 'word' into the digram object. The first and last ## digrams in the list are specially marked with $ to ## indicate the beginning and end of the word digrams = [] digrams.append(self.eow + word[0]) # mark the beginning for i in range(len(word)): digrams.append(word[i:i+2]) digrams[-1] = digrams[-1] + self.eow # mark the end _digrams = self._digrams for digram in digrams: set = _digrams.get(digram) if set is None: _digrams[digram] = set = intSet() set.insert(self.counter) self._digrams = _digrams counter = self.counter self.counter = self.counter + 1 return counter def get(self, pattern): """ Query the lexicon for words matching a pattern. """ wc_set = [self.multi_wc, self.single_wc] digrams = [] for i in range(len(pattern)): if pattern[i] in wc_set: continue if i == 0: digrams.insert(i, (self.eow + pattern[i]) ) digrams.append((pattern[i] + pattern[i+1])) else: try: if pattern[i+1] not in wc_set: digrams.append( pattern[i] + pattern[i+1] ) except IndexError: digrams.append( (pattern[i] + self.eow) ) ## now get all of the intsets that contain the result digrams result = None for digram in digrams: if self._digrams.has_key(digram): set = self._digrams[digram] if set is not None: if result is None: result = set else: result.intersection(set) if result is None: return () else: ## now we have narrowed the list of possible candidates ## down to those words which contain digrams. However, ## some words may have been returned that match digrams, ## but do not match 'pattern'. This is because some words ## may contain all matching digrams, but in the wrong ## order. expr = re.compile(self.translate(pattern)) words = [] hits = [] for x in result: if expr.search(self._inverseLex[x]): hits.append(x) return hits def __getitem__(self, word): """ """ return self.get(word) def query_hook(self, q): """expand wildcards """ words = [] for w in q: if ( (self.multi_wc in w) or (self.single_wc in w) ): wids = self.get(w) for wid in wids: if words: words.append(Or) words.append(self._inverseLex[wid]) else: words.append(w) return words def Splitter(self, astring, words=None): """ wrap the splitter """ ## don't do anything, less efficient but there's not much ## sense in stemming a globbing lexicon. return Splitter(astring) def translate(self, pat): """Translate a PATTERN to a regular expression. There is no way to quote meta-characters. """ i, n = 0, len(pat) res = '' while i < n: c = pat[i] i = i+1 if c == self.multi_wc: res = res + '.*' elif c == self.single_wc: res = res + '.' else: res = res + re.escape(c) return res + "$" lib/python/SearchIndex/Index.py100644 1007 144 23711 6716621325 16256 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## """Simple column indices""" __version__='$Revision: 1.26 $'[11:-2] from Globals import Persistent from BTree import BTree from intSet import intSet import operator from Missing import MV import string ListType=type([]) StringType=type('s') def nonEmpty(s): "returns true if a non-empty string or any other (nonstring) type" if type(s) is StringType: if s: return 1 else: return 0 else: return 1 class Index(Persistent): """Index object interface""" def __init__(self, data=None, schema=None, id=None, ignore_ex=None, call_methods=None): """Create an index The arguments are: 'data' -- a mapping from integer object ids to objects or records, 'schema' -- a mapping from item name to index into data records. If 'data' is a mapping to objects, then schema should ne 'None'. 'id' -- the name of the item attribute to index. This is either an attribute name or a record key. """ ###################################################################### # For b/w compatability, have to allow __init__ calls with zero args if not data==schema==id==ignore_ex==call_methods==None: self._data = data self._schema = schema self.id = id self.ignore_ex=ignore_ex self.call_methods=call_methods self._index = BTree() self._reindex() else: pass # for b/w compatability _init = __init__ def dpHasUniqueValuesFor(self, name): ' has unique values for column NAME ' if name == self.id: return 1 else: return 0 def dpUniqueValues(self, name=None, withLengths=0): """\ returns the unique values for name if withLengths is true, returns a sequence of tuples of (value, length) """ if name is None: name = self.id elif name != self.id: return [] if not withLengths: return tuple( filter(nonEmpty,self._index.keys()) ) else: rl=[] for i in self._index.keys(): if not nonEmpty(i): continue else: rl.append((i, len(self._index[i]))) return tuple(rl) def clear(self): self._index = BTree() def _reindex(self, start=0): """Recompute index data for data with ids >= start.""" index=self._index get=index.get if not start: index.clear() id = self.id if self._schema is None: f=getattr else: f = operator.__getitem__ id = self._schema[id] for i,row in self._data.items(start): k=f(row,id) if k is None or k == MV: continue set=get(k) if set is None: index[k] = set = intSet() set.insert(i) def index_item(self, i, obj=None): """Recompute index data for data with ids >= start.""" index = self._index id = self.id if (self._schema is None) or (obj is not None): f = getattr else: f = operator.__getitem__ id = self._schema[id] if obj is None: obj = self._data[i] try: k=f(obj, id) except: return if self.call_methods: k=k() if k is None or k == MV: return set = index.get(k) if set is None: index[k] = set = intSet() set.insert(i) def unindex_item(self, i, obj=None): """Recompute index data for data with ids >= start.""" index = self._index id = self.id if self._schema is None: f = getattr else: f = operator.__getitem__ id = self._schema[id] if obj is None: obj = self._data[i] try: k=f(obj, id) except: return if self.call_methods: k=k() if k is None or k == MV: return set = index.get(k) if set is not None: set.remove(i) def _apply_index(self, request, cid=''): """Apply the index to query parameters given in the argument, request The argument should be a mapping object. If the request does not contain the needed parameters, then None is returned. If the request contains a parameter with the name of the column + '_usage', it is sniffed for information on how to handle applying the index. Otherwise two objects are returned. The first object is a ResultSet containing the record numbers of the matching records. The second object is a tuple containing the names of all data fields used. """ id = self.id #name of the column cidid = "%s/%s" % (cid,id) has_key = request.has_key if has_key(cidid): keys = request[cidid] elif has_key(id): keys = request[id] else: return None if type(keys) is not ListType: keys=[keys] index = self._index r = None anyTrue = 0 opr = None if request.has_key(id+'_usage'): # see if any usage params are sent to field opr=string.split(string.lower(request[id+"_usage"]),':') opr, opr_args=opr[0], opr[1:] if opr=="range": if 'min' in opr_args: lo = min(keys) else: lo = None if 'max' in opr_args: hi = max(keys) else: hi = None anyTrue=1 try: if hi: setlist = index.items(lo,hi) else: setlist = index.items(lo) for k,set in setlist: if r is None: r = set else: r = r.union(set) except KeyError: pass else: #not a range get = index.get for key in keys: if key: anyTrue = 1 set=get(key) if set is not None: if r is None: r = set else: r = r.union(set) if r is None: if anyTrue: r=intSet() else: return None return r, (id,) lib/python/SearchIndex/Lexicon.py100644 1007 144 22546 7217463573 16623 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## import string, regex, ts_regex import regsub __doc__=""" Module breaks out Zope specific methods and behavior. In addition, provides the Lexicon class which defines a word to integer mapping. """ from Splitter import Splitter from Persistence import Persistent from Acquisition import Implicit import OIBTree, BTree OIBTree=OIBTree.BTree OOBTree=BTree.BTree import re class Lexicon(Persistent, Implicit): """Maps words to word ids and then some The Lexicon object is an attempt to abstract vocabularies out of Text indexes. This abstraction is not totally cooked yet, this module still includes the parser for the 'Text Index Query Language' and a few other hacks. """ # default for older objects stop_syn={} def __init__(self, stop_syn=None): if stop_syn is None: self.stop_syn = {} else: self.stop_syn = stop_syn self.initialize() def initialize(self): # clear and initialize the lexicon self._lexicon = OIBTree() self.counter = 0 def set_stop_syn(self, stop_syn): """ pass in a mapping of stopwords and synonyms. Format is: {'word' : [syn1, syn2, ..., synx]} Vocabularies do not necesarily need to implement this if their splitters do not support stemming or stoping. """ self.stop_syn = stop_syn def set(self, word): """ return the word id of 'word' """ if self._lexicon.has_key(word): return self._lexicon[word] else: if not hasattr(self, 'counter'): self.counter = 0 self._lexicon[intern(word)] = self.counter self.counter = self.counter + 1 return self.counter - 1 def get(self, key, default=None): """ """ return [self._lexicon.get(key, default)] def __getitem__(self, key): return self.get(key) def __len__(self): return len(self._lexicon) def Splitter(self, astring, words=None): """ wrap the splitter """ if words is None: words = self.stop_syn return Splitter(astring, words) def grep(self, query): """ regular expression search through the lexicon he he. Do not use unless you know what your doing!!! """ expr = re.compile(query) hits = [] for x in self._lexicon.keys(): if expr.search(x): hits.append(x) return hits def query_hook(self, q): """ we don't want to modify the query cuz we're dumb """ return q stop_words=( 'am', 'ii', 'iii', 'per', 'po', 're', 'a', 'about', 'above', 'across', 'after', 'afterwards', 'again', 'against', 'all', 'almost', 'alone', 'along', 'already', 'also', 'although', 'always', 'am', 'among', 'amongst', 'amoungst', 'amount', 'an', 'and', 'another', 'any', 'anyhow', 'anyone', 'anything', 'anyway', 'anywhere', 'are', 'around', 'as', 'at', 'back', 'be', 'became', 'because', 'become', 'becomes', 'becoming', 'been', 'before', 'beforehand', 'behind', 'being', 'below', 'beside', 'besides', 'between', 'beyond', 'bill', 'both', 'bottom', 'but', 'by', 'can', 'cannot', 'cant', 'con', 'could', 'couldnt', 'cry', 'describe', 'detail', 'do', 'done', 'down', 'due', 'during', 'each', 'eg', 'eight', 'either', 'eleven', 'else', 'elsewhere', 'empty', 'enough', 'even', 'ever', 'every', 'everyone', 'everything', 'everywhere', 'except', 'few', 'fifteen', 'fifty', 'fill', 'find', 'fire', 'first', 'five', 'for', 'former', 'formerly', 'forty', 'found', 'four', 'from', 'front', 'full', 'further', 'get', 'give', 'go', 'had', 'has', 'hasnt', 'have', 'he', 'hence', 'her', 'here', 'hereafter', 'hereby', 'herein', 'hereupon', 'hers', 'herself', 'him', 'himself', 'his', 'how', 'however', 'hundred', 'i', 'ie', 'if', 'in', 'inc', 'indeed', 'interest', 'into', 'is', 'it', 'its', 'itself', 'keep', 'last', 'latter', 'latterly', 'least', 'less', 'made', 'many', 'may', 'me', 'meanwhile', 'might', 'mill', 'mine', 'more', 'moreover', 'most', 'mostly', 'move', 'much', 'must', 'my', 'myself', 'name', 'namely', 'neither', 'never', 'nevertheless', 'next', 'nine', 'no', 'nobody', 'none', 'noone', 'nor', 'not', 'nothing', 'now', 'nowhere', 'of', 'off', 'often', 'on', 'once', 'one', 'only', 'onto', 'or', 'other', 'others', 'otherwise', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 'per', 'perhaps', 'please', 'pre', 'put', 'rather', 're', 'same', 'see', 'seem', 'seemed', 'seeming', 'seems', 'serious', 'several', 'she', 'should', 'show', 'side', 'since', 'sincere', 'six', 'sixty', 'so', 'some', 'somehow', 'someone', 'something', 'sometime', 'sometimes', 'somewhere', 'still', 'such', 'take', 'ten', 'than', 'that', 'the', 'their', 'them', 'themselves', 'then', 'thence', 'there', 'thereafter', 'thereby', 'therefore', 'therein', 'thereupon', 'these', 'they', 'thick', 'thin', 'third', 'this', 'those', 'though', 'three', 'through', 'throughout', 'thru', 'thus', 'to', 'together', 'too', 'toward', 'towards', 'twelve', 'twenty', 'two', 'un', 'under', 'until', 'up', 'upon', 'us', 'very', 'via', 'was', 'we', 'well', 'were', 'what', 'whatever', 'when', 'whence', 'whenever', 'where', 'whereafter', 'whereas', 'whereby', 'wherein', 'whereupon', 'wherever', 'whether', 'which', 'while', 'whither', 'who', 'whoever', 'whole', 'whom', 'whose', 'why', 'will', 'with', 'within', 'without', 'would', 'yet', 'you', 'your', 'yours', 'yourself', 'yourselves', ) stop_word_dict={} for word in stop_words: stop_word_dict[word]=None lib/python/SearchIndex/Query.c100644 1007 144 64116 6671334650 16114 0ustar petrilliusers/***************************************************************************** Zope Public License (ZPL) Version 1.0 ------------------------------------- Copyright (c) Digital Creations. All rights reserved. This license has been certified as Open Source(tm). Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions in source code must retain the above copyright notice, this list of conditions, and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Digital Creations requests that attribution be given to Zope in any manner possible. Zope includes a "Powered by Zope" button that is installed by default. While it is not a license violation to remove this button, it is requested that the attribution remain. A significant investment has been put into Zope, and this effort will continue if the Zope community continues to grow. This is one way to assure that growth. 4. All advertising materials and documentation mentioning features derived from or use of this software must display the following acknowledgement: "This product includes software developed by Digital Creations for use in the Z Object Publishing Environment (http://www.zope.org/)." In the event that the product being advertised includes an intact Zope distribution (with copyright and license included) then this clause is waived. 5. Names associated with Zope or Digital Creations must not be used to endorse or promote products derived from this software without prior written permission from Digital Creations. 6. Modified redistributions of any form whatsoever must retain the following acknowledgment: "This product includes software developed by Digital Creations for use in the Z Object Publishing Environment (http://www.zope.org/)." Intact (re-)distributions of any official Zope release do not require an external acknowledgement. 7. Modifications are encouraged but must be packaged separately as patches to official Zope releases. Distributions that do not clearly separate the patches from the original work must be clearly labeled as unofficial distributions. Modifications which do not carry the name Zope may be packaged in any form, as long as they conform to all of the clauses above. Disclaimer THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. This software consists of contributions made by Digital Creations and many individuals on behalf of Digital Creations. Specific attributions are listed in the accompanying credits file. ****************************************************************************/ #include "Python.h" #define UNLESS(E) if(!(E)) #define ASSIGN(V,E) { PyObject *__e; __e = (E); Py_XDECREF(V); (V) = __e; } #define UNLESS_ASSIGN(V,E) ASSIGN(V, E) if (!(V)) static PyTypeObject *RegexType; static PyObject *string_lower; #define Regex_Check(ob) (ob->ob_type == RegexType) typedef struct { PyObject_HEAD PyObject *key; PyObject *tests; } FieldTestobject; staticforward PyTypeObject AttrTesttype, CompAttrTesttype, MethodTesttype; staticforward PyTypeObject ItemTesttype; typedef struct { PyObject_HEAD PyObject *tests; } Testobject; staticforward PyTypeObject Andtype, Ortype, Cmptype, Regextype, Stringtype; typedef struct { PyObject_HEAD PyObject *low; PyObject *high; } Rangeobject; staticforward PyTypeObject Rangetype; static PyObject *reprformat=NULL; static PyObject * Field_repr(FieldTestobject *self) { PyObject *r; UNLESS(reprformat) UNLESS(reprformat=PyString_FromString("%s%s")) return NULL; UNLESS(r=Py_BuildValue("s(OO)", self->ob_type->tp_name, self->key, self->tests)) return NULL; ASSIGN(r,PyString_Format(reprformat,r)); return r; } static PyObject * Test_repr(Testobject *self) { PyObject *r; UNLESS(reprformat) UNLESS(reprformat=PyString_FromString("%s%s")) return NULL; UNLESS(r=Py_BuildValue("s(O)", self->ob_type->tp_name, self->tests)) return NULL; ASSIGN(r,PyString_Format(reprformat,r)); return r; } static PyObject * Range_repr(Rangeobject *self) { PyObject *r; UNLESS(reprformat) UNLESS(reprformat=PyString_FromString("%s%s")) return NULL; UNLESS(r=Py_BuildValue("s(OO)", self->ob_type->tp_name, self->low, self->high)) return NULL; ASSIGN(r,PyString_Format(reprformat,r)); return r; } FieldTestobject * new_FieldTestobject(PyObject *key, PyObject *tests, PyTypeObject *type) { FieldTestobject *self; if (!(self = PyObject_NEW(FieldTestobject, type))) return NULL; Py_INCREF(self->key = key); Py_INCREF(self->tests = tests); return self; } static void FieldTest_dealloc(FieldTestobject *self) { Py_XDECREF(self->key); Py_XDECREF(self->tests); PyMem_DEL(self); } Testobject * new_Testobject(PyObject *tests, PyTypeObject *type) { Testobject *self; if (!(self = PyObject_NEW(Testobject, type))) return NULL; Py_INCREF(self->tests = tests); return self; } static void Test_dealloc(Testobject *self) { Py_XDECREF(self->tests); PyMem_DEL(self); } Rangeobject * new_Rangeobject(PyObject *low, PyObject *high) { Rangeobject *self; if (!(self = PyObject_NEW(Rangeobject, &Rangetype))) return NULL; Py_INCREF(self->high = high); Py_INCREF(self->low = low); return self; } static void Range_dealloc(Rangeobject *self) { Py_XDECREF(self->high); Py_XDECREF(self->low); PyMem_DEL(self); } static PyObject * Query__call__(PyObject *self, PyObject *args, PyObject *kw) { PyObject *arg; if (!(PyArg_ParseTuple(args, "O", &arg))) return NULL; return PyObject_GetItem(self, arg); } static PyObject * AttrTest__getitem__(FieldTestobject *self, PyObject *key) { PyObject *ob; UNLESS(ob=PyObject_GetAttr(key, self->key)) return NULL; ASSIGN(ob, PyObject_GetItem(self->tests, ob)); return ob; } static PyObject * CompAttrTest__getitem__(FieldTestobject *self, PyObject *key) { PyObject *ob; UNLESS(ob=PyObject_GetAttr(key, self->key)) return NULL; UNLESS_ASSIGN(ob, PyObject_CallObject(ob, NULL)) return NULL; ASSIGN(ob, PyObject_GetItem(self->tests, ob)); return ob; } static PyObject * MethodTest__getitem__(FieldTestobject *self, PyObject *key) { PyObject *ob; UNLESS(ob=PyObject_GetAttr(key, self->key)) return NULL; ASSIGN(ob,PyObject_CallObject(ob, self->tests)); return ob; } static PyObject * meaningless_len(PyObject *self) { return PyInt_FromLong(1); /* Use one so we are considered "true" */ } static PyMappingMethods AttrTest_as_mapping = { (inquiry)meaningless_len, /*mp_length*/ (binaryfunc)AttrTest__getitem__, /*mp_subscript*/ (objobjargproc)0, /*mp_ass_subscript*/ }; static PyMappingMethods CompAttrTest_as_mapping = { (inquiry)meaningless_len, /*mp_length*/ (binaryfunc)CompAttrTest__getitem__, /*mp_subscript*/ (objobjargproc)0, /*mp_ass_subscript*/ }; static PyMappingMethods MethodTest_as_mapping = { (inquiry)meaningless_len, /*mp_length*/ (binaryfunc)MethodTest__getitem__, /*mp_subscript*/ (objobjargproc)0, /*mp_ass_subscript*/ }; static PyObject * AttrTest(PyObject *self, PyObject *args) { PyObject *key, *test; if (!PyArg_ParseTuple(args, "OO", &key, &test)) return NULL; return (PyObject *)new_FieldTestobject(key, test, &AttrTesttype); } static PyObject * CompAttrTest(PyObject *self, PyObject *args) { PyObject *key, *test; if (!PyArg_ParseTuple(args, "OO", &key, &test)) return NULL; return (PyObject *)new_FieldTestobject(key, test, &CompAttrTesttype); } static PyObject * MethodTest(PyObject *self, PyObject *args) { PyObject *key, *test; if (!PyArg_ParseTuple(args, "OO", &key, &test)) return NULL; UNLESS(PyTuple_Check(test)) { PyErr_SetString(PyExc_TypeError, "second argument to MethodTest must be an " "argument tuple"); return NULL; } self=(PyObject *)new_FieldTestobject(key, test, &MethodTesttype); return self; } static PyTypeObject AttrTesttype = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "AttrTest", /*tp_name*/ sizeof(FieldTestobject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ (destructor)FieldTest_dealloc, /*tp_dealloc*/ (printfunc)0, /*tp_print*/ (getattrfunc)0, /*tp_getattr*/ (setattrfunc)0, /*tp_setattr*/ (cmpfunc)0, /*tp_compare*/ (reprfunc)Field_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ &AttrTest_as_mapping, /*tp_as_mapping*/ (hashfunc)0, /*tp_hash*/ (ternaryfunc)Query__call__, /*tp_call*/ (reprfunc)0, /*tp_str*/ /* Space for future expansion */ 0L,0L,0L,0L, "Test objects that apply sub-tests to attribute values", }; static PyTypeObject CompAttrTesttype = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "CompAttrTest", /*tp_name*/ sizeof(FieldTestobject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ (destructor)FieldTest_dealloc, /*tp_dealloc*/ (printfunc)0, /*tp_print*/ (getattrfunc)0, /*tp_getattr*/ (setattrfunc)0, /*tp_setattr*/ (cmpfunc)0, /*tp_compare*/ (reprfunc)Field_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ &CompAttrTest_as_mapping, /*tp_as_mapping*/ (hashfunc)0, /*tp_hash*/ (ternaryfunc)Query__call__, /*tp_call*/ (reprfunc)0, /*tp_str*/ /* Space for future expansion */ 0L,0L,0L,0L, "Test objects that apply sub-tests to computed attribute values", }; static PyTypeObject MethodTesttype = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "MethodTest", /*tp_name*/ sizeof(FieldTestobject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ (destructor)FieldTest_dealloc, /*tp_dealloc*/ (printfunc)0, /*tp_print*/ (getattrfunc)0, /*tp_getattr*/ (setattrfunc)0, /*tp_setattr*/ (cmpfunc)0, /*tp_compare*/ (reprfunc)Field_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ &MethodTest_as_mapping, /*tp_as_mapping*/ (hashfunc)0, /*tp_hash*/ (ternaryfunc)Query__call__, /*tp_call*/ (reprfunc)0, /*tp_str*/ /* Space for future expansion */ 0L,0L,0L,0L, "Test objects that call test methods", }; static PyObject * ItemTest__getitem__(FieldTestobject *self, PyObject *key) { PyObject *ob; UNLESS(ob=PyObject_GetItem(key, self->key)) return NULL; UNLESS_ASSIGN(ob, PyObject_GetItem(self->tests, ob)) return NULL; return ob; } static PyMappingMethods ItemTest_as_mapping = { (inquiry)meaningless_len, /*mp_length*/ (binaryfunc)ItemTest__getitem__, /*mp_subscript*/ (objobjargproc)0, /*mp_ass_subscript*/ }; static PyObject * ItemTest(PyObject *self, PyObject *args) { PyObject *key, *test; if (!PyArg_ParseTuple(args, "OO", &key, &test)) return NULL; return (PyObject *)new_FieldTestobject(key, test, &ItemTesttype); } static char ItemTesttype__doc__[] = ""; static PyTypeObject ItemTesttype = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "ItemTest", /*tp_name*/ sizeof(FieldTestobject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ (destructor)FieldTest_dealloc, /*tp_dealloc*/ (printfunc)0, /*tp_print*/ (getattrfunc)0, /*tp_getattr*/ (setattrfunc)0, /*tp_setattr*/ (cmpfunc)0, /*tp_compare*/ (reprfunc)Field_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ &ItemTest_as_mapping, /*tp_as_mapping*/ (hashfunc)0, /*tp_hash*/ (ternaryfunc)Query__call__, /*tp_call*/ (reprfunc)0, /*tp_str*/ /* Space for future expansion */ 0L,0L,0L,0L, ItemTesttype__doc__ /* Documentation string */ }; static PyObject * FieldTest(PyObject *self, PyObject *args) { PyObject *key, *test; if (!PyArg_ParseTuple(args, "OO", &key, &test)) return NULL; return (PyObject *)new_FieldTestobject(key, test, (PyString_Check(key) ? &AttrTesttype : &ItemTesttype)); } static PyObject * And__getitem__(Testobject *self, PyObject *key) { int len, i; PyObject *val=0; if ((len = PySequence_Length(self->tests)) < 0) return NULL; for (i = 0; i < len; i++) { UNLESS_ASSIGN(val,PySequence_GetItem(self->tests, i)) return NULL; UNLESS_ASSIGN(val, PyObject_GetItem(val, key)) return NULL; if (!PyObject_IsTrue(val)) return val; /* No need to convert to 0 */ } if(val) return val; else return PyInt_FromLong(1); } static PyMappingMethods And_as_mapping = { (inquiry)meaningless_len, /* mp_length */ (binaryfunc)And__getitem__, /* mp_subscript */ (objobjargproc)0, /* mp_ass_subscript */ }; static PyObject * And(PyObject *self, PyObject *args) { return (PyObject *)new_Testobject(args, &Andtype); } static char Andtype__doc__[] = ""; static PyTypeObject Andtype = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "And", /*tp_name*/ sizeof(Testobject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ (destructor)Test_dealloc, /*tp_dealloc*/ (printfunc)0, /*tp_print*/ (getattrfunc)0, /*tp_getattr*/ (setattrfunc)0, /*tp_setattr*/ (cmpfunc)0, /*tp_compare*/ (reprfunc)Test_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ &And_as_mapping, /*tp_as_mapping*/ (hashfunc)0, /*tp_hash*/ (ternaryfunc)Query__call__, /*tp_call*/ (reprfunc)0, /*tp_str*/ /* Space for future expansion */ 0L,0L,0L,0L, Andtype__doc__ /* Documentation string */ }; static PyObject * Or__getitem__(Testobject *self, PyObject *key) { int len, i; PyObject *val=0; if ((len = PySequence_Length(self->tests)) < 0) return NULL; for (i = 0; i < len; i++) { UNLESS_ASSIGN(val, PySequence_GetItem(self->tests, i)) return NULL; UNLESS_ASSIGN(val, PyObject_GetItem(val, key)) return NULL; if (PyObject_IsTrue(val)) return val; } if(val) return val; else return PyInt_FromLong(0); } static PyMappingMethods Or_as_mapping = { (inquiry)meaningless_len, /*mp_length*/ (binaryfunc)Or__getitem__, /*mp_subscript*/ (objobjargproc)0, /*mp_ass_subscript*/ }; static PyObject * Or(PyObject *self, PyObject *args) { return (PyObject *)new_Testobject(args, &Ortype); } static char Ortype__doc__[] = ""; static PyTypeObject Ortype = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "Or", /*tp_name*/ sizeof(Testobject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ (destructor)Test_dealloc, /*tp_dealloc*/ (printfunc)0, /*tp_print*/ (getattrfunc)0, /*tp_getattr*/ (setattrfunc)0, /*tp_setattr*/ (cmpfunc)0, /*tp_compare*/ (reprfunc)Test_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ &Or_as_mapping, /*tp_as_mapping*/ (hashfunc)0, /*tp_hash*/ (ternaryfunc)Query__call__, /*tp_call*/ (reprfunc)0, /*tp_str*/ /* Space for future expansion */ 0L,0L,0L,0L, Ortype__doc__ /* Documentation string */ }; static PyObject * Range__getitem__(Rangeobject *self, PyObject *key) { if ((self->low ==Py_None || PyObject_Compare(key, self->low) >= 0) && (self->high==Py_None || PyObject_Compare(key, self->high) <= 0)) { return PyInt_FromLong(1); } return PyInt_FromLong(0); } static PyMappingMethods Range_as_mapping = { (inquiry)meaningless_len, /*mp_length*/ (binaryfunc)Range__getitem__, /*mp_subscript*/ (objobjargproc)0, /*mp_ass_subscript*/ }; static PyObject * Range(PyObject *self, PyObject *args) { PyObject *high, *low; if (!PyArg_ParseTuple(args, "OO", &low, &high)) return NULL; return (PyObject *)new_Rangeobject(low, high); } static PyTypeObject Rangetype = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "Range", /*tp_name*/ sizeof(Rangeobject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ (destructor)Range_dealloc, /*tp_dealloc*/ (printfunc)0, /*tp_print*/ (getattrfunc)0, /*tp_getattr*/ (setattrfunc)0, /*tp_setattr*/ (cmpfunc)0, /*tp_compare*/ (reprfunc)Range_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ &Range_as_mapping, /*tp_as_mapping*/ (hashfunc)0, /*tp_hash*/ (ternaryfunc)Query__call__, /*tp_call*/ (reprfunc)0, /*tp_str*/ /* Space for future expansion */ 0L,0L,0L,0L, "Range test", /* Documentation string */ }; static PyObject * Cmp__getitem__(Testobject *self, PyObject *key) { if (PyObject_Compare(self->tests, key) == 0) { return PyInt_FromLong(1); } return PyInt_FromLong(0); } static PyMappingMethods Cmp_as_mapping = { (inquiry)meaningless_len, /*mp_length*/ (binaryfunc)Cmp__getitem__, /*mp_subscript*/ (objobjargproc)0, /*mp_ass_subscript*/ }; static PyObject * Cmp(PyObject *self, PyObject *args) { return (PyObject *)new_Testobject(args, &Cmptype); } static char Cmptype__doc__[] = ""; static PyTypeObject Cmptype = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "Cmp", /*tp_name*/ sizeof(Testobject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ (destructor)Test_dealloc, /*tp_dealloc*/ (printfunc)0, /*tp_print*/ (getattrfunc)0, /*tp_getattr*/ (setattrfunc)0, /*tp_setattr*/ (cmpfunc)0, /*tp_compare*/ (reprfunc)Test_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ &Cmp_as_mapping, /*tp_as_mapping*/ (hashfunc)0, /*tp_hash*/ (ternaryfunc)Query__call__, /*tp_call*/ (reprfunc)0, /*tp_str*/ /* Space for future expansion */ 0L,0L,0L,0L, Cmptype__doc__ /* Documentation string */ }; static PyObject * Regex__getitem__(Testobject *self, PyObject *key) { PyObject *t, *search_res, *ret = NULL; if (!(t = PyTuple_New(1))) return NULL; PyTuple_SET_ITEM(t, 0, key); Py_INCREF(key); if (!(search_res = PyObject_CallObject(self->tests, t))) goto finally; if (!PyInt_Check(search_res)) { PyErr_SetString(PyExc_TypeError, "search method of regular " "expression must return an integer"); goto finally; } if (PyInt_AS_LONG((PyIntObject *)search_res) >= 0) { ret = PyInt_FromLong(1); goto finally; } ret = PyInt_FromLong(0); finally: Py_XDECREF(t); Py_XDECREF(search_res); return ret; } static PyMappingMethods Regex_as_mapping = { (inquiry)meaningless_len, /*mp_length*/ (binaryfunc)Regex__getitem__, /*mp_subscript*/ (objobjargproc)0, /*mp_ass_subscript*/ }; static PyObject * Regex(PyObject *self, PyObject *args) { PyObject *search, *r; static PyObject *search_str = 0; if (!search_str) if (!(search_str = PyString_FromString("search"))) return NULL; if (!Regex_Check(args)) { PyErr_SetString(PyExc_TypeError, "argument must be compiled regular" " expression"); return NULL; } if (!(search = PyObject_GetAttr(args, search_str))) return NULL; r = (PyObject *)new_Testobject(search, &Regextype); Py_DECREF(search); return r; } static char Regextype__doc__[] = ""; static PyTypeObject Regextype = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "Regex", /*tp_name*/ sizeof(Testobject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ (destructor)Test_dealloc, /*tp_dealloc*/ (printfunc)0, /*tp_print*/ (getattrfunc)0, /*tp_getattr*/ (setattrfunc)0, /*tp_setattr*/ (cmpfunc)0, /*tp_compare*/ (reprfunc)Test_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ &Regex_as_mapping, /*tp_as_mapping*/ (hashfunc)0, /*tp_hash*/ (ternaryfunc)Query__call__, /*tp_call*/ (reprfunc)0, /*tp_str*/ /* Space for future expansion */ 0L,0L,0L,0L, Regextype__doc__ /* Documentation string */ }; static PyObject * String__getitem__(Testobject *self, PyObject *key) { char *ckey, *ctests; int lkey, ltests, k; if (!PyString_Check(key)) { PyErr_SetString(PyExc_TypeError, "argument must be string"); return NULL; } UNLESS(ckey=PyString_AsString(key)) return NULL; if((lkey=PyString_Size(key)) < 0) return NULL; UNLESS(ctests=PyString_AsString(self->tests)) return NULL; if((ltests=PyString_Size(self->tests)) < 0) return NULL; if(lkey != ltests) return PyInt_FromLong(0); while(lkey--) { k=*ckey++; if(tolower(k) != *ctests++) return PyInt_FromLong(0); } return PyInt_FromLong(1); } static PyMappingMethods String_as_mapping = { (inquiry)meaningless_len, /*mp_length*/ (binaryfunc)String__getitem__, /*mp_subscript*/ (objobjargproc)0, /*mp_ass_subscript*/ }; static PyObject * String(PyObject *self, PyObject *args) { UNLESS(args=PyObject_CallObject(string_lower,args)) return NULL; UNLESS_ASSIGN(args,(PyObject *)new_Testobject(args, &Stringtype)); return args; } static char Stringtype__doc__[] = ""; static PyTypeObject Stringtype = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "String", /*tp_name*/ sizeof(Testobject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ (destructor)Test_dealloc, /*tp_dealloc*/ (printfunc)0, /*tp_print*/ (getattrfunc)0, /*tp_getattr*/ (setattrfunc)0, /*tp_setattr*/ (cmpfunc)0, /*tp_compare*/ (reprfunc)Test_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ &String_as_mapping, /*tp_as_mapping*/ (hashfunc)0, /*tp_hash*/ (ternaryfunc)Query__call__, /*tp_call*/ (reprfunc)0, /*tp_str*/ /* Space for future expansion */ 0L,0L,0L,0L, Stringtype__doc__ /* Documentation string */ }; static struct PyMethodDef Query_methods[] = { {"AttrTest", (PyCFunction)AttrTest, 1, "AttrTest(name,subtest) -- Define a test on an attribute."}, {"CompAttrTest", (PyCFunction)CompAttrTest, 1, "CompAttrTest(name,subtest) -- Define a test on a computed attribute."}, {"MethodTest", (PyCFunction)MethodTest, 1, "MethodTest(method,arg) -- Define a test on method and argument.\n\n" "Matches occur for those objects for which: o.method(arg)\n" "is true." }, {"ItemTest", (PyCFunction)ItemTest, 1, "ItemTest(key,subtest) -- Define a test on an item."}, {"FieldTest", (PyCFunction)FieldTest, 1, "FieldTest(index,subtest) -- Define a test on an item or attribute.\n" "\n" "If 'index' is an integer, then an item test will be done, otherwise\n" "an attribute test will be done." }, {"And", (PyCFunction)And, 1, "And(*subtests) -- Create a test that is an and of several sub-tests"}, {"Or", (PyCFunction)Or, 1, "Or(*subtests) -- Create a test that is an or of several sub-tests"}, {"Range", (PyCFunction)Range, 1, "Range(min,max) -- Create a range test"}, {"Cmp", (PyCFunction)Cmp, 0, "Cmp(value) -- Define a straight comparison test with the given value"}, {"Regex", (PyCFunction)Regex, 0, "Regex(compiled_regex) -- Define a Regular-expression test"}, {"String", (PyCFunction)String, 1, "String(str) -- Define a case-insensitive test"}, {NULL, NULL} /* sentinel */ }; static char Query_module_documentation[] = "Query blocks for composing complex queries without execing Python code\n" "\n" "This module provides a number of simple types that can be used to\n" "compose complex queries that execute in a reasonable effiecient manner\n" "against objects. Each object type define objects that can be called\n" "with a single argument or with 'getitem' to check whether an object,\n" "such as a database record or a collection item satisfies a query.\n" "\n$Id: Query.c,v 1.8 1999/03/10 00:15:36 klm Exp $" ; void initQuery() { PyObject *m, *d, *regex, *string; char *rev="$Revision: 1.8 $"; AttrTesttype.ob_type =&PyType_Type; CompAttrTesttype.ob_type =&PyType_Type; MethodTesttype.ob_type =&PyType_Type; ItemTesttype.ob_type =&PyType_Type; Andtype.ob_type =&PyType_Type; Ortype.ob_type =&PyType_Type; Rangetype.ob_type =&PyType_Type; Cmptype.ob_type =&PyType_Type; Regextype.ob_type =&PyType_Type; Stringtype.ob_type =&PyType_Type; UNLESS(regex = PyImport_ImportModule("regex")) return; UNLESS_ASSIGN(regex, PyObject_CallMethod(regex, "compile", "s", "a")) return; RegexType = regex->ob_type; Py_DECREF(regex); UNLESS(string = PyImport_ImportModule("string")) return; UNLESS(string_lower = PyObject_GetAttrString(string, "lower")) return; Py_DECREF(string); /* Create the module and add the functions */ m = Py_InitModule4("Query", Query_methods, Query_module_documentation, (PyObject*)NULL,PYTHON_API_VERSION); /* Add some symbolic constants to the module */ d = PyModule_GetDict(m); PyDict_SetItemString(d, "__version__", PyString_FromStringAndSize(rev+11,strlen(rev+11)-2)); /* Check for errors */ if (PyErr_Occurred()) Py_FatalError("can't initialize module BTree"); } lib/python/SearchIndex/ResultList.py100644 1007 144 14056 7075621255 17324 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## class ResultList: def __init__(self, d, words, index, TupleType=type(())): self._index = index self._words = words if (type(d) is TupleType): self._dict = { d[0] : d[1] } else: self._dict = d def __len__(self): return len(self._dict) def __getitem__(self, key): return self._dict[key] def keys(self): return self._dict.keys() def has_key(self, key): return self._dict.has_key(key) def items(self): return self._dict.items() def __and__(self, x): result = {} dict = self._dict xdict = x._dict xhas = xdict.has_key for id, score in dict.items(): if xhas(id): result[id] = xdict[id]+score return self.__class__(result, self._words+x._words, self._index) def and_not(self, x): result = {} dict = self._dict xdict = x._dict xhas = xdict.has_key for id, score in dict.items(): if not xhas(id): result[id] = score return self.__class__(result, self._words, self._index) def __or__(self, x): result = {} dict = self._dict has = dict.has_key xdict = x._dict xhas = xdict.has_key for id, score in dict.items(): if xhas(id): result[id] = xdict[id]+score else: result[id] = score for id, score in xdict.items(): if not has(id): result[id] = score return self.__class__(result, self._words+x._words, self._index) def near(self, x): result = {} dict = self._dict xdict = x._dict xhas = xdict.has_key positions = self._index.positions for id, score in dict.items(): if not xhas(id): continue p=(map(lambda i: (i,0), positions(id,self._words))+ map(lambda i: (i,1), positions(id,x._words))) p.sort() d = lp = 9999 li = None lsrc = None for i,src in p: if i is not li and src is not lsrc and li is not None: d = min(d,i-li) li = i lsrc = src if d==lp: score = min(score,xdict[id]) # synonyms else: score = (score+xdict[id])/d result[id] = score return self.__class__(result, self._words+x._words, self._index) lib/python/SearchIndex/Splitter.c100644 1007 144 32733 7200023222 16570 0ustar petrilliusers/***************************************************************************** Zope Public License (ZPL) Version 1.0 ------------------------------------- Copyright (c) Digital Creations. All rights reserved. This license has been certified as Open Source(tm). Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions in source code must retain the above copyright notice, this list of conditions, and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Digital Creations requests that attribution be given to Zope in any manner possible. Zope includes a "Powered by Zope" button that is installed by default. While it is not a license violation to remove this button, it is requested that the attribution remain. A significant investment has been put into Zope, and this effort will continue if the Zope community continues to grow. This is one way to assure that growth. 4. All advertising materials and documentation mentioning features derived from or use of this software must display the following acknowledgement: "This product includes software developed by Digital Creations for use in the Z Object Publishing Environment (http://www.zope.org/)." In the event that the product being advertised includes an intact Zope distribution (with copyright and license included) then this clause is waived. 5. Names associated with Zope or Digital Creations must not be used to endorse or promote products derived from this software without prior written permission from Digital Creations. 6. Modified redistributions of any form whatsoever must retain the following acknowledgment: "This product includes software developed by Digital Creations for use in the Z Object Publishing Environment (http://www.zope.org/)." Intact (re-)distributions of any official Zope release do not require an external acknowledgement. 7. Modifications are encouraged but must be packaged separately as patches to official Zope releases. Distributions that do not clearly separate the patches from the original work must be clearly labeled as unofficial distributions. Modifications which do not carry the name Zope may be packaged in any form, as long as they conform to all of the clauses above. Disclaimer THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. This software consists of contributions made by Digital Creations and many individuals on behalf of Digital Creations. Specific attributions are listed in the accompanying credits file. ****************************************************************************/ #include "Python.h" #include #define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;} #define UNLESS(E) if(!(E)) #define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V) static PyObject *next_word(); typedef struct { PyObject_HEAD PyObject *text, *synstop; char *here, *end; int index; } Splitter; static void Splitter_reset(Splitter *self) { self->here = PyString_AsString(self->text); self->index = -1; } static void Splitter_dealloc(Splitter *self) { Py_XDECREF(self->text); Py_XDECREF(self->synstop); PyMem_DEL(self); } static int Splitter_length(Splitter *self) { PyObject *res=0; Splitter_reset(self); while(1) { UNLESS_ASSIGN(res,next_word(self,NULL,NULL)) return -1; UNLESS(PyString_Check(res)) { Py_DECREF(res); break; } } return self->index+1; } static PyObject * Splitter_concat(Splitter *self, PyObject *other) { PyErr_SetString(PyExc_TypeError, "Cannot concatenate Splitters."); return NULL; } static PyObject * Splitter_repeat(Splitter *self, long n) { PyErr_SetString(PyExc_TypeError, "Cannot repeat Splitters."); return NULL; } /* Map an input word to an output word by applying standard filtering/mapping words, including synonyms/stop words. Input is a word. Output is: None -- The word is a stop word sometext -- A replacement for the word */ static PyObject * check_synstop(Splitter *self, PyObject *word) { PyObject *value; char *cword; int len; cword = PyString_AsString(word); len = PyString_Size(word); if(len < 2) /* Single-letter words are stop words! */ { Py_INCREF(Py_None); return Py_None; } /************************************************************* Test whether a word has any letters. * */ for (; --len >= 0 && ! isalpha((unsigned char)cword[len]); ); if (len < 0) { Py_INCREF(Py_None); return Py_None; } /* * If no letters, treat it as a stop word. *************************************************************/ Py_INCREF(word); if (self->synstop == NULL) return word; while ((value = PyObject_GetItem(self->synstop, word)) && PyString_Check(value)) { ASSIGN(word,value); if(len++ > 100) break; /* Avoid infinite recurssion */ } if (value == NULL) { PyErr_Clear(); return word; } return value; /* Which must be None! */ } #define MAX_WORD 64 /* Words longer than MAX_WORD are stemmed */ static PyObject * next_word(Splitter *self, char **startpos, char **endpos) { char wbuf[MAX_WORD]; char *end, *here, *b; int i = 0, c; PyObject *pyword, *res; here=self->here; end=self->end; b=wbuf; while (here < end) { /* skip hyphens */ if ((i > 0) && (*here == '-')) { here++; while (isspace(*here) && (here < end)) here++; continue; } c=tolower(*here); /* Check to see if this character is part of a word */ if(isalnum((unsigned char)c) || c=='/') { /* Found a word character */ if(startpos && i==0) *startpos=here; if(i++ < MAX_WORD) *b++ = c; } else if (i != 0) { /* We've found the end of a word */ if(i >= MAX_WORD) i=MAX_WORD; /* "stem" the long word */ UNLESS(pyword = PyString_FromStringAndSize(wbuf, i)) { self->here=here; return NULL; } UNLESS(res = check_synstop(self, pyword)) { self->here=here; Py_DECREF(pyword); return NULL; } if (res != Py_None) { if(endpos) *endpos=here; self->here=here; Py_DECREF(pyword); self->index++; return res; } /* The word is a stopword, so ignore it */ Py_DECREF(res); Py_DECREF(pyword); i = 0; b=wbuf; } here++; } self->here=here; /* We've reached the end of the string */ if(i >= MAX_WORD) i=MAX_WORD; /* "stem" the long word */ if (i == 0) { /* No words */ self->here=here; Py_INCREF(Py_None); return Py_None; } UNLESS(pyword = PyString_FromStringAndSize(wbuf, i)) return NULL; if(endpos) *endpos=here; res = check_synstop(self, pyword); Py_DECREF(pyword); if(PyString_Check(res)) self->index++; return res; } static PyObject * Splitter_item(Splitter *self, int i) { PyObject *word = NULL; if (i <= self->index) Splitter_reset(self); while(self->index < i) { Py_XDECREF(word); UNLESS(word = next_word(self,NULL,NULL)) return NULL; if (word == Py_None) { Py_DECREF(word); PyErr_SetString(PyExc_IndexError, "Splitter index out of range"); return NULL; } } return word; } static PyObject * Splitter_slice(Splitter *self, int i, int j) { PyErr_SetString(PyExc_TypeError, "Cannot slice Splitters."); return NULL; } static PySequenceMethods Splitter_as_sequence = { (inquiry)Splitter_length, /*sq_length*/ (binaryfunc)Splitter_concat, /*sq_concat*/ (intargfunc)Splitter_repeat, /*sq_repeat*/ (intargfunc)Splitter_item, /*sq_item*/ (intintargfunc)Splitter_slice, /*sq_slice*/ (intobjargproc)0, /*sq_ass_item*/ (intintobjargproc)0, /*sq_ass_slice*/ }; static PyObject * Splitter_pos(Splitter *self, PyObject *args) { char *start, *end, *ctext; PyObject *res; int i; UNLESS(PyArg_Parse(args, "i", &i)) return NULL; if (i <= self->index) Splitter_reset(self); while(self->index < i) { UNLESS(res=next_word(self, &start, &end)) return NULL; if(PyString_Check(res)) { self->index++; Py_DECREF(res); continue; } Py_DECREF(res); PyErr_SetString(PyExc_IndexError, "Splitter index out of range"); return NULL; } ctext=PyString_AsString(self->text); return Py_BuildValue("(ii)", start - ctext, end - ctext); } static PyObject * Splitter_indexes(Splitter *self, PyObject *args) { PyObject *word, *r, *w=0, *index=0; int i=0; UNLESS(PyArg_ParseTuple(args,"O",&word)) return NULL; UNLESS(r=PyList_New(0)) return NULL; UNLESS(word=check_synstop(self, word)) goto err; Splitter_reset(self); while(1) { UNLESS_ASSIGN(w,next_word(self, NULL, NULL)) goto err; UNLESS(PyString_Check(w)) break; if(PyObject_Compare(word,w)==0) { UNLESS_ASSIGN(index,PyInt_FromLong(i)) goto err; if(PyList_Append(r,index) < 0) goto err; } i++; } Py_XDECREF(w); Py_XDECREF(index); return r; err: Py_DECREF(r); Py_XDECREF(index); return NULL; } static struct PyMethodDef Splitter_methods[] = { { "pos", (PyCFunction)Splitter_pos, 0, "pos(index) -- Return the starting and ending position of a token" }, { "indexes", (PyCFunction)Splitter_indexes, METH_VARARGS, "indexes(word) -- Return al list of the indexes of word in the sequence", }, { NULL, NULL } /* sentinel */ }; static PyObject * Splitter_getattr(Splitter *self, char *name) { return Py_FindMethod(Splitter_methods, (PyObject *)self, name); } static char SplitterType__doc__[] = ""; static PyTypeObject SplitterType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "Splitter", /*tp_name*/ sizeof(Splitter), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ (destructor)Splitter_dealloc, /*tp_dealloc*/ (printfunc)0, /*tp_print*/ (getattrfunc)Splitter_getattr, /*tp_getattr*/ (setattrfunc)0, /*tp_setattr*/ (cmpfunc)0, /*tp_compare*/ (reprfunc)0, /*tp_repr*/ 0, /*tp_as_number*/ &Splitter_as_sequence, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ (hashfunc)0, /*tp_hash*/ (ternaryfunc)0, /*tp_call*/ (reprfunc)0, /*tp_str*/ /* Space for future expansion */ 0L,0L,0L,0L, SplitterType__doc__ /* Documentation string */ }; static PyObject * get_Splitter(PyObject *modinfo, PyObject *args) { Splitter *self; PyObject *doc, *synstop = NULL; UNLESS(PyArg_ParseTuple(args,"O|O",&doc,&synstop)) return NULL; UNLESS(self = PyObject_NEW(Splitter, &SplitterType)) return NULL; if(synstop) { self->synstop=synstop; Py_INCREF(synstop); } else self->synstop=NULL; UNLESS(self->text = PyObject_Str(doc)) goto err; UNLESS(self->here=PyString_AsString(self->text)) goto err; self->end = self->here + PyString_Size(self->text); self->index = -1; return (PyObject*)self; err: Py_DECREF(self); return NULL; } static struct PyMethodDef Splitter_module_methods[] = { { "Splitter", (PyCFunction)get_Splitter, METH_VARARGS, "Splitter(doc[,synstop]) -- Return a word splitter" }, { NULL, NULL } }; static char Splitter_module_documentation[] = "Parse source strings into sequences of words\n" "\n" "for use in an inverted index\n" "\n" "$Id: Splitter.c,v 1.14 2000/11/01 14:20:02 jim Exp $\n" ; void initSplitter() { PyObject *m, *d; char *rev="$Revision: 1.14 $"; /* Create the module and add the functions */ m = Py_InitModule4("Splitter", Splitter_module_methods, Splitter_module_documentation, (PyObject*)NULL,PYTHON_API_VERSION); /* Add some symbolic constants to the module */ d = PyModule_GetDict(m); PyDict_SetItemString(d, "__version__", PyString_FromStringAndSize(rev+11,strlen(rev+11)-2)); if (PyErr_Occurred()) Py_FatalError("can't initialize module Splitter"); } lib/python/SearchIndex/TextIndex.py100644 1007 144 35044 7063524666 17133 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## """Text Index Notes on a new text index design The current inverted index algoirthm works well enough for our needs. Speed of the algorithm does not seem to be a problem, however, data management *is* a significant problem. In particular: - Process size grows unacceptably *during mass indexing*. - Data load and store seems to take too long. For example, clearing an inverted index and committing takes a significant amount of time. - The current trie data structure contributes significantly to the number of objects in the system. - Removal/update of documents is especially problematic. We have to either: - Unindex old version of an object before updating it. This is a real hassle for apps like sws. - Tool through entire index looking for object references. This is *totally* impractical. Some observations of competition: - Xerox system can index "5-million word document in 256k". What does this mean? - Does the system save word positions as we do? - What is the index indexing? - What was the vocabulary of the system? Let\'s see. Assume a 10,000 word vocabulary. Then we use 25-bytes per entry. Hm..... - Verity has some sense of indexing in phases and packing index. Verity keeps the index in multiple chunks and a search may operate on multiple chunks. This means that we can add data without updating large records. This may be especially handy for mass updates, like we do in cv3. In a sense we do this in cv3 and sws. We index a large batch of documents to a temporary index and then merge changes in. If "temporary" index was integral to system, then maybe merger could be done as a background task.... Tree issues Tree structures benefit small updates, because an update to an entry does not cause update of entire tree, however, each node in tree introduces overhead. Trie structure currently introduces an excessive number of nodes. Typically, a node per two or three words. Trie has potential to reduce storage because key storage is shared between words. Maybe an alternative to a Trie is some sort of nested BTree. Or maybe a Trie with some kind of binary-search-based indexing. Suppose that: - database objects were at leaves of tree - vocabulary was finite - we don\'t remove a leaf when it becomes empty Then: - After some point, tree objects no longer change If this is case, then it doesn\'t make sense to optimize tree for change. Additional notes Stemming reduces the number of words substantially. Proposal -- new TextIndex TextIndex -- word -> textSearchResult Implemented with: InvertedIndex -- word -> idSet ResultIndex -- id -> docData where: word -- is a token, typically a word, but could be a name or a number textSearchResult -- id -> (score, positions) id -- integer, say 4-byte. positions -- sequence of integers. score -- numeric measure of relevence, f(numberOfWords, positions) numberOfWords -- number of words in source document. idSet -- set of ids docData -- numberOfWords, word->positions Note that ids and positions are ints. We will build C extensions for efficiently storing and pickling structures with lots of ints. This should significantly improve space overhead and storage/retrieveal times, as well as storeage space. """ __version__='$Revision: 1.25 $'[11:-2] from Globals import Persistent import BTree, IIBTree BTree=BTree.BTree IIBTree=IIBTree.Bucket from intSet import intSet import operator from Splitter import Splitter from string import strip import string, ts_regex, regex from Lexicon import Lexicon, query, stop_word_dict from ResultList import ResultList class TextIndex(Persistent): def __init__(self, data=None, schema=None, id=None, ignore_ex=None, call_methods=None): """Create an index The arguments are: 'data' -- a mapping from integer object ids to objects or records, 'schema' -- a mapping from item name to index into data records. If 'data' is a mapping to objects, then schema should ne 'None'. 'id' -- the name of the item attribute to index. This is either an attribute name or a record key. 'ignore_ex' -- Tells the indexer to ignore exceptions that are rasied when indexing an object. 'call_methods' -- Tells the indexer to call methods instead of getattr or getitem to get an attribute. """ ###################################################################### # For b/w compatability, have to allow __init__ calls with zero args if not data==schema==id==ignore_ex==call_methods==None: self._data=data self._schema=schema self.id=id self.ignore_ex=ignore_ex self.call_methods=call_methods self._index=BTree() self._syn=stop_word_dict self._reindex() else: pass # for backwards compatability _init = __init__ def clear(self): self._index = BTree() def positions(self, docid, words): """Return the positions in the document for the given document id of the word, word.""" id = self.id if self._schema is None: f = getattr else: f = operator.__getitem__ id = self._schema[id] row = self._data[docid] if self.call_methods: doc = str(f(row, id)()) else: doc = str(f(row, id)) r = [] for word in words: r = r+Splitter(doc, self._syn).indexes(word) return r def index_item(self, i, obj=None, un=0): """Recompute index data for data with ids >= start. if 'obj' is passed in, it is indexed instead of _data[i]""" id = self.id if (self._schema is None) or (obj is not None): f = getattr else: f = operator.__getitem__ id = self._schema[id] if obj is None: obj = self._data[i] try: if self.call_methods: k = str(f(obj, id)()) else: k = str(f(obj, id)) self._index_document(k, i ,un) except: pass def unindex_item(self, i, obj=None): return self.index_item(i, obj, 1) def _reindex(self, start=0): """Recompute index data for data with ids >= start.""" for i in self._data.keys(start): self.index_item(i) def _index_document(self, document_text, id, un=0, tupleType=type(()), dictType=type({}), ): src = Splitter(document_text, self._syn) d = {} old = d.has_key last = None for s in src: if s[0] == '\"': last=self.subindex(s[1:-1], d, old, last) else: if old(s): if s != last: d[s] = d[s]+1 else: d[s] = 1 index = self._index get = index.get if un: for word,score in d.items(): r = get(word) if r is not None: if type(r) is tupleType: del index[word] else: if r.has_key(id): del r[id] if type(r) is dictType: if len(r) < 2: if r: for k, v in r.items(): index[word] = k,v else: del index[word] else: index[word] = r else: for word,score in d.items(): r = get(word) if r is not None: r = index[word] if type(r) is tupleType: r = {r[0]:r[1]} r[id] = score index[word] = r elif type(r) is dictType: if len(r) > 4: b = IIBTree() for k, v in r.items(): b[k] = v r = b r[id] = score index[word] = r else: r[id] = score else: index[word] = id, score def _subindex(self, isrc, d, old, last): src = Splitter(isrc, self._syn) for s in src: if s[0] == '\"': last=self.subindex(s[1:-1],d,old,last) else: if old(s): if s != last: d[s] = d[s]+1 else: d[s] = 1 return last def __getitem__(self, word): """Return an InvertedIndex-style result "list" """ src = tuple(Splitter(word, self._syn)) if not src: return ResultList({}, (word,), self) if len(src) == 1: src=src[0] if src[:1]=='"' and src[-1:]=='"': return self[src] r = self._index.get(word,None) if r is None: r = {} return ResultList(r, (word,), self) r = None for word in src: rr = self[word] if r is None: r = rr else: r = r.near(rr) return r def _apply_index(self, request, cid='', ListType=[]): """ Apply the index to query parameters given in the argument, request The argument should be a mapping object. If the request does not contain the needed parameters, then None is returned. Otherwise two objects are returned. The first object is a ResultSet containing the record numbers of the matching records. The second object is a tuple containing the names of all data fields used. """ id = self.id cidid = "%s/%s" % (cid, id) has_key = request.has_key if has_key(cidid): keys = request[cidid] elif has_key(id): keys =request[id] else: return None if type(keys) is type(''): if not keys or not strip(keys): return None keys = [keys] r = None for key in keys: key = strip(key) if not key: continue rr = intSet() try: for i,score in query(key,self).items(): if score: rr.insert(i) except KeyError: pass if r is None: r = rr else: # Note that we *and*/*narrow* multiple search terms. r = r.intersection(rr) if r is not None: return r, (id,) return intSet(), (id,) lib/python/SearchIndex/UnIndex.py100644 1007 144 30610 7217457337 16564 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## """Simple column indices""" __version__='$Revision: 1.15.12.3 $'[11:-2] import BTree import IOBTree import operator import string, pdb from Globals import Persistent from Acquisition import Implicit from intSet import intSet from zLOG import LOG, ERROR def nonEmpty(s, StringType=type("")): "returns true if a non-empty string or any other (nonstring) type" if type(s) is StringType: if s: return 1 else: return 0 else: return 1 class UnIndex(Persistent, Implicit): """UnIndex object interface""" meta_type = 'Field Index' def __init__(self, id=None, ignore_ex=None, call_methods=None): """Create an unindex UnIndexes are indexes that contain two index components, the forward index (like plain index objects) and an inverted index. The inverted index is so that objects can be unindexed even when the old value of the object is not known. The arguments are: 'id' -- the name of the item attribute to index. This is either an attribute name or a record key. 'ignore_ex' -- should be set to true if you want the index to ignore exceptions raised while indexing instead of propagating them. 'call_methods' -- should be set to true if you want the index to call the attribute 'id' (note: 'id' should be callable!) You will also need to pass in an object in the index and uninded methods for this to work. """ ###################################################################### # For b/w compatability, have to allow __init__ calls with zero args if not id==ignore_ex==call_methods==None: self.id = id self.ignore_ex=ignore_ex # currently unimplimented self.call_methods=call_methods self._index = BTree.BTree() self._unindex = IOBTree.BTree() else: pass def __len__(self): return len(self._unindex) def index_object(self, i, obj, threshold=None): """ Index an object 'obj' with integer id 'i'. Please note that this will first deal with any unindexing operations.""" # First let's set up some short cuts to save typing, and name # lookup costs in the future. index = self._index # Forward index unindex = self._unindex # Reverse index indexName = self.id # My index name documentId = i # For readability # We need to first unindex the existing record so we don't get # KeyErrors in the future. self.unindex_object(documentId) try: datum = getattr(obj, indexName) if callable(datum): datum = datum() except AttributeError: return 0 currentIndexSet = index.get(datum) if currentIndexSet is None: index[datum] = currentIndexSet = intSet() currentIndexSet.insert(documentId) unindex[documentId] = datum # Kick the persistence machienery since we're dealing with a mutable # object here self._index = index self._unindex = unindex return 1 def unindex_object(self, i): """ Unindex the object with integer id 'i' and don't raise an exception if we fail """ # First let's set up some shortcuts to save time index = self._index # Forward index unindex = self._unindex # Reverse index documentId = i # For readability # Find out what forward entry we're in unindexEntry = unindex.get(documentId, None) if unindexEntry is None: return None # Now find that forward entry and try and remove us from it indexSet = index.get(unindexEntry, None) if indexSet is not None: try: indexSet.remove(i) except: LOG('UnIndex', ERROR, ('unindex_object could not remove ' 'integer id %s from index %s. This ' 'should not happen.' % (str(i), str(k)))) else: LOG('UnIndex', ERROR, ('unindex_object tried to retrieve set %s ' 'from index %s but couldn\'t. This ' 'should not happen.' % (repr(set),str(k)))) # Now that we've removed all the forward entries, we can remove the # reverse entry del unindex[documentId] # Kick the persistence machienery since this is a mutable object self._index = index self._unindex = unindex def _apply_index(self, request, cid='', ListType=type([])): """Apply the index to query parameters given in the argument, request The argument should be a mapping object, with cid being the container id used to distinguish search items from the Search Interface. If the request does not contain the needed parameters, then None is returned. If the request contains a parameter with the name of the column + '_usage', it is sniffed for information on how to handle applying the index. Otherwise two objects are returned. The first object is a ResultSet containing the record numbers of the matching records. The second object is a tuple containing the names of all data fields used. """ indexName = self.id # Name of the index, a shortcut # First look for the "classified" search item, if you can't # find that, then look for the unclassified, otherwise # there's nothing of interest for us. cidid = "%s/%s" % (cid, indexName) if request.has_key("%s/%s" % (cid, indexName)): keys = request[cidid] elif request.has_key(indexName): keys = request[indexName] else: return None if type(keys) is not ListType: keys=[keys] index = self._index results = None anyTrue = 0 opr = None if request.has_key(indexName + '_usage'): # see if any usage params are sent to field opr = string.split(string.lower( request[indexName + "_usage"]), ':') opr, opr_args = opr[0], opr[1:] if opr == "range": if 'min' in opr_args: lo = min(keys) else: lo = None if 'max' in opr_args: hi = max(keys) else: hi = None anyTrue = 1 try: if hi: setlist = index.items(lo, hi) else: setlist = index.items(lo) for k, set in setlist: if results is None: results = set else: results = result.union(set) except KeyError: pass else: #not a range for key in keys: if key: anyTrue = 1 set = index.get(key) if set is not None: if results is None: results = set else: results = result.union(set) if result is None: if anyTrue: results = intSet() else: return None return results, (indexName,) def hasUniqueValuesFor(self, name): """ has unique values for column NAME """ if name == self.id: return 1 else: return 0 def uniqueValues(self, name=None, withLengths=0): """returns the unique values for name if withLengths is true, returns a sequence of tuples of (value, length) """ resultList = [] if name is None: name = self.id elif name != self.id: return () if not withLengths: resultList = filter(nonEmpty, self._index.keys()) else: for key in self._index.keys(): if not nonEmpty(key): continue else: resultList.append( (key, len(self._index[key]) ) ) return tuple(resultList) def clear(self): self._index = BTree.BTree() self._unindex = IOBTree.BTree() lib/python/SearchIndex/UnKeywordIndex.py100644 1007 144 14151 7217443051 20117 0ustar petrilliusers# # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## from UnIndex import UnIndex, intSet from zLOG import LOG, ERROR class UnKeywordIndex(UnIndex): meta_type = 'Keyword Index' """Like an UnIndex only it indexes sequences of items Searches match any keyword. This should have an _apply_index that returns a relevance score """ def index_object(self, i, obj, threshold=None): """ index an object 'obj' with integer id 'i'""" # First let's set up some short cuts to save typing, and name # lookup costs in the future. indexName = self.id # This is the name of the index documentId = i # Keep this for more readable use index = self._index # Forward index unindex = self._unindex # Reverse index # We need to first unindex the existing record so we don't get # KeyErrors in the future. self.unindex_object(documentId) try: keywords = getattr(obj, indexName) if callable(keywords): keywords = keywords() except: return 0 # index each item in the sequence for kw in keywords: indexSet = index.get(kw) if indexSet is None: index[kw] = indexSet = intSet() indexSet.insert(documentId) unindex[documentId] = keywords # Kick the persiste4nce machienery since we're using a mutable object self._index = index self._unindex = unindex return 1 def unindex_object(self, i): """ carefully unindex the object with integer id 'i' and do not fail if it does not exist """ # First let's set up some short cuts to save typing, and name # lookup costs in the future. documentId = i # Keep this for more readable use index = self._index # Forward index unindex = self._unindex # Reverse index keywords = unindex.get(documentId, None) if keywords is None: return None for kw in keywords: indexSet = index.get(kw, None) if set is not None: indexSet.remove(documentId) else: LOG('UnKeywordIndex', ERROR, ('unindex_object could not ' 'remove %s from set' % str(i))) del unindex[documentId] # Kick the persistence machienery self._index = index self._unindex = unindex lib/python/SearchIndex/UnTextIndex.py100644 1007 144 57167 7217461253 17441 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## """Text Index The UnTextIndex falls under the 'I didnt have a better name for it' excuse. It is an 'Un' Text index because it stores a little bit of undo information so that objects can be unindexed when the old value is no longer known. """ __version__='$Revision: 1.29.12.4 $'[11:-2] import BTree, IIBTree, IOBTree, OIBTree import operator import string, regex, regsub, ts_regex from Globals import Persistent from Acquisition import Implicit from intSet import intSet from Splitter import Splitter from string import strip from zLOG import LOG, ERROR from Lexicon import Lexicon, stop_word_dict from ResultList import ResultList # Set up local aliases because the class name is the same in each one. BTree = BTree.BTree IOBTree = IOBTree.BTree IIBucket = IIBTree.Bucket OIBTree = OIBTree.BTree AndNot = 'andnot' And = 'and' Or = 'or' Near = '...' QueryError='TextIndex.QueryError' class UnTextIndex(Persistent, Implicit): meta_type = 'Text Index' def __init__(self, id=None, ignore_ex=None, call_methods=None, lexicon=None): """Create an index The arguments are: 'id' -- the name of the item attribute to index. This is either an attribute name or a record key. 'ignore_ex' -- Tells the indexer to ignore exceptions that are rasied when indexing an object. 'call_methods' -- Tells the indexer to call methods instead of getattr or getitem to get an attribute. 'lexicon' is the lexicon object to specify, if None, the index will use a private lexicon. There is a ZCatalog UML model that sheds some light on what is going on here. '_index' is a BTree which maps word ids to mapping from document id to score. Something like: {'bob' : {1 : 5, 2 : 3, 42 : 9}} {'uncle' : {1 : 1}} The '_unindex' attribute is a mapping from document id to word ids. This mapping allows the catalog to unindex an object: {42 : ('bob', 'is', 'your', 'uncle') This isn't exactly how things are represented in memory, many optimizations happen along the way. """ if not id==ignore_ex==call_methods==None: self.id=id self.ignore_ex=ignore_ex self.call_methods=call_methods self._index=IOBTree() self._unindex=IOBTree() else: pass if lexicon is None: ## if no lexicon is provided, create a dumb one self._lexicon=Lexicon() else: self._lexicon = lexicon def __setstate(self, state): Persistent.__setstate__(self, state) if hasattr(self, '_syn'): del self._syn def getLexicon(self, vocab_id): """ bit of a hack, indexes have been made acquirers so that they can acquire a vocabulary object from the object system in Zope. I don't think indexes were ever intended to participate in this way, but I don't see too much of a problem with it. """ if type(vocab_id) is not type(""): vocab = vocab_id else: vocab = getattr(self, vocab_id) return vocab.lexicon def __len__(self): return len(self._unindex) def clear(self): # Re-initialize the trees to empty self._index = IOBTree() self._unindex = IOBTree() def index_object(self, i, obj, threshold=None, TupleType=type(()), DictType=type({}), callable=callable): """ Index an object: 'i' is the integer id of the document 'obj' is the object to be indexed 'threshold' is the number of words to process between commiting subtransactions. If 'None' subtransactions are not used. the next four arguments are default optimizations. """ # First let's set up some short cuts to save typing, and name # lookup costs in the future. indexName = self.id # This is the name of the index documentId = i # Keep this for more readable use index = self._index # Forward index unindex = self._unindex # Reverse index lexicon = self.getLexicon(self._lexicon) # Lexicon # If we've gotten this far, then we need to first unindex the existing # record so we don't get KeyErrors in the future... EVIL. self.unindex_object(documentId) # Check to see if the object to be indexed has something we're # interested in, and deal with it, calling it if necessary. try: datum = getattr(obj, indexName) if callable(datum): datum = str(datum()) else: datum = str(datum) except AttributeError: # Nope, doesn't have it return 0 # Get a list of stemmed words, using the built-in splitter # Note that this will also remove all stop words stemmedWords = self.getLexicon(self._lexicon).Splitter(datum) # wordTree is the data structure that holds a list of words to # be indexed. This is done for scalability of large documents. wordTree = OIBTree() lastWordIndexed = None for word in stemmedWords: if word[0] == '\"': lastWordIndexed = self.subindex(word[1:-1], wordTree, wordTree.has_key, last) else: if wordTree.has_key(word): # If we've already got the word in our tree if word != lastWordIndexed: # If it's not a duplicate, increment its score wordTree[word] = wordTree[word]+1 else: # Add it to the tree and set its score to 1 wordTree[word] = 1 # Initialize the reverse index (for unindexing) unindex[documentId] = [] # Number of words counted. This is also used for triggering # the subtransaction machienery. wordCount = 0 # Now we need to go through the entire wordTree and actually deal # with each individual entry for word, score in wordTree.items(): if threshold is not None: # If we're using subtransactions, check them... if (wordCount % threshold == 1) and (wordCount != 0): # commit a subtransaction get_transaction().commit(1) # kick the cache self._p_jar.cacheFullSweep(1) # Make sure this word is in the lexicon, which returns the 'id' # that the con uses to identify the word lexiconWordId = lexicon.set(word) # Try and get the row out of the index for this word indexRow = index.get(lexiconWordId) if indexRow is not None: if type(indexRow) is TupleType: indexRow = { indexRow[0] : indexRow[1] } # Dict indexRow[documentId] = score index[lexiconWordId] = indexRow unindex[documentId].append(lexiconWordId) elif type(indexRow) is DictType: if len(indexRow) > 4: indexRowBucket = IIBucket() for k, v in indexRow.items(): indexRowBucket[k] = v indexRow = indexRowBucket indexRow[documentId] = score index[lexiconWordId] = indexRow unindex[documentId].append(lexiconWordId) else: indexRow[documentId] = score unindex[documentId].append(lexiconWordId) else: index[lexiconWordId] = documentId, score unindex[documentId].append(lexiconWordId) wordCount = wordCount + 1 unindex[documentId] = tuple(unindex[documentId]) # l = len(unindex[documentId]) # This reassignment kicks the persistence machienery. Since this # is a mutable object, we have to treat it immutably. self._index = index self._unindex = unindex # Return the number of words indexed return wordCount def unindex_object(self, i, TupleType=type(()) ): """ carefully unindex document with integer id 'i' from the text index and do not fail if it does not exist """ # Grab our own copies, this is to help make sure the persistence # machienery is notified. index = self._index # Forward index unindex = self._unindex # Reverse index # First, grab the entry for this record from the reverse index. # This index has a list of forward index entries we need to update. entriesToUpdate = unindex.get(i, None) if entriesToUpdate is not None: # Now, loop through each and kill it for entry in entriesToUpdate: # Retrieve "row" of the forward index that includes references # to that which is being unindexed indexRow = index.get(entry, None) if type(indexRow) is TupleType: del index[entry] elif indexRow is not None: try: del index[entry][i] except (KeyError, IndexError, TypeError): LOG('UnTextIndex', ERROR, 'unindex_object tried to unindex nonexistent' ' document %s' % str(i)) # Now delete the row from the reverse index and reassign to # trigger the persistence machienery. del unindex[i] self._index = index self._unindex = unindex def __getitem__(self, word): """ Return an InvertedIndex-style result "list" """ stemmedWords = tuple(self.getLexicon(self._lexicon).Splitter(word)) if not stemmedWords: return ResultList({}, (word,), self) if len(stemmedWords) == 1: stemmedWords = stemmedWords[0] if stemmedWords[:1]=='"' and stemmedWords[-1:]=='"': return self[stemmedWords] result = self._index.get( self.getLexicon(self._lexicon).get(stemmedWords)[0], None) if result is None: result = {} return ResultList(result, (stemmedWords,), self) result = None for aWord in stemmedWords: rr = self[word] if result is None: result = rr else: result = result.near(rr) return result def _apply_index(self, request, cid='', ListType=[]): """ Apply the index to query parameters given in the argument, request The argument should be a mapping object, with 'cid' being the container id used to distinguish search items from the Search Interface. If the request does not contain the needed parameters, then None is returned. Otherwise two objects are returned. The first object is a ResultSet containing the record numbers of the matching records. The second object is a tuple containing the names of all data fields used. """ indexName = self.id # Name of the index # First look for the "classified" search item, if you can't # find that, then look for the unclassified, otherwise # there's nothing of interest for us. cidid = "%s/%s" % (cid, indexName) if request.has_key("%s/%s" % (cid, indexName)): keys = request[cidid] elif request.has_key(indexName): keys = request[indexName] else: return None if type(keys) is type(''): if not keys or not strip(keys): return None keys = [keys] results = None for key in keys: key = strip(key) if not key: continue rr = IIBucket() try: for i, score in self.query(key).items(): if score: rr[i] = score except KeyError: pass if results is None: results = rr else: # Note that we *and*/*narrow* multiple search terms. results = results.intersection(rr) if results is not None: return results, (indexName,) return IIBucket(), (indexName,) def positions(self, docid, words, obj): """Return the positions in the document for the given document id of the word, word.""" id = self.id if self._schema is None: f = getattr else: f = operator.__getitem__ id = self._schema[id] if self.call_methods: doc = str(f(obj, id)()) else: doc = str(f(obj, id)) r = [] for word in words: r = r+self.getLexicon(self._lexicon).Splitter(doc).indexes(word) return r def _subindex(self, isrc, d, old, last): stemmedWords = self.getLexicon(self._lexicon).Splitter(isrc) for word in stemmedWords: if word[0] == '\"': last = self.subindex(word[1:-1],d,old,last) else: if old(word): if word != last: d[word] = d[word]+1 else: d[word] = 1 return last def query(self, s, default_operator = Or, ws = (string.whitespace,)): """ This is called by TextIndexes. A 'query term' which is a string 's' is passed in, along with an index object. s is parsed, then the wildcards are parsed, then something is parsed again, then the whole thing is 'evaluated' """ # First replace any occurences of " and not " with " andnot " s = ts_regex.gsub( '[%s]+[aA][nN][dD][%s]*[nN][oO][tT][%s]+' % (ws * 3), ' andnot ', s) # do some parsing q = parseFirstPass(s) ## here, we give lexicons a chance to transform the query. ## For example, substitute wildcards, or translate words into ## various languages. q = self.getLexicon(self._lexicon).query_hook(q) # do some more parsing q = parseSecondPass(q, default_operator) ## evalute the final 'expression' return self.evaluate(q) def get_operands(self, q, i, ListType=type([]), StringType=type('')): '''Evaluate and return the left and right operands for an operator''' try: left = q[i - 1] right = q[i + 1] except IndexError: raise QueryError, "Malformed query" t = type(left) if t is ListType: left = evaluate(left, self) elif t is StringType: left = self[left] t = type(right) if t is ListType: right = evaluate(right, self) elif t is StringType: right = self[right] return (left, right) def evaluate(self, q, ListType=type([])): '''Evaluate a parsed query''' ## import pdb ## pdb.set_trace() if (len(q) == 1): if (type(q[0]) is ListType): return evaluate(q[0], self) return self[q[0]] i = 0 while (i < len(q)): if q[i] is AndNot: left, right = self.get_operands(q, i) val = left.and_not(right) q[(i - 1) : (i + 2)] = [ val ] else: i = i + 1 i = 0 while (i < len(q)): if q[i] is And: left, right = self.get_operands(q, i) val = left & right q[(i - 1) : (i + 2)] = [ val ] else: i = i + 1 i = 0 while (i < len(q)): if q[i] is Or: left, right = self.get_operands(q, i) val = left | right q[(i - 1) : (i + 2)] = [ val ] else: i = i + 1 i = 0 while (i < len(q)): if q[i] is Near: left, right = self.get_operands(q, i) val = left.near(right) q[(i - 1) : (i + 2)] = [ val ] else: i = i + 1 if (len(q) != 1): raise QueryError, "Malformed query" return q[0] def parseFirstPass(inputString): '''Parse parentheses and quotes''' # First convert it to lower case so that we're only dealing with # things in a case-insensitive way inputString = string.lower(inputString) l = [] while (1): listOfParens = parens(inputString) if (listOfParens is None): # No parentheses found. Look for quotes then exit. l = l + quotes(inputString) break else: # Look for quotes in the section of the string before # the parentheses, then parse the string inside the parens l = l + quotes(inputString[:(listOfParens[0] - 1)]) l.append(parseFirstPass(inputString[listOfParens[0] : listOfParens[1]])) # continue looking through the rest of the string inputString = inputString[(listOfParens[1] + 1):] return l def parseSecondPass(q, default_operator, operator_dict = {AndNot: AndNot, And: And, Or: Or, Near: Near}, ListType=type([]), ): '''Find operators and operands''' i = 0 isop=operator_dict.has_key while (i < len(q)): if (type(q[i]) is ListType): q[i] = parseSecondPass(q[i], default_operator) # every other item, starting with the first, should be an operand if ((i % 2) != 0): # This word should be an operator; if it is not, splice in # the default operator. if type(q[i]) is not ListType and isop(q[i]): q[i] = operator_dict[q[i]] else: q[i : i] = [ default_operator ] i = i + 1 return q def parens(s, parens_re = regex.compile('(\|)').search): index=open_index=paren_count = 0 while 1: index = parens_re(s, index) if index < 0 : break if s[index] == '(': paren_count = paren_count + 1 if open_index == 0 : open_index = index + 1 else: paren_count = paren_count - 1 if paren_count == 0: return open_index, index else: index = index + 1 if paren_count == 0: # No parentheses Found return None else: raise QueryError, "Mismatched parentheses" def quotes(s, ws = (string.whitespace,)): # split up quoted regions splitted = ts_regex.split(s, '[%s]*\"[%s]*' % (ws * 2)) split=string.split if (len(splitted) > 1): if ((len(splitted) % 2) == 0): raise QueryError, "Mismatched quotes" for i in range(1,len(splitted),2): # split the quoted region into words splitted[i] = filter(None, split(splitted[i])) # put the Proxmity operator in between quoted words for j in range(1, len(splitted[i])): splitted[i][j : j] = [ Near ] for i in range(len(splitted)-1,-1,-2): # split the non-quoted region into words splitted[i:i+1] = filter(None, split(splitted[i])) splitted = filter(None, splitted) else: # No quotes, so just split the string into words splitted = filter(None, split(s)) return splitted lib/python/SearchIndex/__init__.py100644 1007 144 7605 6671334650 16734 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## __doc__='''Collected utilities to support database indexing. $Id: __init__.py,v 1.6 1999/03/10 00:15:36 klm Exp $''' __version__='$Revision: 1.6 $'[11:-2] import Query lib/python/SearchIndex/config.c100644 1007 144 6676 7216441050 16230 0ustar petrilliusers/* Generated automatically from /usr/local/python1.5.2/lib/python1.5/config/config.c.in by makesetup. */ /* -*- C -*- *********************************************** Copyright 1991-1995 by Stichting Mathematisch Centrum, Amsterdam, The Netherlands. All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that copyright notice and this permission notice appear in supporting documentation, and that the names of Stichting Mathematisch Centrum or CWI or Corporation for National Research Initiatives or CNRI not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. While CWI is the initial source for this software, a modified version is made available by the Corporation for National Research Initiatives (CNRI) at the Internet address ftp://ftp.python.org. STICHTING MATHEMATISCH CENTRUM AND CNRI DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM OR CNRI BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ******************************************************************/ /* Module configuration */ /* !!! !!! !!! This file is edited by the makesetup script !!! !!! !!! */ /* This file contains the table of built-in modules. See init_builtin() in import.c. */ #include "Python.h" extern void initthread(); extern void initregex(); extern void initpcre(); extern void initposix(); extern void initsignal(); extern void initarray(); extern void initcmath(); extern void initmath(); extern void initstrop(); extern void initstruct(); extern void inittime(); extern void initoperator(); extern void initfcntl(); extern void initpwd(); extern void initgrp(); extern void initselect(); extern void initsocket(); extern void initerrno(); extern void initmd5(); extern void initsha(); extern void initrotor(); extern void initnew(); extern void initbinascii(); extern void initparser(); extern void initcStringIO(); extern void initcPickle(); /* -- ADDMODULE MARKER 1 -- */ extern void PyMarshal_Init(); extern void initimp(); struct _inittab _PyImport_Inittab[] = { {"thread", initthread}, {"regex", initregex}, {"pcre", initpcre}, {"posix", initposix}, {"signal", initsignal}, {"array", initarray}, {"cmath", initcmath}, {"math", initmath}, {"strop", initstrop}, {"struct", initstruct}, {"time", inittime}, {"operator", initoperator}, {"fcntl", initfcntl}, {"pwd", initpwd}, {"grp", initgrp}, {"select", initselect}, {"socket", initsocket}, {"errno", initerrno}, {"md5", initmd5}, {"sha", initsha}, {"rotor", initrotor}, {"new", initnew}, {"binascii", initbinascii}, {"parser", initparser}, {"cStringIO", initcStringIO}, {"cPickle", initcPickle}, /* -- ADDMODULE MARKER 2 -- */ /* This module "lives in" with marshal.c */ {"marshal", PyMarshal_Init}, /* This lives it with import.c */ {"imp", initimp}, /* These entries are here for sys.builtin_module_names */ {"__main__", NULL}, {"__builtin__", NULL}, {"sys", NULL}, /* Sentinel */ {0, 0} }; lib/python/Products/ 40755 1007 144 0 7217466751 14131 5ustar petrilliuserslib/python/Products/ZCatalog/ 40755 1007 144 0 7217467075 15635 5ustar petrilliuserslib/python/Products/ZCatalog/Catalog.gif100644 1007 144 265 6714567700 17753 0ustar petrilliusersGIF89aƄ!,@bPHBmFOE0E"h"b[JI,Ljq9ieA8XL!FB/TRMxz/R0›a:#JOP |zT6V1;lib/python/Products/ZCatalog/Catalog.py100644 1007 144 54572 7217465163 17707 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## __version__ = "$Revision: 1.57.2.2 $"[11:-2] from Persistence import Persistent import Acquisition import ExtensionClass import BTree, OIBTree, IOBTree, IIBTree IIBucket=IIBTree.Bucket from intSet import intSet from SearchIndex import UnIndex, UnTextIndex, UnKeywordIndex, Query from SearchIndex.Lexicon import Lexicon import regex, pdb from MultiMapping import MultiMapping from string import lower from Missing import MV from zLOG import LOG, ERROR from Lazy import LazyMap, LazyFilter, LazyCat from CatalogBrains import AbstractCatalogBrain, NoBrainer class KWMultiMapping(MultiMapping): def has_key(self, name): try: r=self[name] return 1 except KeyError: return 0 def orify(seq, query_map={ type(regex.compile('')): Query.Regex, type(''): Query.String, }): subqueries=[] for q in seq: try: q=query_map[type(q)](q) except: q=Query.Cmp(q) subqueries.append(q) return apply(Query.Or,tuple(subqueries)) class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): """ An Object Catalog An Object Catalog maintains a table of object metadata, and a series of manageable indexes to quickly search for objects (references in the metadata) that satisfy a search query. This class is not Zope specific, and can be used in any python program to build catalogs of objects. Note that it does require the objects to be Persistent, and thus must be used with ZODB3. """ # Default brain class _v_brains = NoBrainer def __init__(self, vocabulary=None, brains=None): self.schema = {} # mapping from attribute name to column number self.names = () # sequence of column names self.indexes = {} # mapping from index name to index object # The catalog maintains a BTree of object meta_data for # convenient display on result pages. meta_data attributes # are turned into brain objects and returned by # searchResults. The indexing machinery indexes all records # by an integer id (rid). self.data is a mapping from the # integer id to the meta_data, self.uids is a mapping of the # object unique identifier to the rid, and self.paths is a # mapping of the rid to the unique identifier. self.data = BTree.BTree() # mapping of rid to meta_data self.uids = OIBTree.BTree() # mapping of uid to rid self.paths = IOBTree.BTree() # mapping of rid to uid # indexes can share a lexicon or have a private copy. Here, # we instantiate a lexicon to be shared by all text indexes. # This may change. if type(vocabulary) is type(''): self.lexicon = vocabulary else: #ack! self.lexicon = Lexicon() if brains is not None: self._v_brains = brains self.updateBrains() def updateBrains(self): self.useBrains(self._v_brains) def __setstate__(self, state): """ initialize your brains. This method is called when the catalog is first activated (from the persistent storage) """ Persistent.__setstate__(self, state) self.updateBrains() if not hasattr(self, 'lexicon'): self.lexicon = Lexicon() def useBrains(self, BrainClass): """ Set up the Catalog to wrap results, so that the tuple stored in the self.data Btree can be accessed by name, as specified in the catalog schema. Allow additional result record behaviors to be specified in a BrainClass. """ # Copy the catalog schema and extend it with standard brain data scopy = self.schema.copy() nextslot = len(scopy) scopy['data_record_id_'] = nextslot scopy['data_record_score_'] = nextslot + 1 scopy['data_record_normalized_score_'] = nextslot + 2 # We need to subclass BrainClass so that we have a place to # provide shared instance data (such as the schema) without # mutating BrainClass or copying the the data into all of the # instances. We also mix in standard brain behavior so that # BrainClass doesn't need to be a subclass. class CustomBrain(AbstractCatalogBrain, BrainClass): pass CustomBrain.__record_schema__ = scopy # Save both classes for future reference self._v_brains = BrainClass self._v_result_class = CustomBrain def addColumn(self, name, default_value=None): """ adds a row to the meta data schema """ schema = self.schema names = list(self.names) if schema.has_key(name): raise 'Column Exists', 'The column exists' if name[0] == '_': raise 'Invalid Meta-Data Name', \ 'Cannot cache fields beginning with "_"' if not schema.has_key(name): if schema.values(): schema[name] = max(schema.values())+1 else: schema[name] = 0 names.append(name) if default_value is None or default_value == '': default_value = MV for key in self.data.keys(): rec = list(self.data[key]) rec.append(default_value) self.data[key] = tuple(rec) self.names = tuple(names) self.schema = schema # new column? update the brain self.updateBrains() def delColumn(self, name): """ deletes a row from the meta data schema """ names = list(self.names) _index = names.index(name) if not self.schema.has_key(name): LOG('Catalog', ERROR, ('delColumn attempted to delete ' 'nonexistent column %s.' % str(name))) return names.remove(name) # rebuild the schema i=0; schema = {} for name in names: schema[name] = i i = i + 1 self.schema = schema self.names = tuple(names) # update the brain self.updateBrains() # remove the column value from each record for key in self.data.keys(): rec = list(self.data[key]) rec.remove(rec[_index]) self.data[key] = tuple(rec) def addIndex(self, name, type): """Create a new index, of one of the following types Types: 'FieldIndex', 'TextIndex', 'KeywordIndex'. """ if self.indexes.has_key(name): raise 'Index Exists', 'The index specified already exists' if name[0] == '_': raise 'Invalid Index Name', 'Cannot index fields beginning with "_"' # this is currently a succesion of hacks. Indexes should be # pluggable and managable indexes = self.indexes if type == 'FieldIndex': indexes[name] = UnIndex.UnIndex(name) elif type == 'TextIndex': indexes[name] = UnTextIndex.UnTextIndex(name, None, None, self.lexicon) elif type == 'KeywordIndex': indexes[name] = UnKeywordIndex.UnKeywordIndex(name) else: raise 'Unknown Index Type', ("%s invalid - must be one of %s" % (type, ['FieldIndex', 'TextIndex', 'KeywordIndex'])) self.indexes = indexes def delIndex(self, name): """ deletes an index """ if not self.indexes.has_key(name): raise 'No Index', 'The index specified does not exist' indexes = self.indexes del indexes[name] self.indexes = indexes # the cataloging API def catalogObject(self, object, uid, threshold=None): """ Adds an object to the Catalog by iteratively applying it all indexes. 'object' is the object to be cataloged 'uid' is the unique Catalog identifier for this object """ data = self.data if self.uids.has_key(uid): i = self.uids[uid] elif data: i = data.keys()[-1] + 1 # find the next available unique id else: i = 0 self.uids[uid] = i self.paths[i] = uid # meta_data is stored as a tuple for efficiency data[i] = self.recordify(object) total = 0 for x in self.indexes.values(): ## tricky! indexes need to acquire now, and because they ## are in a standard dict __getattr__ isn't used, so ## acquisition doesn't kick in, we must explicitly wrap! x = x.__of__(self) if hasattr(x, 'index_object'): blah = x.index_object(i, object, threshold) total = total + blah else: LOG('Catalog', ERROR, ('catalogObject was passed ' 'bad index object %s.' % str(x))) self.data = data return total def uncatalogObject(self, uid): """ Uncatalog and object from the Catalog. and 'uid' is a unique Catalog identifier Note, the uid must be the same as when the object was catalogued, otherwise it will not get removed from the catalog This method should not raise an exception if the uid cannot be found in the catalog. """ data = self.data uids = self.uids paths = self.paths indexes = self.indexes rid = uids.get(uid, None) if rid is not None: for x in indexes.values(): x = x.__of__(self) if hasattr(x, 'unindex_object'): x.unindex_object(rid) # this should never raise an exception for btree in (data, paths): try: del btree[rid] except KeyError: LOG('Catalog', ERROR, ('uncatalogObject unsuccessfully ' 'attempted to delete rid %s ' 'from paths or data btree.' % rid)) del uids[uid] self.data = data else: LOG('Catalog', ERROR, ('uncatalogObject unsuccessfully ' 'attempted to uncatalog an object ' 'with a uid of %s. ' % str(uid))) def clear(self): """ clear catalog """ self.data = BTree.BTree() self.uids = OIBTree.BTree() self.paths = IOBTree.BTree() for x in self.indexes.values(): x.clear() # Finally, clear the lexicon too... if type(self.lexicon) is type(''): getattr(self, self.lexicon).clear() else: self.lexicon.clear() def uniqueValuesFor(self, name): """ return unique values for FieldIndex name """ return self.indexes[name].uniqueValues() def hasuid(self, uid): """ return the rid if catalog contains an object with uid """ if self.uids.has_key(uid): return self.uids[uid] else: return None def recordify(self, object): """ turns an object into a record tuple """ record = [] # the unique id is allways the first element for x in self.names: try: attr = getattr(object, x) if(callable(attr)): attr = attr() except: attr = MV record.append(attr) return tuple(record) def buildWrappedRecord(self, index, score=1, normalized_score=1): """ Fetch and wrap a record using the brain class. """ r=self._v_result_class(self.data[index]) r.data_record_id_ = index r.data_record_score_ = score r.data_record_normalized_score_ = normalized_score # Wrap the record in the acquisition context of the Catalog's # parent (the ZCatalog). return r.__of__(self.aq_parent) __getitem__ = buildWrappedRecord def _indexedSearch(self, args, sort_index, append, used, IIBType=type(IIBucket()), intSType=type(intSet())): """ Iterate through the indexes, applying the query to each one. Do some magic to join result sets. Be intelligent about handling intSets and IIBuckets. """ ## import pdb ## pdb.set_trace() ## I use this so much I'm just leaving it commented out -michel rs=None data=self.data if used is None: used={} for i in self.indexes.keys(): try: index = self.indexes[i].__of__(self) if hasattr(index,'_apply_index'): r=index._apply_index(args) if r is not None: r, u = r for name in u: used[name]=1 if rs is None: rs = r else: # you can't intersect an IIBucket into an # intSet, but you can go the other way # around. Make sure we're facing the # right direction... if type(rs) is intSType and type(r) is IIBType: rs=r.intersection(rs) else: rs=rs.intersection(r) except: return used if rs is None: if sort_index is None: rs=data.keys() append(LazyMap(self.buildWrappedRecord, rs)) else: for k, intset in sort_index._index.items(): append((k,LazyMap(self.buildWrappedRecord, intset))) elif rs: if sort_index is None and type(rs) is IIBType: # then there is score information. Build a new result # set, sort it by score, reverse it, compute the # normalized score, and Lazify it. rset = [] for key, score in rs.items(): rset.append((score, key)) rset.sort() rset.reverse() max = float(rset[0][0]) rs = [] for score, key in rset: rs.append( (key, score, int((score/max)*100), ) ) append(LazyMap(self.buildWrappedRecord, rs)) elif sort_index is None and type(rs) is intSType: # no scores? Just Lazify. append(LazyMap(self.buildWrappedRecord, rs)) else: # sort. If there are scores, then this block is not # reached, therefor 'sort-on' does not happen in the # context of text index query. This should probably # sort by relevance first, then the 'sort-on' attribute. if len(rs)>len(sort_index._index): for k, intset in sort_index._index.items(): if type(rs) is IIBType: intset=rs.intersection(intset) # Since we still have an IIBucket, let's convert # it to its set of keys intset=intset.keys() else: intset=intset.intersection(rs) if intset: append( (k, LazyMap(self.buildWrappedRecord, intset), ) ) else: if type(rs) is IIBType: rs=rs.keys() for r in rs: append( (sort_index._unindex[r], LazyMap(self.buildWrappedRecord,[r]) ) ) return used def searchResults(self, REQUEST=None, used=None, query_map={ type(regex.compile('')): Query.Regex, type([]): orify, type(''): Query.String, }, **kw): # Set up search arguments # If we have neither a request, nor a set of search arguments. if REQUEST is None and not kw: try: REQUEST = self.REQUEST except: pass # If we got extra keyword arguments if kw: if REQUEST: m = KWMultiMapping() # Let's stack multiple mappings m.push(REQUEST) m.push(kw) kw = m # Otherwise just use the REQUEST object iself. elif REQUEST: kw = REQUEST # Make sure batch size is set if REQUEST and not REQUEST.has_key('batch_size'): try: batch_size = self.default_batch_size except: batch_size = 20 REQUEST['batch_size'] = batch_size # Compute "sort_index", which is a sort index, or none: if kw.has_key('sort-on'): sort_index = kw['sort-on'] elif hasattr(self, 'sort-on'): sort_index = getattr(self, 'sort-on') elif kw.has_key('sort_on'): sort_index = kw['sort_on'] else: sort_index = None sort_order='' if (sort_index is not None and sort_index in self.indexes.keys()): sort_index = self.indexes[sort_index] # Perform searches with indexes and sort_index r = [] used=self._indexedSearch(kw, sort_index, r.append, used) if not r: return r # Sort/merge sub-results if len(r) == 1: if sort_index is None: r = r[0] else: r = r[0][1] else: if sort_index is None: r = LazyCat(r) else: r.sort() if kw.has_key('sort-order'): so = kw['sort-order'] elif hasattr(self, 'sort-order'): so = getattr(self, 'sort-order') elif kw.has_key('sort_order'): so = kw['sort_order'] else: so = None if (type(so) is type('') and lower(so) in ('reverse', 'descending')): r.reverse() r = LazyCat(map(lambda i: i[1], r)) return r __call__ = searchResults lib/python/Products/ZCatalog/CatalogAwareness.py100644 1007 144 23410 7217441723 21537 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## """ZCatalog Findable class""" import urllib, string, sys from Globals import HTMLFile from Acquisition import aq_base class CatalogAware: """ This is a Mix-In class to make objects automaticly catalog and uncatalog themselves in Zope, and to provide some other basic attributes that are useful to catalog. Note that if your class or ZClass subclasses CatalogAware, it will only catalog itself after the following actions: - Add - Clone - Delete If you make changes to your own object, you are responsible for calling your object's index_object method. """ meta_type = 'CatalogAware' default_catalog = 'Catalog' manage_editCatalogerForm=HTMLFile('editCatalogerForm', globals()) def manage_editCataloger(self, default, REQUEST=None): """Update the internal settings that are used to figure out which ZCatalog to index into.""" self.default_catalog = default message = "Your changes have been saved" if REQUEST is not None: return self.manage_main(self, REQUEST, manage_tabs_message=message) def manage_afterAdd(self, item, container): """This is the hook called after an object has been added.""" self.index_object() for object in self.objectValues(): # First we need to find out the current state of the object try: objectState = object._p_changed except: objectState = 0 # Now look to see if it has the hook we want, and call it try: if hasattr(aq_base(object), 'manage_afterAdd'): object.manage_afterAdd(item, container) except BeforeDeleteException, ob: raise except: LOG('Zope', ERROR, 'manage_beforeDelete() threw', error=sys.exc_info()) # If objectState is None, then the object was a ghost when we # called it and we need to re-ghost it. if objectState is None: object._p_deactivate() def manage_afterClone(self, item): """This is the hook called when an object needs to be cloned. This is part of the copy support and factories.""" self.index_object() for object in self.objectValues(): # First we need to find out the current state of the object try: objectState = object._p_changed except: objectState = 0 # Now look to see if it has the hook we want, and call it try: if hasattr(aq_base(object), 'manage_afterClone'): object.manage_afterClone(item, container) except BeforeDeleteException, ob: raise except: LOG('Zope', ERROR, 'manage_afterClone() threw', error=sys.exc_info()) # If objectState is None, then the object was a ghost when we # called it and we need to re-ghost it. if objectState is None: object._p_deactivate() def manage_beforeDelete(self, item, container): """The hook called right before deletion happens.""" self.unindex_object() for object in self.objectValues(): # First we need to find out the current state of the object try: objectState = object._p_changed except: objectState = 0 # Now look to see if it has the hook we want, and call it try: if hasattr(aq_base(object), 'manage_beforeDelete'): object.manage_beforeDelete(item, container) except BeforeDeleteException, ob: raise except: LOG('Zope', ERROR, 'manage_beforeDelete() threw', error=sys.exc_info()) # If objectState is None, then the object was a ghost when we # called it and we need to re-ghost it. if objectState is None: object._p_deactivate() def creator(self): """Return a sequence of user names who have the local Owner role on an object. The name creator is used for this method to conform to Dublin Core.""" users=[] for user, roles in self.get_local_roles(): if 'Owner' in roles: users.append(user) return string.join(users, ', ') def onDeleteObject(self): """Object delete handler. I think this is obsoleted by manage_beforeDelete """ self.unindex_object() def getPath(self): """Return the physical path for an object.""" return string.join(self.getPhysicalPath(), '/') def summary(self, summaryLength=200): """Return a summary of the text content of the object.""" if not hasattr(self, 'text_content'): return '' attr = getattr(self, 'text_content') if callable(attr): text = attr() else: text = attr n = min(summaryLength, len(text)) return text[:n] def index_object(self): """A common method to allow Findables to index themselves.""" if hasattr(self, self.default_catalog): getattr(self, self.default_catalog).catalog_object(self, self.getPath()) def unindex_object(self): """A common method to allow Findables to unindex themselves.""" if hasattr(self, self.default_catalog): getattr(self, self.default_catalog).uncatalog_object(self.getPath()) def reindex_object(self): """ Suprisingly useful """ self.unindex_object() self.index_object() def reindex_all(self, obj=None): """ """ if obj is None: obj=self if hasattr(aq_base(obj), 'index_object'): obj.index_object() if hasattr(aq_base(obj), 'objectValues'): for item in obj.objectValues(): reindex_all(item) lib/python/Products/ZCatalog/CatalogAwarenessInterface.py100644 1007 144 2275 7063535462 23351 0ustar petrilliusersfrom Zope.Interfaces.Interface import Interface class CatalogAware: """ Description of the Item interface """ def creator(self): """ Return a sequence of user names who have the local Owner role on an object. The name creator is used for this method to conform to Dublin Core. """ def summary(self, num=200): """ Returns the summary of the text contents of the object (if text content is present). Summary length is 'num'. """ def index_object(self): """ This object will try and catalog itself when this method is called. """ def unindex_object(self): """ This object will try and uncatalog itself when this method is called. """ def reindex_all(self, obj=None): """ This method will cause this object to get reindexed. Of this object is a container, then all child objects will try to be recursivly reindexed. 'obj' can be passed as an alternative object to try and reindex. """ CatalogAwareInterface=Interface(CatalogAware) # create the interface object lib/python/Products/ZCatalog/CatalogBrains.py100644 1007 144 11423 7216304722 21022 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## __version__ = "$Revision: 1.1.6.1 $"[11:-2] import Acquisition, Record class AbstractCatalogBrain(Record.Record, Acquisition.Implicit): """Abstract base brain that handles looking up attributes as required, and provides just enough smarts to let us get the URL, path, and cataloged object without having to ask the catalog directly. """ def has_key(self, key): return self.__record_schema__.has_key(key) def getPath(self): """Get the physical path for this record""" return self.aq_parent.getpath(self.data_record_id_) def getURL(self): """Try to generate a URL for this record""" try: return self.REQUEST.physicalPathToURL(self.getPath()) except: return self.getPath() def getObject(self): """Try to return the object for this record""" try: return self.aq_parent.restrictedTraverse(self.getPath()) except: pass class NoBrainer: """ This is an empty class to use when no brain is specified. """ pass lib/python/Products/ZCatalog/Lazy.py100644 1007 144 20741 7106600730 17227 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 0.9.7 # --------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## __doc__='''$Id: Lazy.py,v 1.2 2000/05/11 18:54:16 jim Exp $''' __version__='$Revision: 1.2 $'[11:-2] class Lazy: # Allow (reluctantly) access to unprotected attributes __allow_access_to_unprotected_subobjects__=1 def __repr__(self): return `list(self)` def __len__(self): try: return self._len except AttributeError: pass l=len(self._data) while 1: try: self[l] l=l+1 except: self._len=l return l def __getslice__(self,i1,i2): r=[] for i in range(i1,i2): try: r.append(self[i]) except IndexError: return r return r slice=__getslice__ class LazyCat(Lazy): # Lazy concatenation of one or more sequences. Should be handy # for accessing small parts of big searches. def __init__(self, sequences): self._seq=sequences self._data=[] self._sindex=0 self._eindex=-1 def __getitem__(self,index): data=self._data try: seq=self._seq except AttributeError: return data[index] i=index if i < 0: i=len(self)+i if i < 0: raise IndexError, index ind=len(data) if i < ind: return data[i] ind=ind-1 sindex=self._sindex try: s=seq[sindex] except: raise IndexError, index eindex=self._eindex while i > ind: try: eindex=eindex+1 v=s[eindex] data.append(v) ind=ind+1 except IndexError: self._sindex=sindex=sindex+1 try: s=self._seq[sindex] except: del self._seq del self._sindex del self._eindex raise IndexError, index self._eindex=eindex=-1 self._eindex=eindex return data[i] class LazyMap(Lazy): # Act like a sequence, but get data from a filtering process. # Don't access data until necessary def __init__(self,func,seq): self._seq=seq self._len=len(seq) self._data=[] self._func=func def __getitem__(self,index): data=self._data try: s=self._seq except AttributeError: return data[index] i=index if i < 0: i=len(self)+i if i < 0: raise IndexError, index ind=len(data) if i < ind: return data[i] ind=ind-1 func=self._func while i > ind: try: ind=ind+1 data.append(func(s[ind])) except IndexError: del self._func del self._seq raise IndexError, index return data[i] class LazyFilter(Lazy): # Act like a sequence, but get data from a filtering process. # Don't access data until necessary def __init__(self,test,seq): self._seq=seq self._data=[] self._eindex=-1 self._test=test def __getitem__(self,index): data=self._data try: s=self._seq except AttributeError: return data[index] i=index if i < 0: i=len(self)+i if i < 0: raise IndexError, index ind=len(data) if i < ind: return data[i] ind=ind-1 test=self._test e=self._eindex while i > ind: try: e=e+1 v=s[e] if test(v): data.append(v) ind=ind+1 except IndexError: del self._test del self._seq del self._eindex raise IndexError, index self._eindex=e return data[i] class LazyMop(Lazy): # Act like a sequence, but get data from a filtering process. # Don't access data until necessary def __init__(self,test,seq): self._seq=seq self._data=[] self._eindex=-1 self._test=test def __getitem__(self,index): data=self._data try: s=self._seq except AttributeError: return data[index] i=index if i < 0: i=len(self)+i if i < 0: raise IndexError, index ind=len(data) if i < ind: return data[i] ind=ind-1 test=self._test e=self._eindex while i > ind: try: e=e+1 v=s[e] try: v=test(v) data.append(v) ind=ind+1 except: pass except IndexError: del self._test del self._seq del self._eindex raise IndexError, index self._eindex=e return data[i] lib/python/Products/ZCatalog/README.txt100644 1007 144 706 6772164416 17411 0ustar petrilliusersZCatalog Notes for Zope 2.0 Catalog now comes with the Zope core, there is no need to install it. This is the ZCatalog product for the Z Object Publishing Environment. ****** IMPORTANT ****** This product contains no binaries, and should install identically on all platforms. Installation o Extract the distribution file into the top-level directory of your Zope installation. o Restart your Zope installation. lib/python/Products/ZCatalog/Vocabulary.py100644 1007 144 15044 7217464524 20433 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## """ZCatalog product""" from Globals import HTMLFile, MessageDialog import Globals, AccessControl.Role from Acquisition import Implicit from Persistence import Persistent from OFS.SimpleItem import Item from SearchIndex import Lexicon, GlobbingLexicon from SearchIndex.Lexicon import stop_word_dict manage_addVocabularyForm=HTMLFile('addVocabulary',globals()) def manage_addVocabulary(self, id, title, globbing=None, REQUEST=None): """Add a Vocabulary object """ id=str(id) title=str(title) if globbing: globbing=1 c=Vocabulary(id, title, globbing) self._setObject(id, c) if REQUEST is not None: return self.manage_main(self,REQUEST) class Vocabulary(Item, Persistent, Implicit, AccessControl.Role.RoleManager, ): """ A Vocabulary is a user managable relization of a Lexicon object. """ meta_type = "Vocabulary" _isAVocabulary = 1 manage_options=( ( {'label': 'Vocabulary', 'action': 'manage_main', 'help' : ('ZCatalog', 'Vocabulary_Vocabulary.stx')}, {'label': 'Query', 'action': 'manage_query', 'help': ('ZCatalog', 'Vocabulary_Query.stx')}, ) +Item.manage_options +AccessControl.Role.RoleManager.manage_options ) __ac_permissions__=( ('Manage Vocabulary', ['manage_main', 'manage_vocab', 'manage_query'], ['Manager']), ('Query Vocabulary', ['query',], ['Anonymous', 'Manager']), ) manage_main = HTMLFile('manage_vocab', globals()) manage_query = HTMLFile('vocab_query', globals()) def __init__(self, id, title='', globbing=None): """ create the lexicon to manage... """ self.id = id self.title = title self.globbing = globbing if globbing: self.lexicon = GlobbingLexicon.GlobbingLexicon() else: self.lexicon = Lexicon.Lexicon(stop_word_dict) def clear(self): # Clear the vocabulary. This is in response to the desire to # clear it without having to recreate it at the same time. self.lexicon.initialize() def query(self, pattern): """ """ result = [] for x in self.lexicon.get(pattern): if self.globbing: result.append(self.lexicon._inverseLex[x]) else: result.append(pattern) return result def manage_insert(self, word='', URL1=None, RESPONSE=None): """ doc string """ self.insert(word) if RESPONSE: RESPONSE.redirect(URL1 + '/manage_main') def manage_stop_syn(self, stop_syn, REQUEST=None): pass def insert(self, word=''): self.lexicon.set(word) def words(self): return self.lexicon._lexicon.items() lib/python/Products/ZCatalog/ZCatalog.py100644 1007 144 57776 7216304722 20042 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## """ZCatalog product""" from Globals import HTMLFile, MessageDialog import Globals from OFS.Folder import Folder from OFS.FindSupport import FindSupport from DateTime import DateTime from SearchIndex import Query import string, regex, urlparse, urllib, os, sys, time import Products from Acquisition import Implicit from Persistence import Persistent from DocumentTemplate.DT_Util import InstanceDict, TemplateDict from DocumentTemplate.DT_Util import Eval, expr_globals from AccessControl.Permission import name_trans from Catalog import Catalog, orify from SearchIndex import UnIndex, UnTextIndex from Vocabulary import Vocabulary import IOBTree from AccessControl import getSecurityManager manage_addZCatalogForm=HTMLFile('addZCatalog',globals()) def manage_addZCatalog(self, id, title, vocab_id=None, REQUEST=None): """Add a ZCatalog object """ id=str(id) title=str(title) vocab_id=str(vocab_id) if vocab_id == 'create_default_catalog_': vocab_id = None c=ZCatalog(id, title, vocab_id, self) self._setObject(id, c) if REQUEST is not None: return self.manage_main(self, REQUEST) class ZCatalog(Folder, Persistent, Implicit): """ZCatalog object A ZCatalog contains arbirary index like references to Zope objects. ZCatalog's can index either 'Field' values of object, or 'Text' values. ZCatalog does not store references to the objects themselves, but rather to a unique identifier that defines how to get to the object. In Zope, this unique idenfier is the object's relative path to the ZCatalog (since two Zope object's cannot have the same URL, this is an excellent unique qualifier in Zope). Most of the dirty work is done in the _catalog object, which is an instance of the Catalog class. An interesting feature of this class is that it is not Zope specific. You can use it in any Python program to catalog objects. """ meta_type = "ZCatalog" icon='misc_/ZCatalog/ZCatalog.gif' manage_options=Folder.manage_options + ( {'label': 'Cataloged Objects', 'action': 'manage_catalogView', 'target': 'manage_main', 'help':('ZCatalog','ZCatalog_Cataloged-Objects.stx')}, {'label': 'Find Items to ZCatalog', 'action': 'manage_catalogFind', 'target':'manage_main', 'help':('ZCatalog','ZCatalog_Find-Items-to-ZCatalog.stx')}, {'label': 'MetaData Table', 'action': 'manage_catalogSchema', 'target':'manage_main', 'help':('ZCatalog','ZCatalog_MetaData-Table.stx')}, {'label': 'Indexes', 'action': 'manage_catalogIndexes', 'target':'manage_main', 'help':('ZCatalog','ZCatalog_Indexes.stx')}, {'label': 'Status', 'action': 'manage_catalogStatus', 'target':'manage_main', 'help':('ZCatalog','ZCatalog_Status.stx')}, ) __ac_permissions__=( ('Manage ZCatalog Entries', ['manage_catalogObject', 'manage_uncatalogObject', 'catalog_object', 'uncatalog_object', 'manage_catalogView', 'manage_catalogFind', 'manage_catalogSchema', 'manage_catalogIndexes', 'manage_catalogStatus', 'manage_catalogReindex', 'manage_catalogFoundItems', 'manage_catalogClear', 'manage_addColumn', 'manage_delColumns', 'manage_addIndex', 'manage_delIndexes', 'manage_main', 'manage_normalize_paths',], ['Manager']), ('Search ZCatalog', ['searchResults', '__call__', 'uniqueValuesFor', 'getpath', 'schema', 'indexes', 'index_objects', 'all_meta_types', 'valid_roles', 'resolve_url', 'getobject'], ['Anonymous', 'Manager']), ) manage_catalogAddRowForm = HTMLFile('catalogAddRowForm', globals()) manage_catalogView = HTMLFile('catalogView',globals()) manage_catalogFind = HTMLFile('catalogFind',globals()) manage_catalogSchema = HTMLFile('catalogSchema', globals()) manage_catalogIndexes = HTMLFile('catalogIndexes', globals()) manage_catalogStatus = HTMLFile('catalogStatus', globals()) threshold=10000 _v_total=0 def __init__(self, id, title='', vocab_id=None, container=None): self.id=id self.title=title self.vocab_id = vocab_id self.threshold = 10000 self._v_total = 0 if vocab_id is None: v = Vocabulary('Vocabulary', 'Vocabulary', globbing=1) self._setObject('Vocabulary', v) v = 'Vocabulary' else: v = vocab_id self._catalog = Catalog(vocabulary=v) self._catalog.addColumn('id') self._catalog.addIndex('id', 'FieldIndex') self._catalog.addColumn('title') self._catalog.addIndex('title', 'TextIndex') self._catalog.addColumn('meta_type') self._catalog.addIndex('meta_type', 'FieldIndex') self._catalog.addColumn('bobobase_modification_time') self._catalog.addIndex('bobobase_modification_time', 'FieldIndex') self._catalog.addColumn('summary') self._catalog.addIndex('PrincipiaSearchSource', 'TextIndex') def getVocabulary(self): """ more ack! """ return getattr(self, self.vocab_id) def manage_edit(self, RESPONSE, URL1, threshold=1000, REQUEST=None): """ edit the catalog """ if type(threshold) is not type(1): threshold=string.atoi(threshold) self.threshold = threshold RESPONSE.redirect(URL1 + '/manage_main?manage_tabs_message=Catalog%20Changed') def manage_subbingToggle(self, REQUEST, RESPONSE, URL1): """ toggle subtransactions """ if self.threshold: self.threshold = None else: self.threshold = 10000 RESPONSE.redirect(URL1 + '/manage_catalogStatus?manage_tabs_message=Catalog%20Changed') def manage_catalogObject(self, REQUEST, RESPONSE, URL1, urls=None): """ index all Zope objects that 'urls' point to """ if urls: for url in urls: obj = self.resolve_url(url) if obj is not None: self.catalog_object(obj, url) RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=Object%20Cataloged') def manage_uncatalogObject(self, REQUEST, RESPONSE, URL1, urls=None): """ removes Zope object 'urls' from catalog """ if urls: for url in urls: self.uncatalog_object(url) RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=Object%20Uncataloged') def manage_catalogReindex(self, REQUEST, RESPONSE, URL1): """ clear the catalog, then re-index everything """ elapse = time.time() c_elapse = time.clock() paths = tuple(self._catalog.paths.values()) self._catalog.clear() for p in paths: obj = self.resolve_url(p) if obj is not None: self.catalog_object(obj, p) elapse = time.time() - elapse c_elapse = time.clock() - c_elapse RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=' + urllib.quote('Catalog Updated
Total time: %s
Total CPU time: %s' % (`elapse`, `c_elapse`))) def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None): """ clears the whole enchelada """ self._catalog.clear() if REQUEST and RESPONSE: RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=Catalog%20Cleared') def manage_catalogFoundItems(self, REQUEST, RESPONSE, URL1, obj_metatypes=None, obj_ids=None, obj_searchterm=None, obj_expr=None, obj_mtime=None, obj_mspec=None, obj_roles=None, obj_permission=None): """ Find object according to search criteria and Catalog them """ elapse = time.time() c_elapse = time.clock() words = 0 obj = REQUEST.PARENTS[1] path = string.join(obj.getPhysicalPath(), '/') results = self.ZopeFindAndApply(obj, obj_metatypes=obj_metatypes, obj_ids=obj_ids, obj_searchterm=obj_searchterm, obj_expr=obj_expr, obj_mtime=obj_mtime, obj_mspec=obj_mspec, obj_permission=obj_permission, obj_roles=obj_roles, search_sub=1, REQUEST=REQUEST, apply_func=self.catalog_object, apply_path=path) elapse = time.time() - elapse c_elapse = time.clock() - c_elapse RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=' + urllib.quote('Catalog Updated
Total time: %s
Total CPU time: %s' % (`elapse`, `c_elapse`))) def manage_addColumn(self, name, REQUEST=None, RESPONSE=None, URL1=None): """ add a column """ self._catalog.addColumn(name) if REQUEST and RESPONSE: RESPONSE.redirect(URL1 + '/manage_catalogSchema?manage_tabs_message=Column%20Added') def manage_delColumns(self, names, REQUEST=None, RESPONSE=None, URL1=None): """ del a column """ for name in names: self._catalog.delColumn(name) if REQUEST and RESPONSE: RESPONSE.redirect(URL1 + '/manage_catalogSchema?manage_tabs_message=Column%20Deleted') def manage_addIndex(self, name, type, REQUEST=None, RESPONSE=None, URL1=None): """ add an index """ self._catalog.addIndex(name, type) if REQUEST and RESPONSE: RESPONSE.redirect(URL1 + '/manage_catalogIndexes?manage_tabs_message=Index%20Added') def manage_delIndexes(self, names, REQUEST=None, RESPONSE=None, URL1=None): """ del an index """ for name in names: self._catalog.delIndex(name) if REQUEST and RESPONSE: RESPONSE.redirect(URL1 + '/manage_catalogIndexes?manage_tabs_message=Index%20Deleted') def catalog_object(self, obj, uid): """ wrapper around catalog """ self._v_total = (self._v_total + self._catalog.catalogObject(obj, uid, self.threshold)) if self.threshold is not None: if self._v_total > self.threshold: # commit a subtransaction get_transaction().commit(1) # kick the chache, this may be overkill but ya never know self._p_jar.cacheFullSweep(1) self._v_total = 0 def uncatalog_object(self, uid): """ wrapper around catalog """ self._catalog.uncatalogObject(uid) def uniqueValuesFor(self, name): """ returns the unique values for a given FieldIndex """ return self._catalog.uniqueValuesFor(name) def getpath(self, rid): """ Return the path to a cataloged object given a 'data_record_id_' """ return self._catalog.paths[rid] def getobject(self, rid): """ Return a cataloged object given a 'data_record_id_' """ return self.resolve_url(self.getpath(rid)) def schema(self): return self._catalog.schema.keys() def indexes(self): return self._catalog.indexes.keys() def index_objects(self): return self._catalog.indexes.values() def _searchable_arguments(self): r = {} n={'optional':1} for name in self._catalog.schema.keys(): r[name]=n return r def _searchable_result_columns(self): r = [] for name in self._catalog.schema.keys(): i = {} i['name'] = name i['type'] = 's' i['parser'] = str i['width'] = 8 r.append(i) r.append({'name': 'data_record_id_', 'type': 's', 'parser': str, 'width': 8}) return r def searchResults(self, REQUEST=None, used=None, query_map={ type(regex.compile('')): Query.Regex, type([]): orify, type(()): orify, type(''): Query.String, }, **kw): """ Search the catalog according to the ZTables search interface. Search terms can be passed in the REQUEST or as keyword arguments. """ return apply(self._catalog.searchResults, (REQUEST,used, query_map), kw) __call__=searchResults ## this stuff is so the find machinery works meta_types=() # Sub-object types that are specific to this object def all_meta_types(self): pmt=() if hasattr(self, '_product_meta_types'): pmt=self._product_meta_types elif hasattr(self, 'aq_acquire'): try: pmt=self.aq_acquire('_product_meta_types') except: pass return self.meta_types+Products.meta_types+pmt def valid_roles(self): "Return list of valid roles" obj=self dict={} dup =dict.has_key x=0 while x < 100: if hasattr(obj, '__ac_roles__'): roles=obj.__ac_roles__ for role in roles: if not dup(role): dict[role]=1 if not hasattr(obj, 'aq_parent'): break obj=obj.aq_parent x=x+1 roles=dict.keys() roles.sort() return roles def ZopeFindAndApply(self, obj, obj_ids=None, obj_metatypes=None, obj_searchterm=None, obj_expr=None, obj_mtime=None, obj_mspec=None, obj_permission=None, obj_roles=None, search_sub=0, REQUEST=None, result=None, pre='', apply_func=None, apply_path=''): """Zope Find interface and apply This is a *great* hack. Zope find just doesn't do what we need here; the ability to apply a method to all the objects *as they're found* and the need to pass the object's path into that method. """ if result is None: result=[] if obj_metatypes and 'all' in obj_metatypes: obj_metatypes=None if obj_mtime and type(obj_mtime)==type('s'): obj_mtime=DateTime(obj_mtime).timeTime() if obj_permission: obj_permission=p_name(obj_permission) if obj_roles and type(obj_roles) is type('s'): obj_roles=[obj_roles] if obj_expr: # Setup expr machinations md=td() obj_expr=(Eval(obj_expr, expr_globals), md, md._push, md._pop) base=obj if hasattr(obj, 'aq_base'): base=obj.aq_base if not hasattr(base, 'objectItems'): return result try: items=obj.objectItems() except: return result try: add_result=result.append except: raise AttributeError, `result` for id, ob in items: if pre: p="%s/%s" % (pre, id) else: p=id dflag=0 if hasattr(ob, '_p_changed') and (ob._p_changed == None): dflag=1 if hasattr(ob, 'aq_base'): bs=ob.aq_base else: bs=ob if ( (not obj_ids or absattr(bs.id) in obj_ids) and (not obj_metatypes or (hasattr(bs, 'meta_type') and bs.meta_type in obj_metatypes)) and (not obj_searchterm or (hasattr(ob, 'PrincipiaSearchSource') and string.find(ob.PrincipiaSearchSource(), obj_searchterm) >= 0 )) and (not obj_expr or expr_match(ob, obj_expr)) and (not obj_mtime or mtime_match(ob, obj_mtime, obj_mspec)) and ( (not obj_permission or not obj_roles) or \ role_match(ob, obj_permission, obj_roles) ) ): if apply_func: apply_func(ob, (apply_path+'/'+p)) else: add_result((p, ob)) dflag=0 if search_sub and hasattr(bs, 'objectItems'): self.ZopeFindAndApply(ob, obj_ids, obj_metatypes, obj_searchterm, obj_expr, obj_mtime, obj_mspec, obj_permission, obj_roles, search_sub, REQUEST, result, p, apply_func, apply_path) if dflag: ob._p_deactivate() return result def resolve_url(self, path): """ Attempt to resolve a url into an object in the Zope namespace. The url may be absolute or a catalog path style url. If no object is found, None is returned. No exceptions are raised. """ try: return self.restrictedTraverse(path) except: pass def manage_normalize_paths(self): """Ensure that all catalog paths are full physical paths This should only be used with ZCatalogs in which all paths can be resolved with unrestrictedTraverse.""" paths = self._catalog.paths uids = self._catalog.uids unchanged = 0 fixed = [] removed = [] for path, rid in uids.items(): ob = None if path[:1] == '/': ob = self.resolve_url(path[1:]) if ob is None: ob = self.resolve_url(path) if ob is None: removed.append(path) continue ppath = string.join(ob.getPhysicalPath(), '/') if path != ppath: fixed.append((path, ppath)) else: unchanged = unchanged + 1 for path, ppath in fixed: rid = uids[path] del uids[path] paths[rid] = ppath uids[ppath] = rid for path in removed: self.uncatalog_object(path) return MessageDialog(title='Done Normalizing Paths', message='%s paths normalized, %s paths removed, and ' '%s unchanged.' % (len(fixed), len(removed), unchanged), action='./manage_main') Globals.default__class_init__(ZCatalog) def p_name(name): return '_' + string.translate(name, name_trans) + '_Permission' def absattr(attr): if callable(attr): return attr() return attr class td(TemplateDict): def validate(self, inst, parent, name, value, md): return getSecurityManager().validate(inst, parent, name, value) def expr_match(ob, ed, c=InstanceDict, r=0): e, md, push, pop=ed push(c(ob, md)) try: r=e.eval(md) finally: pop() return r def mtime_match(ob, t, q, fn=hasattr): if not fn(ob, '_p_mtime'): return 0 return q=='<' and (ob._p_mtime < t) or (ob._p_mtime > t) def role_match(ob, permission, roles, lt=type([]), tt=type(())): pr=[] fn=pr.append while 1: if hasattr(ob, permission): p=getattr(ob, permission) if type(p) is lt: map(fn, p) if hasattr(ob, 'aq_parent'): ob=ob.aq_parent continue break if type(p) is tt: map(fn, p) break if p is None: map(fn, ('Manager', 'Anonymous')) break if hasattr(ob, 'aq_parent'): ob=ob.aq_parent continue break for role in roles: if not (role in pr): return 0 return 1 lib/python/Products/ZCatalog/ZCatalog.txt100755 1007 144 42712 6756115345 20225 0ustar petrilliusersZCatalog Tutorial This document provides a tutorial for 'ZCatalog', the new search engine machinery in Zope. The audience for the document is content managers. Contents o What is it? What's it for? Why's it so cool? o Installing ZCatalog o ZCatalog Objects o Example using ZCatalog o Creating Search Forms And Result Reports o Using ZCatalog In A Zope Site o ZCatalog vs. Catalog What is it? What's it for? Why's it so cool? The 'ZCatalog' provides powerful indexing and searching on a Zope database using a Zope management interface. A 'ZCatalog' is a Zope object that can be added to a Folder, managed through the web, and extended in many ways. The 'ZCatalog' is a very significant project, providing a number of compelling features: o **Searches are fast**. The data structures used by the index provide extremely quick searches without consuming much memory. o **Searches are robust**. The 'ZCatalog' supports boolean search terms, proximity searches, synonyms and stopwords. o **Indexing is wildly flexible**. A 'ZCatalog' can catalog custom properties and track unique values. Since 'ZCatalog' catalogs objects instead of file handles, you can index any content that can have a Python object wrapped around it. This also lets objects participate in how they are cataloged, e.g. de-HTML-ifying contents or extracting PDF properties. o **Usable outside of Zope**. The software is broken into a Python 'Catalog' which wrapped by a 'ZCatalog'. The Python 'Catalog' can be used in any Python program; all it requires is the Z object database and the indexing machinery from Zope. o **Transactional**. An indexing operation is part of a Zope transaction. If something goes wrong after content is indexed, the index is restored to its previous condition. This also means that Undo will restore an index to its previous condition. Finally, a 'ZCatalog' can be altered privately in a Version, meaning no one else can see the changes to the index. o **Cache-friendly**. The index is internally broken into different "buckets", with each bucket being a separate Zope database object. Thus, only the part of the index that is needed is loaded into memory. Alternatively, an un-needed part of the index can be removed from memory. o **Results are lazy**. A search that returns a tremendous number of matches won't return a large result set. Only the part of the results, such as the second batch of twenty, are returned. The 'ZCatalog' is a free, Open Source part of the Zope software repository and thus is covered under the same license as Zope. It is being developed in conjunction with the Zope Portal Toolkit effort. However, the 'ZCatalog' product is managed as its own module in CVS. Installing ZCatalog 'ZCatalog' can be downloaded from the Zope download area and is also a module in the public CVS for Zope. Untar it while in the root directory of your Zope installation:: $ cd Zope-2.0.0a3-src/ $ tar xzf ../ZCatalog-x.x.tgz Windows users can use WinZip or a similar utility to accomplish the same thing. Also, Zope 2.0.0a3 does not have the latest version of UnIndex and UnTextIndex which fix a couple of bugs in the alpha 3 versions. The latest CVS of the SearchIndex packages *must* be used. Remember, you have to restart your Zope server before you will see 'ZCatalog'. ZCatalog Objects A 'ZCatalog' performs two activities: indexing information and performing searches. Most the work is done in the first step, which is getting objects into the index. This is done in two ways. First, if your objects are ZCatalog-aware they automatically update the index when they are added, edited or directly deleted. A ZCatalog-aware object is one that is an instance of a 'Z Class' that informs the 'ZCatalog' of changes. *Directly deleted* means the object was deleted from a Folder, not the deletion of a containing Folder. The second way that site contents get updated is by "finding" information "into" the 'ZCatalog'. An operation based on Zope's Find view traverses Folders looking for objects matching the criterion. The objects are then registered with the Catalog. Objects in the index but no longer in the site are removed from the Catalog. Either way -- automatically updating or walking the Folders -- 'ZCatalog' indexes the objects it finds. The 'ZCatalog' is set up to look for properties, each of which are added to the index. There are two kinds of indexes, called FieldIndex and TextIndex. FieldIndex indexes treat data atomically. The entire contents of a FieldIndex-indexed property is treated as a unit. With a TextIndex index, it is broken into words which are indexed individually. A TextIndex is also known as *full-text index*. Note that the 'ZCatalog' doesn't track ZCatalog-unaware objects after it has indexed them. This means that the 'ZCatalog' must reindex its objects occasionally when the objects have been changed. Out of date indexes can be prevented by inheriting from a ZCatalog-aware class which can tell the 'ZCatalog' to reindex it whenever a change is made. Just such a class will be included with the Portal toolkit. ZCatalogs are "searchable objects", meaning they cooperate with Z Search Interfaces documented in Z SQL Methods. Creating a search form for a 'ZCatalog' is a simple matter of adding a Z Search Interface from the management screen and filling in a form. ZCatalogs can also be queried directly from DTML, as shown in the example below. Example using Z Classes The first example shows how to give your Zope site a long-desired feature: full text-searches of your content. The example assumes you already have a number of DTML Methods/Documents to catalog. o Install 'ZCatalog' as instructed above o In the root folder of your Zope server, add a 'ZCatalog'. o Type in the id 'catalog' and hit 'Add'. You now have a brand new 'ZCatalog' named 'catalog' in your root folder. o Click on it. Now you are looking at the 'ZCatalog' 'Contents' view. It says the catalog is empty. We'll catalog some objects in a moment, but first we have to tell it what portions of objects we are specifically interested in. o Click on 'Indexes'. This management view is where the attributes to be indexed are defined. o In the 'Add index' field, type 'raw'. o Click 'Add'. Now that the indexes are defined, a set of objects can be selected for cataloging. o Click on 'Find items to ZCatalog'. For this example, we are only interested in DTML Documents and Methods. o Deselect 'All type'. o Select 'DTML Method' and 'DTML Document'. o Click 'Find'. ZCatalog will report how many items it found, and then present an interface for excluding specific objects. o Click 'Catalog Items'. Great, now that the catalog is stocked, we can create a user interface to it. o Return to the root folder's management view. o Add a 'Z Search Interface'. 'ZCatalog' participates in the Zope Search architecture. You simply have to fill in this form, and a basic user interface will be created. o Select 'catalog' in the list beside 'Select one or more searchable objects'. o Beside 'Report Id', type 'report'. o Beside 'Search Input Id', type 'search'. 'report' and 'search' are the Ids of two DTML Methods which will be created in your root folder. o Click 'Add'. Congratulations, if all has gone well, you can now find references to any word in your DTML pages. Try it by viewing 'search'. Type a common word in the 'Raw' field, and you should be presented with a list of hits. However, none of the results returned can be clicked on. To fix this, go to the management view of 'report'. 'report' is called by 'search' to display the results from 'catalog'. 'report' is just a simple '' loop with a few refinements. 'catalog' knows which results to return by looking at the REQUEST variable, which contains the input from the 'search' form. o In the source of 'report', find the following line:: o Replace it with this:: "> This is a little confusing at first. Keep in mind that ZCatalog does not return a list of your database objects. What it returns are actually fairly unintelligent instances of a Record subclass. These record objects contain copies of data from attributes of catalog objects. The 'ZCatalog' 'MetaData Table' view defines which attributes are copied. (By default, these record objects are just SLIGHTLY more intelligent than a raw tuple. 'Catalog' can be told to use a custom, intelligent class for results. Please see the 'Catalog' __init__ method in 'lib/python/Products/ZCatalog/Catalog.py' for more information.) Fortunately, ZCatalog provides a utility function for going from result objects to the object's path. It is called, aptly enough, 'getpath'. 'getpath' expects to be passed the unique integer identifier of the cataloged object. Results store that id as 'data_record_id_'. Commit this change, and perform another search. Now the title can be clicked on to take you to the full page. Example cataloging custom objects As if full-text searches of your entire site weren't good enough, ZCatalog can also catalog Z Classes, Products, and in fact any Python object you can put in a ZODB. Here is an example using a Z Class, but the principles apply to any kind of object. First, we're going to need something to catalog. Follow the 'Z Class' tutorial to create the CD 'Z Class'. Back? Good. o Create a folder, 'CDs', and create a number of instances of the CD Z Class in it. 'cd1' through 'cd5' should be plenty. Remember to fill each of them in from their Properties view. Now we want to create a searchable catalog of CDs. o Go to the 'CDs' folder and create a 'ZCatalog' with an ID 'cd_cat'. o Click on the objects Indexes view. This screen shows that, by default, 'ZCatalog' is interested in an object's 'id','title', 'meta_type', and 'bobobase_modification_time'. You will almost always want to index additional information. In this case, we would also like to index the artist and description of CDs. o Type 'artist' into the 'Add Index' field. For the sake of example, we're going to use a FieldIndex index for artist. This will give us the option of putting an HTML SELECT box for artists on the search form. o Select FieldIndex from the Index type drop down, and click 'Add'. o Also add an index for 'description', but leave TextIndex selected. This will allow us to search for individual words within the description. o Click on 'MetaData Table'. This is where we tell the 'ZCatalog' what attributes of cataloged objects to cache. These cached values are available from search results without having to look up the actual indexed object. The tradeoff for the speed is extra memory, as information from the content is duplicated in the 'ZCatalog'. You will probably want to keep the schema light-weight, so we're not going to add 'description' to it. Type 'artist' in the 'Add column' field and click 'Add'. o Click on the 'Find Items to Catalog' view. This is the interface you use to tell the 'ZCatalog' which items to index. Right now, beside 'Find objects of type:', 'All types' is selected. o Deselect 'All types'. O Scroll down and select CD. You could use the rest of the form to be more specific, but since we want to catalog all the CDs, o Click 'Find'. 'ZCatalog' will report 'Found 5 items.' It is now giving you an opportunity to exclude some of the matched items from the index. Again, we want all of them, so, o Click 'Catalog Items'. You should at this point see a list of the indexed objects. Also of note is the 'Update Catalog' button. You have to use it whenever you want your 'ZCatalog' to notice changes you've made to the objects it's indexed. Creating Search Forms And Result Reports This catalog isn't much good without some way of querying it. o Go back to your 'CDs' folder's management screen and add a Z Search Interface. The search add form will automatically detect your cd_cat 'ZCatalog' and offer it as a searchable document. Make sure it is selected. o Fill in 'cd_report' for 'Report ID' and 'cd_search' for 'Search Input ID'. Those are the ids of two DTML methods that will be generated in the 'CDs' folder. o Click 'Add'. o View the 'cd_search' Catalog (at, for example, http://localhost:9673/CDs/cd_search). You will see a basic search interface, with fields for searching on 'title', modification date, 'id', 'artist', 'meta type' and 'description'. If you fill in one more more of the fields and click 'Submit Query', cd_report will be displayed. It is passed the search criteria and uses it to get a list from cd_cat to iterate over. It is merely displaying the information from the ZCatalog's MetaData table, but of course it can be enriched. Try a few more searches. You'll find that you can type any single word from the title or description and get a match, but for artist you must type the exact string. That's because artist was indexed as a FieldIndex, which gives us an opportunity to present a more convenient interface. Go back to the 'cd_search' management interface, and change it's source to look like this:: <!--#var standard_html_header--> <form action="cd_report" method="get"> <h2><!--#var document_title--></h2> Enter query parameters:<br><table> <tr><th>Title</th> <td><input name="title" width=30 value=""></td></tr> <tr><th>Artist</th> <td> <select name="artist"> <option value="">All</option> <!--#in expr="cd_cat.uniqueValuesFor('artist')"--> <option value="<!--#var sequence-item-->"> <!--#var sequence-item--> </option> <!--#/in--> </select> </td> </tr> <tr><th>Description</th> <td><input name="description" width=30 value=""></td></tr> <tr><td colspan=2 align=center> <input type="SUBMIT" name="SUBMIT" value="Submit Query"> </td></tr> </table> </form> <!--#var standard_html_footer--> This is a search form somewhat more appropriate for the CD 'Z Class'. Unrelated fields have been removed, and the 'artist' field has been changed to a drop-down menu. Let's augment the output of 'cd_report' to make the title a link to the actual CD object. Taking a look at 'cd_report', note that the search results are obtained with a simple '' tag. The search criteria is automatically obtained by the 'ZCatalog' from the form input. The line we're interested in is this one:: Change it to read:: "> Now, assuming you have added the index_html document template to your CD 'Z Class', clicking on a search result will take you to the CD's detailed display. Using 'ZCatalog' In A Zope Site The 'ZCatalog' provides high-speed access to what is on your site. Thus, the 'ZCatalog' can be used to re-engineer the way your site is laid out. For instance, a Slashdot-style presentation is simple. Just insert some DTML that asks the 'ZCatalog' for recent items. Alternatively, a Site Map is nothing more than presenting the contents of the catalog. A page with tree-based browsing of software packages by category is also easy. Perhaps you'd like to provide a link that lists all the packages the current user has authored. Thus, the 'ZCatalog' isn't just about searching. It can be used as the DTML-scriptable engine for browsing a site as well. Since the 'ZCatalog' is a normal Zope folderish object, you can also create DTML Methods inside it to present the catalog contents. For instance, perhaps you'd like to dump the contents of the site as an RDF stream, or do content syndication with RSS. These are just DTML Methods that change the 'Content-Type:' and send back XML. All without actually waking up any of the content objects in the site. ZCatalog vs. Catalog The real star of this package is the 'Catalog' module. All the heavy lifting is done by 'Catalog'. 'ZCatalog' is basically a Zope-aware wrapper around Catalog, which can be used on it's own outside the Zope framework. The only requirement is that you are using ZODB as your object store. lib/python/Products/ZCatalog/__init__.py100644 1007 144 11224 7200120123 20027 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## """ZCatalog product""" import ZCatalog, Catalog, CatalogAwareness, Vocabulary, ZClasses from ZClasses import createZClassForBase createZClassForBase( ZCatalog.ZCatalog , globals() , 'ZCatalogBase', 'ZCatalog' ) createZClassForBase( CatalogAwareness.CatalogAware, globals() , 'CatalogAwareBase', 'CatalogAware' ) def initialize(context): context.registerClass( ZCatalog.ZCatalog, permission='Add ZCatalogs', constructors=(ZCatalog.manage_addZCatalogForm, ZCatalog.manage_addZCatalog), icon='www/ZCatalog.gif', ) context.registerClass( Vocabulary.Vocabulary, permission='Add Vocabularies', constructors=(Vocabulary.manage_addVocabularyForm, Vocabulary.manage_addVocabulary), icon='www/Vocabulary.gif', ) context.registerHelp() context.registerHelpTitle('Zope Help') lib/python/Products/ZCatalog/addVocabulary.dtml100644 1007 144 1644 7045357555 21401 0ustar petrilliusers Add Vocabulary

Add Vocabulary

Id
Title
Globbing?

lib/python/Products/ZCatalog/addZCatalog.dtml100755 1007 144 2400 7122503711 20747 0ustar petrilliusers Add Catalog

Add Catalog

Id
Title
Vocabulary
If you do not select a vocabulary, one will be created

lib/python/Products/ZCatalog/catalogAddRowForm.dtml100644 1007 144 456 7010102540 22106 0ustar petrilliusers View Catalog Records
lib/python/Products/ZCatalog/catalogFind.dtml100755 1007 144 4471 7010102540 21006 0ustar petrilliusers Advanced Find

Use this form to locate objects to be cataloged.

Find objects of type:
with ids:
containing:
expr:
modified:
where the roles:
have permission:
lib/python/Products/ZCatalog/catalogIndexes.dtml100644 1007 144 3406 7135202002 21521 0ustar petrilliusers View Catalog Records

This list defines what indexes the Catalog will contain. When objects get cataloged, the values of any attributes they may have which match an index in this list will get indexed. Indexes come in three flavors, Text Indexes, Field Indexes and Keyword Indexes.

Text Indexes break text up into individual words, and are often refered to as full-text indexes. Text indexes sort results by score meaning they return 'hits' in order from the most relevant to the lest relevant.

Field Indexes treat the value of an objects attribute atomically, and can be used, for example, to track only a certain subset of object values, such as 'meta_type'.

Keyword Indexes index a sequence of objects that act as 'keywords' for an object. A Keyword Index will return any objects that have one or more keywords specified in a search query.

  • ()
    •


Add Index:
of Index Type:
lib/python/Products/ZCatalog/catalogSchema.dtml100644 1007 144 3332 7135202002 21320 0ustar petrilliusers View Catalog Records

This list defines what per object meta data the Catalog will store. When objects get cataloged, the values of any attributes they may have which match a name in this list will get stored in a table in the Catalog. The Catalog then uses this information to create result objects that are returned whenever the catalog is searched. It is important to understand that when the Catalog is searched, it returns a list of result objects, not the cataloged objects themselves, so if you want to use the value of an object's attribute in the result of a search, that attribute must be in this list

It is generally a good idea to keep this list lightweight. It is useful, for example, to keep the 'summary' meta data of a text document (like the first 200 characters) but not the text content in it's entirety (it is useful in this example to index the text contents, which is configured in the Indexes View tab). This way, the summary data may be shown in the search results.

    •

column from Meta Data table.

Add column to the Meta Data table:
lib/python/Products/ZCatalog/catalogStatus.dtml100644 1007 144 4455 7217442010 21420 0ustar petrilliusers View Catalog Records

Subtransactions allow Zope to commit small parts of a transaction over a period of time instead of all at once. For ZCatalog, this means using subtransactions can signficantly reduce the memory requirements needed to index huge amounts of text all at once.

If enabled, subtransactions will reduce the memory requirements of ZCatalog, but at the expense of speed. If you choose to enable subtransactions, you can adjust how often ZCatalog commits a subtransactions by adjusting the threshold below.

If you are using ZCatalog and ZSQL Methods in the same transaction, you must disable subtransactions, they are not compatible with ZSQL Methods.

Subtransactions are Enabled Disabled

Subtransactions Subtransactions

The Subtransaction threshold is the number of words the catalog will index before it commits a subtransaction. If this number is low, the Catalog will take longer to index but consume less memory. If this number is higher, the Catalog will index quickly but consume much more memory.

Subtransaction threshold:

Index Status

  • object(s) are indexed in
lib/python/Products/ZCatalog/catalogView.dtml100755 1007 144 5077 7217461113 21060 0ustar petrilliusers View Catalog Records

Updating the catalog will update all catalog records and remove invalid records. Clearing the catalog will remove all entries. You can also remove or update individual catalog records.

contains record(s).

[Previous &dtml-previous-sequence-size; entries] [Next &dtml-next-sequence-size; entries]

&dtml-meta_type; &dtml-getPath; (&dtml-title;)
[Previous &dtml-previous-sequence-size; entries] [Next &dtml-next-sequence-size; entries]

There are no objects in the Catalog.

 lib/python/Products/ZCatalog/editCatalogerForm.dtml100644 1007 144 1045 7010102540 22155 0ustar petrilliusers View Catalog Records

This element controls which Catalog this object will try index and unindex itself to.

Edit default Catalog:
lib/python/Products/ZCatalog/edit_stop_syn.dtml100644 1007 144 435 7067175207 21455 0ustar petrilliusers Edit <dtml-var title_or_id>

lib/python/Products/ZCatalog/help/ 40755 1007 144 0 7216440760 16554 5ustar petrilliuserslib/python/Products/ZCatalog/help/CVS/ 40755 1007 144 0 7216440704 17205 5ustar petrilliuserslib/python/Products/ZCatalog/help/CVS/Root100644 1007 144 20 7216440703 20077 0ustar petrilliusers/cvs-repository lib/python/Products/ZCatalog/help/CVS/Repository100644 1007 144 50 7216440703 21336 0ustar petrilliusersZope2/lib/python/Products/ZCatalog/help lib/python/Products/ZCatalog/help/CVS/Entries100644 1007 144 1701 7216440704 20635 0ustar petrilliusers/Vocabulary.py/1.4/Mon Jul 3 21:55:22 2000//Tpetrilli-Catalog-branch /Vocabulary.stx/1.2/Tue Jun 20 01:59:42 2000/-kk/Tpetrilli-Catalog-branch /Vocabulary_Query.stx/1.2/Tue Jun 20 01:59:42 2000/-kk/Tpetrilli-Catalog-branch /Vocabulary_Vocabulary.stx/1.2/Tue Jun 20 01:59:42 2000/-kk/Tpetrilli-Catalog-branch /ZCatalog.py/1.5/Mon Jul 3 21:55:22 2000/-kk/Tpetrilli-Catalog-branch /ZCatalog.stx/1.1/Fri Jun 16 18:50:36 2000/-kk/Tpetrilli-Catalog-branch /ZCatalog_Add.stx/1.1/Fri Jun 16 18:50:36 2000/-kk/Tpetrilli-Catalog-branch /ZCatalog_Cataloged-Objects.stx/1.1/Fri Jun 16 18:50:36 2000/-kk/Tpetrilli-Catalog-branch /ZCatalog_Find-Items-to-ZCatalog.stx/1.1/Fri Jun 16 18:50:36 2000/-kk/Tpetrilli-Catalog-branch /ZCatalog_Indexes.stx/1.1/Fri Jun 16 18:50:36 2000/-kk/Tpetrilli-Catalog-branch /ZCatalog_MetaData-Table.stx/1.1/Fri Jun 16 18:50:36 2000/-kk/Tpetrilli-Catalog-branch /ZCatalog_Status.stx/1.1/Fri Jun 16 18:50:36 2000/-kk/Tpetrilli-Catalog-branch D lib/python/Products/ZCatalog/help/CVS/Tag100644 1007 144 31 7216440704 17672 0ustar petrilliusersTpetrilli-Catalog-branch lib/python/Products/ZCatalog/help/Vocabulary.py100644 1007 144 10613 7130205712 21342 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## def manage_addVocabulary(self, id, title, globbing=None, REQUEST=None): """ Add a Vocabulary object to an ObjectManager. """ class Vocabulary: """ A Vocabulary manages words and language rules for text indexing. Text indexing is done by the ZCatalog and other third party Products. """ __constructor__=manage_addVocabulary def query(self, pattern): """ Query Vocabulary for words matching pattern. """ def insert(self, word): """ Insert a word in the Vocabulary. """ def words(self): """ Return list of words. """ lib/python/Products/ZCatalog/help/Vocabulary.stx100644 1007 144 352 7123550016 21471 0ustar petrilliusersVocabulary Description Vocabularies display a list of all the unique words that are indexed by a Catalog or Catalogs that use this Vocabulary object. This view allows you to browse all of the words in the Vocabulary lib/python/Products/ZCatalog/help/Vocabulary_Query.stx100644 1007 144 232 7123550016 22653 0ustar petrilliusersVocabulary - Query: Query Vocabulary Description This form lets you query the Vocabulary for words that match a certain pattern. Controls lib/python/Products/ZCatalog/help/Vocabulary_Vocabulary.stx100644 1007 144 151 7123550016 23655 0ustar petrilliusersVocabulary - Vocabulary: Description here Description Details here Controls Details here lib/python/Products/ZCatalog/help/ZCatalog.py100644 1007 144 15526 7130205712 20747 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## def manage_addZCatalog(self, id, title, vocab_id=None): """ Add a ZCatalog object. 'vocab_id' is the name of a Vocabulary object this catalog should use. A value of None will cause the Catalog to create its own private vocabulary. """ class ZCatalog: """ ZCatalog object A ZCatalog contains arbitrary index like references to Zope objects. ZCatalog's can index either 'Field' values of object, 'Text' values, or 'KeyWord' values: Indexes Text -- Text indexes index textual content. The index can be used to search for objects containing certain words. Field -- Field indexes index atomic values. The index can be used to search for objects that have certain properties. Keyword -- Keyword indexes index sequences of values. The index can be used to search for objects that match one or more of the search terms. Meta-data The ZCatalog can maintain a table of extra data about cataloged objects. This information can be used on search result pages to show information about a search result. The meta-data table schema is used to build the schema for ZCatalog Result objects. The objects have the same attributes as the column of the meta-data table. ZCatalog does not store references to the objects themselves, but rather to a unique identifier that defines how to get to the object. In Zope, this unique identifier is the object's relative path to the ZCatalog (since two Zope object's cannot have the same URL, this is an excellent unique qualifier in Zope). """ __constructor__=manage_addZCatalog def catalog_object(self, obj, uid): """ Catalogs the object 'obj' with the unique identifier 'uid'. """ def uncatalog_object(self, uid): """ Uncatalogs the object with the unique identifier 'uid'. """ def uniqueValuesFor(self, name): """ returns the unique values for a given FieldIndex named 'name'. """ def getpath(self, rid): """ Return the path to a cataloged object given a 'data_record_id_' """ def getobject(self, rid, REQUEST=None): """ Return a cataloged object given a 'data_record_id_' """ def schema(self): """ Returns a sequence of names that correspond to columns in the meta-data table. """ def indexes(self): """ Returns a sequence of names that correspond to indexes. """ def index_objects(self): """ Returns a sequence of actual index objects. """ def searchResults(self, REQUEST=None, **kw): """ Search the catalog. Search terms can be passed in the REQUEST or as keyword arguments. XXX search grammar, how terms are combined, special search terms, format of returned results XXX """ def __call__(self, REQUEST=None, **kw): """ Search the catalog, the same way as 'searchResults'. """ lib/python/Products/ZCatalog/help/ZCatalog.stx100644 1007 144 1060 7122473374 21115 0ustar petrilliusersZCatalog: Index and Search for Zope objects Description ZCatalogs allow you to index and search for Zope objects. But ZCatalog is not just simply a data managemengt system that allows you to search through it content. You have the choice which properties are stored in the Catalog and which attributes are used for searching. Once a set of objects is inserted into the Catalog, you will be able to update this object inventory by updating the objects, deleting and adding them. lib/python/Products/ZCatalog/help/ZCatalog_Add.stx100644 1007 144 777 7122473374 21663 0ustar petrilliusersZCatalog - Add: Create new ZCatalog Description This view allows you to create a new ZCatalog. Controls 'Id' -- Allows you to specify the id of the ZCatalog. 'Title' -- Allows you to specify the title of the ZCatalog. 'Vocabulary' -- Allows you to select whether or not you would like to use a 'Vocabulary' and then choose the Vocabulary from the selection list. 'Add' -- Create a new and empty Catalog. lib/python/Products/ZCatalog/help/ZCatalog_Cataloged-Objects.stx100644 1007 144 3153 7122473374 24454 0ustar petrilliusersZCatalog - Cataloged Objects: Manage catalog entries Description This view allows you to manage catalog entries. The catalog holds records which refer to Zope objects. If the catalog currently contains records they will be listed in this view. Controls 'Update Catalog' -- Clicked, it will update the meta data for all of the objects in the Catalog. **Note:** The Catalog is not being updated after each transaction. 'Clear Catalog' -- Clicked, all objects are removed from the Catalog. 'Size' -- Indicates the amount of objects in the Catalog. '[List of objects]' -- Each lines describes one object. The checkbox in the front of the line, allows you to checkmark the object for deletion ('Delete') or to update the objects ('Update'). Next the object type is listed, followed by the object path and the object title. Clicking on the path and title of the object, you can examine individual catalog records by clicking on the record name links. 'Next/Previos XX entries' -- Allows you to see more of the cataloged objects. These teo links will only appear if you have enough objects in the Catalog. 'Update' -- Clicked, Zope greps the newest content for each checked object and updates the meta data (the cataloged attributes of the object) in the Catalog. 'Remove' -- Clicked, it removes all the selected objects from the Catalog. **Note:** No objects are deleted from the database. lib/python/Products/ZCatalog/help/ZCatalog_Find-Items-to-ZCatalog.stx100644 1007 144 530 7122473374 25237 0ustar petrilliusersZCatalog - Find Items to ZCatalog: Add records to the Catalog Description This view allows you to add records to the catalog. This view operates like a normal find view (see "Folder - Find") to locate objects for cataloging. Objects which match the find criteria will be added to the catalog. lib/python/Products/ZCatalog/help/ZCatalog_Indexes.stx100644 1007 144 3376 7122473374 22610 0ustar petrilliusersZCatalog - Indexes: Manage Catalog Indexes Description This view allows you to manage the catalog indexes. The catalog provides searching by indexing information about cataloged objects. Indexes record information about object attributes. Controls '[List of Indexes]' -- Each line contains one Index entry. The checkbox in the front allows you to mark an index for deletion. Next the Index nameis listed which corresponds to attribute names of various objects, is displayed. 'Delete' -- Clicked, the checkmarked Index(es) will be deleted. 'Add Index' - Name -- Allows you to enter the name of a new index. '[Type of Index]' -- You have the choice between three types of Indexes, called 'TextIndex', 'FieldIndex', and 'KeywordIndex'. Please refer to the table below to see a description of each Index. 'Add' -- Clicked, the new Index will be added to this catalog. This table lists the different Index types and describes their purpose. 'TextIndex' -- Breaks text up into individual words, and are often refered to as full-text indexes. Text indexes sort results by score meaning they return 'hits' in order from the most relevant to the lest relevant. 'FieldIndex' -- Treat the value of an objects attribute atomically, and can be used, for example, to track only a certain subset of object values, such as 'meta_type'. 'KeywordIndex' -- Index a sequence of objects that act as 'keywords' for an object. A Keyword Index will return any objects that have one or more keywords specified in a search query. lib/python/Products/ZCatalog/help/ZCatalog_MetaData-Table.stx100644 1007 144 1574 7122473374 23714 0ustar petrilliusersZCatalog - MetaData Table: Manage Catalog record schema Description This view allows you to manage the catalog record schema. The catalog holds records which describe the cataloged objects. This view allows you to define columns for these records. Controls '[List of Indexes]' -- Each line contains one Meta Data entry (column). The checkbox in the front allows you to mark a meta data entry for deletion. Next the meta data entry's name is listed which corresponds to attribute names of various objects, is displayed. 'Delete' -- Clicked, the checkmarked Meta Data entries (columns) will be deleted. 'Meta Data name' -- Allows you to enter the name of a new meta data entry. 'Add' -- Clicked, the new entry (column) will be added to this catalog. lib/python/Products/ZCatalog/help/ZCatalog_Status.stx100644 1007 144 2004 7122473374 22457 0ustar petrilliusersZCatalog - Status: Control advanced Catalog features Description This view allows you to control advanced catalog features. Subtransactions reduce the memory requirements of ZCatalog, but at the expense of speed. If you choose to enable subtransactions, you can adjust how often ZCatalog commits a subtransactions by adjusting the threshold. The index status reports how many objects are cataloged in each index. Controls 'Enable/Disable' -- If enables, the Subtransaction system is enabled and the threshold specified below is used. 'Subtransaction Threshold' -- Allows you to specify how often the ZCatalog will commit the subtransactions. The default value is 10000. 'Change' -- Clicked, Zope will update the subtransaction threshold. 'Index Status' List -- Each line corresponds to one Index. The line indicates how many objects are cataloged for each Index. lib/python/Products/ZCatalog/help/Vocabulary.pyc100644 1007 144 3235 7216440760 21500 0ustar petrilliusersN a9cs/VeedZadfdYZdS(cs V[dS(s< Add a Vocabulary object to an ObjectManager. N((sselfsidstitlesglobbingsREQUESTsZ/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/Vocabulary.pysmanage_addVocabularyVss Vocabularycs;adZhjeZldZtdZ{dZRS(s A Vocabulary manages words and language rules for text indexing. Text indexing is done by the ZCatalog and other third party Products. cs lqdS(s@ Query Vocabulary for words matching pattern. N((sselfspatternsZ/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/Vocabulary.pysquerylscs tydS(s4 Insert a word in the Vocabulary. N((sselfswordsZ/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/Vocabulary.pysinserttscs {dS(s) Return list of words. N((sselfsZ/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/Vocabulary.pyswords{s(s__doc__smanage_addVocabularys__constructor__squerysinsertswords(sZ/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/Vocabulary.pys Vocabularyas    N(sNonesmanage_addVocabularys Vocabulary(sZ/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/Vocabulary.pys?Vs lib/python/Products/ZCatalog/help/ZCatalog.pyc100644 1007 144 12355 7216440760 21120 0ustar petrilliusersN a9cs,VedZddfdYZdS(cs V_dS(s Add a ZCatalog object. 'vocab_id' is the name of a Vocabulary object this catalog should use. A value of None will cause the Catalog to create its own private vocabulary. N((sselfsidstitlesvocab_idsX/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/ZCatalog.pysmanage_addZCatalogVs sZCatalogcsddZeZdZdZdZdZedZdZ dZ dZ ed Z ed Z RS( sH ZCatalog object A ZCatalog contains arbitrary index like references to Zope objects. ZCatalog's can index either 'Field' values of object, 'Text' values, or 'KeyWord' values: Indexes Text -- Text indexes index textual content. The index can be used to search for objects containing certain words. Field -- Field indexes index atomic values. The index can be used to search for objects that have certain properties. Keyword -- Keyword indexes index sequences of values. The index can be used to search for objects that match one or more of the search terms. Meta-data The ZCatalog can maintain a table of extra data about cataloged objects. This information can be used on search result pages to show information about a search result. The meta-data table schema is used to build the schema for ZCatalog Result objects. The objects have the same attributes as the column of the meta-data table. ZCatalog does not store references to the objects themselves, but rather to a unique identifier that defines how to get to the object. In Zope, this unique identifier is the object's relative path to the ZCatalog (since two Zope object's cannot have the same URL, this is an excellent unique qualifier in Zope). cs dS(sO Catalogs the object 'obj' with the unique identifier 'uid'. N((sselfsobjsuidsX/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/ZCatalog.pyscatalog_objectscs dS(sK Uncatalogs the object with the unique identifier 'uid'. N((sselfsuidsX/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/ZCatalog.pysuncatalog_objectscs dS(sR returns the unique values for a given FieldIndex named 'name'. N((sselfsnamesX/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/ZCatalog.pysuniqueValuesForscs dS(sk Return the path to a cataloged object given a 'data_record_id_' N((sselfsridsX/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/ZCatalog.pysgetpathscs dS(sW Return a cataloged object given a 'data_record_id_' N((sselfsridsREQUESTsX/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/ZCatalog.pys getobjectscs dS(sj Returns a sequence of names that correspond to columns in the meta-data table. N((sselfsX/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/ZCatalog.pysschemascs dS(sK Returns a sequence of names that correspond to indexes. N((sselfsX/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/ZCatalog.pysindexesscs dS(s? Returns a sequence of actual index objects. N((sselfsX/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/ZCatalog.pys index_objectssc s dS(s Search the catalog. Search terms can be passed in the REQUEST or as keyword arguments. XXX search grammar, how terms are combined, special search terms, format of returned results XXX N((sselfsREQUESTskwsX/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/ZCatalog.pys searchResultss c s dS(sF Search the catalog, the same way as 'searchResults'. N((sselfsREQUESTskwsX/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/ZCatalog.pys__call__s(s__doc__smanage_addZCatalogs__constructor__scatalog_objectsuncatalog_objectsuniqueValuesForsgetpathsNones getobjectsschemasindexess index_objectss searchResultss__call__(sX/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/ZCatalog.pysZCatalogds %         N(sNonesmanage_addZCatalogsZCatalog(sX/home/petrilli/src/petrilli-Catalog-branch/lib/python/Products/ZCatalog/help/ZCatalog.pys?Vslib/python/Products/ZCatalog/manage_vocab.dtml100644 1007 144 2662 7121210102 21167 0ustar petrilliusers Edit <dtml-var title_or_id> contains word(s).

[Previous entries] [Next entries]
()
[Previous entries] [Next entries]

There are no words in the Vocabulary.

 lib/python/Products/ZCatalog/version.txt100644 1007 144 17 7121210102 20061 0ustar petrilliusersZCatalog-2-2-0 lib/python/Products/ZCatalog/vocab_manage_main.dtml100644 1007 144 1470 7045357555 22224 0ustar petrilliusers Edit <dtml-var title_or_id>

Edit

lib/python/Products/ZCatalog/vocab_query.dtml100644 1007 144 523 7045354423 21102 0ustar petrilliusers Ignore this file for now Edit <dtml-var title_or_id>
lib/python/Products/ZCatalog/www/ 40755 1007 144 0 7216440704 16446 5ustar petrilliuserslib/python/Products/ZCatalog/www/CVS/ 40755 1007 144 0 7216440704 17101 5ustar petrilliuserslib/python/Products/ZCatalog/www/CVS/Root100644 1007 144 20 7216440704 17774 0ustar petrilliusers/cvs-repository lib/python/Products/ZCatalog/www/CVS/Repository100644 1007 144 47 7216440704 21241 0ustar petrilliusersZope2/lib/python/Products/ZCatalog/www lib/python/Products/ZCatalog/www/CVS/Entries100644 1007 144 216 7216440704 20511 0ustar petrilliusers/Vocabulary.gif/1.1/Mon Jan 31 18:46:18 2000//Tpetrilli-Catalog-branch /ZCatalog.gif/1.1/Tue Jun 22 14:15:19 1999//Tpetrilli-Catalog-branch D lib/python/Products/ZCatalog/www/CVS/Tag100644 1007 144 31 7216440704 17566 0ustar petrilliusersTpetrilli-Catalog-branch lib/python/Products/ZCatalog/www/Vocabulary.gif100644 1007 144 346 7045354172 21327 0ustar petrilliusersGIF89aPPP@@@@@!Made with GIMP!,Q#dihJ l뺢 coFY\%iEB[l7H>r%op!Q0!;lib/python/Products/ZCatalog/www/ZCatalog.gif100644 1007 144 265 6733715167 20734 0ustar petrilliusersGIF89aƄ!,@bPHBmFOE0E"h"b[JI,Ljq9ieA8XL!FB/TRMxz/R0›a:#JOP |zT6V1;lib/python/ZPublisher/ 40755 1007 144 0 7217466751 14415 5ustar petrilliuserslib/python/ZPublisher/HTTPRequest.py100644 1007 144 120634 7216446626 17260 0ustar petrilliusers############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## __version__='$Revision: 1.39.2.1 $'[11:-2] import regex, re, sys, os, string, urllib from string import lower, atoi, rfind, split, strip, join, upper, find from BaseRequest import BaseRequest from HTTPResponse import HTTPResponse from cgi import FieldStorage from urllib import quote, unquote, splittype, splitport from Converters import get_converter from maybe_lock import allocate_lock xmlrpc=None # Placeholder for module that we'll import if we have to. isCGI_NAME = { 'SERVER_SOFTWARE' : 1, 'SERVER_NAME' : 1, 'GATEWAY_INTERFACE' : 1, 'SERVER_PROTOCOL' : 1, 'SERVER_PORT' : 1, 'REQUEST_METHOD' : 1, 'PATH_INFO' : 1, 'PATH_TRANSLATED' : 1, 'SCRIPT_NAME' : 1, 'QUERY_STRING' : 1, 'REMOTE_HOST' : 1, 'REMOTE_ADDR' : 1, 'AUTH_TYPE' : 1, 'REMOTE_USER' : 1, 'REMOTE_IDENT' : 1, 'CONTENT_TYPE' : 1, 'CONTENT_LENGTH' : 1, 'SERVER_URL': 1, }.has_key hide_key={'HTTP_AUTHORIZATION':1, 'HTTP_CGI_AUTHORIZATION': 1, }.has_key default_port={'http': '80', 'https': '443'} _marker=[] class HTTPRequest(BaseRequest): """\ Model HTTP request data. This object provides access to request data. This includes, the input headers, form data, server data, and cookies. Request objects are created by the object publisher and will be passed to published objects through the argument name, REQUEST. The request object is a mapping object that represents a collection of variable to value mappings. In addition, variables are divided into four categories: - Environment variables These variables include input headers, server data, and other request-related data. The variable names are as specified in the CGI specification - Form data These are data extracted from either a URL-encoded query string or body, if present. - Cookies These are the cookie data, if present. - Other Data that may be set by an application object. The form attribute of a request is actually a Field Storage object. When file uploads are used, this provides a richer and more complex interface than is provided by accessing form data as items of the request. See the FieldStorage class documentation for more details. The request object may be used as a mapping object, in which case values will be looked up in the order: environment variables, other variables, form data, and then cookies. """ _hacked_path=None args=() _file=None _urls = () retry_max_count=3 def supports_retry(self): return self.retry_count < self.retry_max_count def retry(self): self.retry_count=self.retry_count+1 self.stdin.seek(0) r=self.__class__(stdin=self.stdin, environ=self._orig_env, response=self.response.retry() ) r.retry_count=self.retry_count return r def setServerURL(self, protocol=None, hostname=None, port=None): """ Set the parts of generated URLs. """ other = self.other server_url = other.get('SERVER_URL', '') if protocol is None and hostname is None and port is None: return server_url oldprotocol, oldhost = splittype(server_url) oldhostname, oldport = splitport(oldhost[2:]) if protocol is None: protocol = oldprotocol if hostname is None: hostname = oldhostname if port is None: port = oldport if (port is None or default_port[protocol] == port): host = hostname else: host = hostname + ':' + port server_url = other['SERVER_URL'] = '%s://%s' % (protocol, host) self._resetURLS() return server_url def setVirtualRoot(self, path, hard=0): """ Treat the current publishing object as a VirtualRoot """ other = self.other if type(path) is type(''): path = filter(None, split(path, '/')) self._script[:] = map(quote, path) del self._steps[:] parents = other['PARENTS'] if hard: del parents[:-1] other['VirtualRootPhysicalPath'] = parents[-1].getPhysicalPath() self._resetURLS() def physicalPathToVirtualPath(self, path): """ Remove the path to the VirtualRoot from a physical path """ if type(path) is type(''): path = split(path, '/') rpp = self.other.get('VirtualRootPhysicalPath', ('',)) i = 0 for name in rpp[:len(path)]: if path[i] == name: i = i + 1 else: break return path[i:] def physicalPathToURL(self, path, relative=0): """ Convert a physical path into a URL in the current context """ path = self._script + map(quote, self.physicalPathToVirtualPath(path)) if relative: path.insert(0, '') else: path.insert(0, self['SERVER_URL']) return join(path, '/') def _resetURLS(self): other = self.other other['URL'] = join([other['SERVER_URL']] + self._script + self._steps, '/') for x in self._urls: del self.other[x] self._urls = () def __init__(self, stdin, environ, response, clean=0): self._orig_env=environ # Avoid the overhead of scrubbing the environment in the # case of request cloning for traversal purposes. If the # clean flag is set, we know we can use the passed in # environ dict directly. if not clean: environ=sane_environment(environ) if environ.has_key('HTTP_AUTHORIZATION'): self._auth=environ['HTTP_AUTHORIZATION'] response._auth=1 del environ['HTTP_AUTHORIZATION'] self.stdin=stdin self.environ=environ have_env=environ.has_key get_env=environ.get self.response=response other=self.other={'RESPONSE': response} self.form={} self.steps=[] self._steps=[] ################################################################ # Get base info first. This isn't likely to cause # errors and might be useful to error handlers. b=script=strip(get_env('SCRIPT_NAME','')) # _script and the other _names are meant for URL construction self._script = map(quote, filter(None, split(script, '/'))) while b and b[-1]=='/': b=b[:-1] p = rfind(b,'/') if p >= 0: b=b[:p+1] else: b='' while b and b[0]=='/': b=b[1:] server_url=get_env('SERVER_URL',None) if server_url is not None: other['SERVER_URL'] = server_url = strip(server_url) else: if have_env('HTTPS') and ( environ['HTTPS'] == "on" or environ['HTTPS'] == "ON"): protocol = 'https' elif (have_env('SERVER_PORT_SECURE') and environ['SERVER_PORT_SECURE'] == "1"): protocol = 'https' else: protocol = 'http' if have_env('HTTP_HOST'): host = strip(environ['HTTP_HOST']) hostname, port = splitport(host) # some clients manage to forget the port :( if port is None and environ.has_key('SERVER_PORT'): s_port=environ['SERVER_PORT'] if s_port not in ('80', '443'): port=s_port else: hostname = strip(environ['SERVER_NAME']) port = environ['SERVER_PORT'] self.setServerURL(protocol=protocol, hostname=hostname, port=port) server_url = other['SERVER_URL'] if server_url[-1:]=='/': server_url=server_url[:-1] if b: self.base="%s/%s" % (server_url,b) else: self.base=server_url while script[:1]=='/': script=script[1:] if script: script="%s/%s" % (server_url,script) else: script=server_url other['URL']=self.script=script ################################################################ # Cookie values should *not* be appended to existing form # vars with the same name - they are more like default values # for names not otherwise specified in the form. cookies={} k=get_env('HTTP_COOKIE','') if k: parse_cookie(k, cookies) for k,item in cookies.items(): if not other.has_key(k): other[k]=item self.cookies=cookies def processInputs( self, # "static" variables that we want to be local for speed SEQUENCE=1, DEFAULT=2, RECORD=4, RECORDS=8, REC=12, # RECORD|RECORDS EMPTY=16, CONVERTED=32, hasattr=hasattr, getattr=getattr, setattr=setattr, search_type=regex.compile(':[a-zA-Z][a-zA-Z0-9_]+$').search, rfind=string.rfind, ): """Process request inputs We need to delay input parsing so that it is done under publisher control for error handling purposes. """ response=self.response environ=self.environ method=environ.get('REQUEST_METHOD','GET') if method != 'GET': fp=self.stdin else: fp=None form=self.form other=self.other meth=None fs=FieldStorage(fp=fp,environ=environ,keep_blank_values=1) if not hasattr(fs,'list') or fs.list is None: # Hm, maybe it's an XML-RPC if (fs.headers.has_key('content-type') and fs.headers['content-type'] == 'text/xml' and method == 'POST'): # Ye haaa, XML-RPC! global xmlrpc if xmlrpc is None: import xmlrpc meth, self.args = xmlrpc.parse_input(fs.value) response=xmlrpc.response(response) other['RESPONSE']=self.response=response other['REQUEST_METHOD']='' # We don't want index_html! self.maybe_webdav_client = 0 else: self._file=fs.file else: fslist=fs.list tuple_items={} lt=type([]) CGI_name=isCGI_NAME defaults={} converter=seqf=None for item in fslist: key=item.name if (hasattr(item,'file') and hasattr(item,'filename') and hasattr(item,'headers')): if (item.file and (item.filename is not None # RFC 1867 says that all fields get a content-type. # or 'content-type' in map(lower, item.headers.keys()) )): item=FileUpload(item) else: item=item.value flags=0 # Loop through the different types and set # the appropriate flags # We'll search from the back to the front. # We'll do the search in two steps. First, we'll # do a string search, and then we'll check it with # a regex search. l=rfind(key,':') if l >= 0: l=search_type(key,l) while l >= 0: type_name=key[l+1:] key=key[:l] c=get_converter(type_name, None) if c is not None: converter=c flags=flags|CONVERTED elif type_name == 'list': seqf=list flags=flags|SEQUENCE elif type_name == 'tuple': seqf=tuple tuple_items[key]=1 flags=flags|SEQUENCE elif type_name == 'method': if l: meth=key else: meth=item elif type_name == 'default_method': if not meth: if l: meth=key else: meth=item elif type_name == 'default': flags=flags|DEFAULT elif type_name == 'record': flags=flags|RECORD elif type_name == 'records': flags=flags|RECORDS elif type_name == 'ignore_empty': if not item: flags=flags|EMPTY l=rfind(key,':') if l < 0: break l=search_type(key,l) # Filter out special names from form: if CGI_name(key) or key[:5]=='HTTP_': continue if flags: # skip over empty fields if flags&EMPTY: continue #Split the key and its attribute if flags&REC: key=split(key,".") key, attr=join(key[:-1],"."), key[-1] # defer conversion if flags&CONVERTED: try: item=converter(item) except: if (not item and not (flags&DEFAULT) and defaults.has_key(key)): item = defaults[key] if flags&RECORD: item=getattr(item,attr) if flags&RECORDS: item.reverse() item = item[0] item=getattr(item,attr) else: raise #Determine which dictionary to use if flags&DEFAULT: mapping_object = defaults else: mapping_object = form #Insert in dictionary if mapping_object.has_key(key): if flags&RECORDS: #Get the list and the last record #in the list reclist = mapping_object[key] reclist.reverse() x=reclist[0] reclist.reverse() if not hasattr(x,attr): #If the attribute does not #exist, setit if flags&SEQUENCE: item=[item] reclist.remove(x) setattr(x,attr,item) reclist.append(x) mapping_object[key] = reclist else: if flags&SEQUENCE: # If the attribute is a # sequence, append the item # to the existing attribute reclist.remove(x) y = getattr(x, attr) y.append(item) setattr(x, attr, y) reclist.append(x) mapping_object[key] = reclist else: # Create a new record and add # it to the list n=record() setattr(n,attr,item) reclist.append(n) mapping_object[key]=reclist elif flags&RECORD: b=mapping_object[key] if flags&SEQUENCE: item=[item] if not hasattr(b,attr): # if it does not have the # attribute, set it setattr(b,attr,item) else: # it has the attribute so # append the item to it setattr(b,attr,getattr(b,attr)+item) else: # it is not a sequence so # set the attribute setattr(b,attr,item) else: # it is not a record or list of records found=mapping_object[key] if type(found) is lt: found.append(item) else: found=[found,item] mapping_object[key]=found else: # The dictionary does not have the key if flags&RECORDS: # Create a new record, set its attribute # and put it in the dictionary as a list a = record() if flags&SEQUENCE: item=[item] setattr(a,attr,item) mapping_object[key]=[a] elif flags&RECORD: # Create a new record, set its attribute # and put it in the dictionary if flags&SEQUENCE: item=[item] r = mapping_object[key]=record() setattr(r,attr,item) else: # it is not a record or list of records if flags&SEQUENCE: item=[item] mapping_object[key]=item else: # This branch is for case when no type was specified. mapping_object = form #Insert in dictionary if mapping_object.has_key(key): # it is not a record or list of records found=mapping_object[key] if type(found) is lt: found.append(item) else: found=[found,item] mapping_object[key]=found else: mapping_object[key]=item #insert defaults into form dictionary if defaults: for keys, values in defaults.items(): if not form.has_key(keys): # if the form does not have the key, # set the default form[keys]=values else: #The form has the key if getattr(values, '__class__',0) is record: # if the key is mapped to a record, get the # record r = form[keys] for k, v in values.__dict__.items(): # loop through the attributes and values # in the default dictionary if not hasattr(r, k): # if the form dictionary doesn't have # the attribute, set it to the default setattr(r,k,v) form[keys] = r elif values == type([]): # the key is mapped to a list l = form[keys] for x in values: # for each x in the list if getattr(x, '__class__',0) is record: # if the x is a record for k, v in x.__dict__.items(): # loop through each # attribute and value in # the record for y in l: # loop through each # record in the form # list if it doesn't # have the attributes # in the default # dictionary, set them if not hasattr(y, k): setattr(y, k, v) else: # x is not a record if not a in l: l.append(a) form[keys] = l else: # The form has the key, the key is not mapped # to a record or sequence so do nothing pass # Convert to tuples if tuple_items: for key in tuple_items.keys(): # Split the key and get the attr k=split(key, ".") k,attr=join(k[:-1], "."), k[-1] a = attr # remove any type_names in the attr while not a=='': a=split(a, ":") a,new=join(a[:-1], ":"), a[-1] attr = new if form.has_key(k): # If the form has the split key get its value item =form[k] if (hasattr(item, '__class__') and item.__class__ is record): # if the value is mapped to a record, check if it # has the attribute, if it has it, convert it to # a tuple and set it if hasattr(item,attr): value=tuple(getattr(item,attr)) setattr(item,attr,value) else: # It is mapped to a list of records for x in item: # loop through the records if hasattr(x, attr): # If the record has the attribute # convert it to a tuple and set it value=tuple(getattr(x,attr)) setattr(x,attr,value) else: # the form does not have the split key if form.has_key(key): # if it has the original key, get the item # convert it to a tuple item=form[key] item=tuple(form[key]) form[key]=item other.update(form) if meth: if environ.has_key('PATH_INFO'): path=environ['PATH_INFO'] while path[-1:]=='/': path=path[:-1] else: path='' other['PATH_INFO']=path="%s/%s" % (path,meth) self._hacked_path=1 def resolve_url(self, url): # Attempt to resolve a url into an object in the Zope # namespace. The url must be a fully-qualified url. The # method will return the requested object if it is found # or raise the same HTTP error that would be raised in # the case of a real web request. If the passed in url # does not appear to describe an object in the system # namespace (e.g. the host, port or script name dont # match that of the current request), a ValueError will # be raised. if find(url, self.script) != 0: raise ValueError, 'Different namespace.' path=url[len(self.script):] while path and path[0]=='/': path=path[1:] while path and path[-1]=='/': path=path[:-1] req=self.clone() rsp=req.response req['PATH_INFO']=path object=None # Try to traverse to get an object. Note that we call # the exception method on the response, but we don't # want to actually abort the current transaction # (which is usually the default when the exception # method is called on the response). try: object=req.traverse(path) except: rsp.exception() if object is None: req.close() raise rsp.errmsg, sys.exc_info()[1] # The traversal machinery may return a "default object" # like an index_html document. This is not appropriate # in the context of the resolve_url method so we need # to ensure we are getting the actual object named by # the given url, and not some kind of default object. if hasattr(object, 'id'): if callable(object.id): name=object.id() else: name=object.id elif hasattr(object, '__name__'): name=object.__name__ else: name='' if name != os.path.split(path)[-1]: object=req.PARENTS[0] req.close() return object def clone(self): # Return a clone of the current request object # that may be used to perform object traversal. environ=self.environ.copy() environ['REQUEST_METHOD']='GET' if self._auth: environ['HTTP_AUTHORIZATION']=self._auth clone=HTTPRequest(None, environ, HTTPResponse(), clean=1) clone['PARENTS']=[self['PARENTS'][-1]] return clone def get_header(self, name, default=None): """Return the named HTTP header, or an optional default argument or None if the header is not found. Note that both original and CGI-ified header names are recognized, e.g. 'Content-Type', 'CONTENT_TYPE' and 'HTTP_CONTENT_TYPE' should all return the Content-Type header, if available. """ environ=self.environ name=upper(join(split(name,"-"),"_")) val=environ.get(name, None) if val is not None: return val if name[:5] != 'HTTP_': name='HTTP_%s' % name return environ.get(name, default) def __getitem__(self,key, default=_marker, # Any special internal marker will do URLmatch=re.compile('URL(PATH)?([0-9]+)$').match, BASEmatch=re.compile('BASE(PATH)?([0-9]+)$').match, ): """Get a variable value Return a value for the required variable name. The value will be looked up from one of the request data categories. The search order is environment variables, other variables, form data, and then cookies. """ #" other=self.other if other.has_key(key): if key=='REQUEST': return self return other[key] if key[:1]=='U': match = URLmatch(key) if match is not None: pathonly, n = match.groups() path = self._script + self._steps n = len(path) - int(n) if n < 0: raise KeyError, key if pathonly: path = [''] + path[:n] else: path = [other['SERVER_URL']] + path[:n] other[key] = URL = join(path, '/') self._urls = self._urls + (key,) return URL if isCGI_NAME(key) or key[:5] == 'HTTP_': environ=self.environ if environ.has_key(key) and (not hide_key(key)): return environ[key] return '' if key=='REQUEST': return self if key[:1]=='B': match = BASEmatch(key) if match is not None: pathonly, n = match.groups() path = self._steps n = int(n) if n: n = n - 1 if len(path) < n: raise KeyError, key v = self._script + path[:n] else: v = self._script[:-1] if pathonly: v.insert(0, '') else: v.insert(0, other['SERVER_URL']) other[key] = URL = join(v, '/') self._urls = self._urls + (key,) return URL if key=='BODY' and self._file is not None: p=self._file.tell() self._file.seek(0) v=self._file.read() self._file.seek(p) self.other[key]=v return v if key=='BODYFILE' and self._file is not None: v=self._file self.other[key]=v return v v=self.common.get(key, default) if v is not _marker: return v raise KeyError, key __getattr__=__getitem__ def get(self, key, default=None): return self.__getitem__(key, default) def has_key(self, key): try: self[key] except: return 0 else: return 1 def keys(self): keys = {} keys.update(self.common) for key in self.environ.keys(): if (isCGI_NAME(key) or key[:5] == 'HTTP_') and \ (not hide_key(key)): keys[key] = 1 n=0 while 1: n=n+1 key = "URL%s" % n if not self.has_key(key): break n=0 while 1: n=n+1 key = "BASE%s" % n if not self.has_key(key): break keys.update(self.other) keys=keys.keys() keys.sort() return keys def __str__(self): result="

form

" row='' for k,v in self.form.items(): result=result + row % (html_quote(k), html_quote(v)) result=result+"
%s%s

cookies

" for k,v in self.cookies.items(): result=result + row % (html_quote(k), html_quote(v)) result=result+"

other

" for k,v in self.other.items(): if k in ('PARENTS','RESPONSE'): continue result=result + row % (html_quote(k), html_quote(v)) for n in "0123456789": key = "URL%s"%n try: result=result + row % (key, html_quote(self[key])) except KeyError: pass for n in "0123456789": key = "BASE%s"%n try: result=result + row % (key, html_quote(self[key])) except KeyError: pass result=result+"

environ

" for k,v in self.environ.items(): if not hide_key(k): result=result + row % (html_quote(k), html_quote(v)) return result+"
" __repr__=__str__ def _authUserPW(self): global base64 auth=self._auth if auth: if lower(auth[:6]) == 'basic ': if base64 is None: import base64 [name,password] = split( base64.decodestring(split(auth)[-1]), ':') return name, password base64=None def sane_environment(env): # return an environment mapping which has been cleaned of # funny business such as REDIRECT_ prefixes added by Apache # or HTTP_CGI_AUTHORIZATION hacks. dict={} for key, val in env.items(): while key[:9]=='REDIRECT_': key=key[9:] dict[key]=val if dict.has_key('HTTP_CGI_AUTHORIZATION'): dict['HTTP_AUTHORIZATION']=dict['HTTP_CGI_AUTHORIZATION'] try: del dict['HTTP_CGI_AUTHORIZATION'] except: pass return dict # This is duplicated from DocumentTemplate.DT_Util to # prevent a dependency on the DocumentTemplate package. # Some folks still use the ZPublisher package as a # standalone publisher without DocumentTemplate. def html_quote(value, character_entities=( (('&'), '&'), (("<"), '<' ), ((">"), '>' ), (('"'), '"'))): #" text=str(value) for re, name in character_entities: if find(text, re) >= 0: text=join(split(text, re), name) return text def str_field(v): if type(v) is ListType: return map(str_field,v) if hasattr(v,'__class__') and v.__class__ is FieldStorage: v=v.value elif type(v) is not StringType: if hasattr(v,'file') and v.file: v=v.file elif hasattr(v,'value'): v=v.value return v class FileUpload: '''\ File upload objects File upload objects are used to represent file-uploaded data. File upload objects can be used just like files. In addition, they have a 'headers' attribute that is a dictionary containing the file-upload headers, and a 'filename' attribute containing the name of the uploaded file. ''' # Allow access to attributes such as headers and filename so # that ZClass authors can use DTML to work with FileUploads. __allow_access_to_unprotected_subobjects__=1 def __init__(self, aFieldStorage): file=aFieldStorage.file if hasattr(file, '__methods__'): methods=file.__methods__ else: methods= ['close', 'fileno', 'flush', 'isatty', 'read', 'readline', 'readlines', 'seek', 'tell', 'truncate', 'write', 'writelines'] d=self.__dict__ for m in methods: if hasattr(file,m): d[m]=getattr(file,m) self.headers=aFieldStorage.headers self.filename=aFieldStorage.filename parse_cookie_lock=allocate_lock() def parse_cookie(text, result=None, qparmre=regex.compile( '\([\0- ]*' '\([^\0- ;,=\"]+\)="\([^"]*\)\"' '\([\0- ]*[;,]\)?[\0- ]*\)' ), parmre=regex.compile( '\([\0- ]*' '\([^\0- ;,=\"]+\)=\([^\0- ;,\"]*\)' '\([\0- ]*[;,]\)?[\0- ]*\)' ), acquire=parse_cookie_lock.acquire, release=parse_cookie_lock.release, ): if result is None: result={} already_have=result.has_key acquire() try: if qparmre.match(text) >= 0: # Match quoted correct cookies name=qparmre.group(2) value=qparmre.group(3) l=len(qparmre.group(1)) elif parmre.match(text) >= 0: # Match evil MSIE cookies ;) name=parmre.group(2) value=parmre.group(3) l=len(parmre.group(1)) else: # this may be an invalid cookie. # We'll simply bail without raising an error # if the cookie is invalid. return result finally: release() if not already_have(name): result[name]=value return apply(parse_cookie,(text[l:],result)) # add class class record: # Allow access to record methods and values from DTML __allow_access_to_unprotected_subobjects__=1 def __getattr__(self, key, default=None): if key in ('get', 'keys', 'items', 'values', 'copy', 'has_key'): return getattr(self.__dict__, key) raise AttributeError, key def __getitem__(self, key): return self.__dict__[key] def __str__(self): L1 = self.__dict__.items() L1.sort() return join(map(lambda item: "%s: %s" %item, L1), ", ") __repr__ = __str__ # Flags SEQUENCE=1 DEFAULT=2 RECORD=4 RECORDS=8 REC=RECORD|RECORDS EMPTY=16 CONVERTED=32