Source Code for Module grassyknoll.tests.nsf

 1  "descriptions of U{NSF Research awards<http://kdd.ics.uci.edu/databases/nsfabs/nsfawards.html>} data" 
 2   
 3  from grassyknoll.lib import Norman 
 4  from grassyknoll.backend.litesql import TableMaker 
 5  from grassyknoll.backend.lucene import LuceneCollection 
 6  import datetime 
 7   
 8  ## NSFNorman describes how to normalize & convert raw data to a document ## 
 9  NSFNorman=Norman.ObjectNorman(unknown='delete') 
10   
11  # unicode fields 
12  NSFNorman.Fld_Applictn = Norman.UnicodeNorman() 
13  NSFNorman.Investigator = Norman.UnicodeNorman() 
14  NSFNorman.Abstract = Norman.UnicodeNorman() 
15  NSFNorman.Award_Instr = Norman.UnicodeNorman() 
16  NSFNorman.Prgm_Manager = Norman.UnicodeNorman() 
17  NSFNorman.Sponsor = Norman.UnicodeNorman() 
18  NSFNorman.File = Norman.UnicodeNorman() 
19  NSFNorman.Program_Ref = Norman.UnicodeNorman() 
20  NSFNorman.NSF_Org = Norman.UnicodeNorman() 
21  NSFNorman.Type = Norman.UnicodeNorman() 
22  NSFNorman.Title = Norman.UnicodeNorman() 
23  NSFNorman.Award_Number = Norman.UnicodeNorman() 
24   
25  # date fields 
26  NSFNorman.Date = Norman.DateNorman() 
27  NSFNorman.Start_Date = Norman.DateNorman() 
28  NSFNorman.Expires = Norman.DateNorman() 
29   
30  # integer fields 
31  NSFNorman.NSF_Program = Norman.IntegerNorman() 
32  NSFNorman.Total_Amt = Norman.IntegerNorman() 
33   
34  # optional s.t. it can be used w/ both ClientCollection & other backends 
35  NSFNorman.__id__ = Norman.UnicodeNorman(optional=True) 
36   
37  ## NSFTable: describes how to store documents in SqliteCollection backend ## 
38  NSFTable=TableMaker.Table('nsf') 
39   
40  # unicode columns 
41  NSFTable.Fld_Applictn = TableMaker.Column(unicode) 
42  NSFTable.Investigator = TableMaker.Column(unicode) 
43  NSFTable.Abstract = TableMaker.Column(unicode) 
44  NSFTable.Award_Instr = TableMaker.Column(unicode) 
45  NSFTable.Prgm_Manager = TableMaker.Column(unicode) 
46  NSFTable.Sponsor = TableMaker.Column(unicode) 
47  NSFTable.File = TableMaker.Column(unicode) 
48  NSFTable.Program_Ref = TableMaker.Column(unicode) 
49  NSFTable.NSF_Org = TableMaker.Column(unicode) 
50  NSFTable.Type = TableMaker.Column(unicode) 
51  NSFTable.Title = TableMaker.Column(unicode) 
52  NSFTable.Award_Number = TableMaker.Column(unicode) 
53   
54  # date columns 
55  NSFTable.Date = TableMaker.Column(datetime.date) 
56  NSFTable.Start_Date = TableMaker.Column(datetime.date) 
57  NSFTable.Expires = TableMaker.Column(datetime.date) 
58   
59  # integer columns 
60  NSFTable.NSF_Program = TableMaker.Column(int) 
61  NSFTable.Total_Amt = TableMaker.Column(int) 
62   
63  ## NSFLuceneNorman: describes how to store documents in LuceneCollection backend ## 
64  NSFLuceneNorman=LuceneCollection.LuceneNorman() 
65   
66  # non-text and limited value fields are not tokenized 
67  NSFLuceneNorman.Start_Date = LuceneCollection.SmartFieldNorman(store=True, index='untokenized') 
68  NSFLuceneNorman.Expires = LuceneCollection.SmartFieldNorman(store=True, index='untokenized') 
69  NSFLuceneNorman.Date = LuceneCollection.SmartFieldNorman(store=True, index='untokenized') 
70  NSFLuceneNorman.Type = LuceneCollection.SmartFieldNorman(store=True, index='untokenized') 
71  NSFLuceneNorman.Program_Ref = LuceneCollection.SmartFieldNorman(store=True, index='untokenized') 
72  NSFLuceneNorman.Total_Amt = LuceneCollection.SmartFieldNorman(store=True, index='untokenized') 
73   
74  # free text fields are tokenized 
75  NSFLuceneNorman.Fld_Applictn = LuceneCollection.SmartFieldNorman(store=True, index='tokenized', alltext=True) 
76  NSFLuceneNorman.Investigator = LuceneCollection.SmartFieldNorman(store=True, index='tokenized', alltext=True) 
77  NSFLuceneNorman.Abstract = LuceneCollection.SmartFieldNorman(store=True, index='tokenized', alltext=True) 
78  NSFLuceneNorman.Award_Instr = LuceneCollection.SmartFieldNorman(store=True, index='tokenized', alltext=True) 
79  NSFLuceneNorman.Prgm_Manager = LuceneCollection.SmartFieldNorman(store=True, index='tokenized', alltext=True) 
80  NSFLuceneNorman.Sponsor = LuceneCollection.SmartFieldNorman(store=True, index='tokenized', alltext=True) 
81  NSFLuceneNorman.File = LuceneCollection.SmartFieldNorman(store=True, index='tokenized', alltext=True) 
82  NSFLuceneNorman.NSF_Program = LuceneCollection.SmartFieldNorman(store=True, index='tokenized', alltext=True) 
83  NSFLuceneNorman.NSF_Org = LuceneCollection.SmartFieldNorman(store=True, index='tokenized', alltext=True) 
84   
85  # stored fields - these can be retrieved with result 
86  NSFLuceneNorman.Title = LuceneCollection.SmartFieldNorman(store=True, index='tokenized', alltext=True) 
87  NSFLuceneNorman.Award_Number = LuceneCollection.SmartFieldNorman(store=True, index='untokenized', alltext=True) 
88