← Back to branch summary

~loggerhead-team/loggerhead/trunk-rich

« back to all changes in this revision

Viewing changes to loggerhead/changecache.py

  • Committer: Robey Pointer
  • Date: 2007-01-14 05:40:40 UTC
  • Revision ID: robey@lag.net-20070114054040-7i9lbhq992e612rq
fix up dev.cfg so that nobody will ever have to edit it, by letting the
important params be overridable in loggerhead.conf.

make start-loggerhead actually daemonize, write a pid file, and write logs
to normal log files, instead of requiring 'nohup' stuff.  ie act like a real
server.  added stop-loggerhead to do a clean shutdown.  changed the README
to clarify how it should work now.

Show diffs side-by-side

added added

removed removed

Lines of Context:
loggerhead/changecache.py
48
48
        if not os.path.exists(cache_path):
49
49
            os.mkdir(cache_path)
50
50
 
 
51
        # keep a separate cache for the diffs, because they're very time-consuming to fetch.
51
52
        self._changes_filename = os.path.join(cache_path, 'changes')
 
53
        self._changes_diffs_filename = os.path.join(cache_path, 'changes-diffs')
52
54
        
53
55
        # use a lockfile since the cache folder could be shared across different processes.
54
56
        self._lock = LockFile(os.path.join(cache_path, 'lock'))
56
58
        
57
59
        # this is fluff; don't slow down startup time with it.
58
60
        def log_sizes():
59
 
            self.log.info('Using change cache %s; %d entries.' % (cache_path, self.size()))
 
61
            s1, s2 = self.sizes()
 
62
            self.log.info('Using change cache %s; %d/%d entries.' % (cache_path, s1, s2))
60
63
        threading.Thread(target=log_sizes).start()
61
64
    
62
65
    @with_lock
73
76
        pass
74
77
    
75
78
    @with_lock
76
 
    def get_changes(self, revid_list):
 
79
    def get_changes(self, revid_list, get_diffs=False):
77
80
        """
78
81
        get a list of changes by their revision_ids.  any changes missing
79
82
        from the cache are fetched by calling L{History.get_change_uncached}
80
83
        and inserted into the cache before returning.
81
84
        """
82
 
        cache = shelve.open(self._changes_filename, 'c', protocol=2)
83
 
 
84
 
        try:
85
 
            out = []
86
 
            fetch_list = []
87
 
            sfetch_list = []
88
 
            for revid in revid_list:
89
 
                # if the revid is in unicode, use the utf-8 encoding as the key
90
 
                srevid = util.to_utf8(revid)
91
 
 
92
 
                if srevid in cache:
93
 
                    out.append(cache[srevid])
94
 
                else:
95
 
                    #self.log.debug('Entry cache miss: %r' % (revid,))
96
 
                    out.append(None)
97
 
                    fetch_list.append(revid)
98
 
                    sfetch_list.append(srevid)
99
 
 
100
 
            if len(fetch_list) > 0:
101
 
                # some revisions weren't in the cache; fetch them
102
 
                changes = self.history.get_changes_uncached(fetch_list)
103
 
                if len(changes) == 0:
104
 
                    return changes
105
 
                for i in xrange(len(revid_list)):
106
 
                    if out[i] is None:
107
 
                        cache[sfetch_list.pop(0)] = out[i] = changes.pop(0)
108
 
            return out
109
 
        finally:
110
 
            cache.close()
 
85
        if get_diffs:
 
86
            cache = shelve.open(self._changes_diffs_filename, 'c', protocol=2)
 
87
        else:
 
88
            cache = shelve.open(self._changes_filename, 'c', protocol=2)
 
89
 
 
90
        out = []
 
91
        fetch_list = []
 
92
        sfetch_list = []
 
93
        for revid in revid_list:
 
94
            # if the revid is in unicode, use the utf-8 encoding as the key
 
95
            srevid = util.to_utf8(revid)
 
96
            
 
97
            if srevid in cache:
 
98
                out.append(cache[srevid])
 
99
            else:
 
100
                #self.log.debug('Entry cache miss: %r' % (revid,))
 
101
                out.append(None)
 
102
                fetch_list.append(revid)
 
103
                sfetch_list.append(srevid)
 
104
        
 
105
        if len(fetch_list) > 0:
 
106
            # some revisions weren't in the cache; fetch them
 
107
            changes = self.history.get_changes_uncached(fetch_list, get_diffs)
 
108
            if changes is None:
 
109
                return changes
 
110
            for i in xrange(len(revid_list)):
 
111
                if out[i] is None:
 
112
                    cache[sfetch_list.pop(0)] = out[i] = changes.pop(0)
 
113
        
 
114
        cache.close()
 
115
        return out
111
116
    
112
117
    @with_lock
113
 
    def full(self):
114
 
        cache = shelve.open(self._changes_filename, 'c', protocol=2)
 
118
    def full(self, get_diffs=False):
 
119
        if get_diffs:
 
120
            cache = shelve.open(self._changes_diffs_filename, 'c', protocol=2)
 
121
        else:
 
122
            cache = shelve.open(self._changes_filename, 'c', protocol=2)
115
123
        try:
116
124
            return (len(cache) >= len(self.history.get_revision_history())) and (util.to_utf8(self.history.last_revid) in cache)
117
125
        finally:
118
126
            cache.close()
119
127
 
120
128
    @with_lock
121
 
    def size(self):
 
129
    def sizes(self):
122
130
        cache = shelve.open(self._changes_filename, 'c', protocol=2)
123
131
        s1 = len(cache)
124
132
        cache.close()
125
 
        return s1
 
133
        cache = shelve.open(self._changes_diffs_filename, 'c', protocol=2)
 
134
        s2 = len(cache)
 
135
        cache.close()
 
136
        return s1, s2
126
137
        
127
138
    def check_rebuild(self, max_time=3600):
128
139
        """