Ticket #89: pxd-cache.patch

a/Cython/Compiler/Code.py

@@ -345 +345 @@
-        If name is provided, it is used as an identifier to avoid inserting
-        code twice. Otherwise, id(codetup) is used as such an identifier.
-        """
-id(codetup)
+codetup[0]
-        if self.check_utility_code_needed_and_register(name):
-            proto, _def = codetup
-            self.utilprotowriter.put(proto)

a/Cython/Compiler/Main.py

@@ -2 +2 @@
-#   Cython Top Level
-#
-
-
+, copy
-if sys.version_info[:2] < (2, 3):
-    sys.stderr.write("Sorry, Cython requires Python 2.3 or later\n")
-    sys.exit(1)
+import cPickle as pickle
-
-try:
-    set
-except NameError:
-    # Python 2.3
-    from sets import Set as set
-
+
+
+
+
+
+
-from time import time
-import Code
-import Errors
@@ -59 +65 @@
-    #  include_directories   [string]
-    #  future_directives     [object]
-    
-
+, pxd_cache=None
-        #self.modules = {"__builtin__" : BuiltinScope()}
-        import Builtin, CythonScope
-        self.modules = {"__builtin__" : Builtin.builtin_scope}
@@ -73 +79 @@
-        standard_include_path = os.path.abspath(
-            os.path.join(os.path.dirname(__file__), '..', 'Includes'))
-        self.include_directories = include_directories + [standard_include_path]
+        self.pxd_cache = pxd_cache
+        
-
-    def create_pipeline(self, pxd):
-        from Visitor import PrintTree
@@ -177 +185 @@
-            if Errors.num_errors == 0:
-                raise
-        return (err, data)
-
+
-    def find_module(self, module_name, 
-
+
+
-        # Finds and returns the module scope corresponding to
-        # the given relative or absolute module name. If this
-        # is the first time the module has been requested, finds
@@ -216 +225 @@
-                scope = scope.find_submodule(name)
-        if debug_find_module:
-            print("...scope =", scope)
-
+ or locate_only
-            if debug_find_module:
-                print("...pxd not loaded")
-            scope.pxd_file_loaded = 1
-            if not pxd_pathname:
-                if debug_find_module:
-                    print("...looking for pxd file")
@@ -228 +236 @@
-                    print("......found ", pxd_pathname)
-                if not pxd_pathname and need_pxd:
-                    error(pos, "'%s.pxd' not found" % module_name)
+            if locate_only:
+                # find_pxd_file is rather weak, so we pass this option to do 
+                # a more extensive search. 
+                return pxd_pathname
+            scope.pxd_file_loaded = 1
-            if pxd_pathname:
-                try:
-                    if debug_find_module:
-                        print("Context.find_module: Parsing %s" % pxd_pathname)
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-                except CompileError:
-                    pass
-        return scope
@@ -490 +520 @@
-    return result
-
-def run_pipeline(source, options, full_module_name = None):
+
+    options.pxd_cache_file = "pxd_cache.pickle"
+    if options.pxd_cache_file:
+        if os.path.exists(options.pxd_cache_file):
+            pxd_cache = pickle.load(open(options.pxd_cache_file))
+            pxd_cache.clean()
+            pxd_cache.verify()
+        else:
+            pxd_cache = PxdCache()
+    else:
+        pxd_cache = None
+
-    # Set up context
-
+, pxd_cache
-
-    # Set up source object
-    cwd = os.getcwd()
@@ -508 +550 @@
-    context.setup_errors(options)
-    err, enddata = context.run_pipeline(pipeline, source)
-    context.teardown_errors(err, options, result)
+    
+    if pxd_cache and pxd_cache.is_dirty():
+        pickle.dump(pxd_cache, open(options.pxd_cache_file, "w"), 2)
+    
-    return result
-
-#------------------------------------------------------------------------
@@ -675 +721 @@
-        return compile_single(source, options, full_module_name)
-    else:
-        return compile_multiple(source, options)
+        
+
+
+class PxdDependancy:
+
+    _file_digest_cache = {}
+
+    def __init__(self, path):
+        self.path = path
+        self.digest = self.file_digest(path)
+        self.dependants = []
+        self.valid = True
+
+    def file_digest(self, pathname):
+        digest = self._file_digest_cache.get(pathname)
+        if digest is None:
+            m = hashlib.md5()
+            m.update(open(pathname).read())
+            digest = self._file_digest_cache[pathname] = m.hexdigest()
+        return digest
+        
+    def verify(self):
+        if not self.valid:
+            return False
+        elif not os.path.exists(self.path):
+            return False
+        else:
+            return self.digest == self.file_digest(self.path)
+
+
+class PxdCache:
+    """
+    This class stores caches the parse results of .pxd files, verifying that
+    they are fresh using md5 hashes. 
+    """
+    _file_digest_cache = {}
+    
+    def __init__(self):
+        self.dirty = False
+        self.pxds = {}
+        self.deps = {}
+        
+    def clean(self):
+        self.dirty = False
+        
+    def is_dirty(self):
+        return self.dirty
+        
+    def __getitem__(self, module_filepath):
+        return self.pxds.get(module_filepath)
+        
+    def cache(self, path, (pxd_codenodes, pxd_scope)):
+        self.dirty = True
+        pxd_scope.filepath = path
+        self.pxds[path] = pxd_codenodes, pxd_scope
+        dependancies = [path] + pxd_scope.included_files + pxd_scope.cimported_module_filenames
+        for dep in dependancies:
+            data = self.deps.get(dep)
+            if data is None:
+                data = self.deps[dep] = PxdDependancy(dep)
+            data.dependants.append(path)
+        
+    def verify(self):
+        PxdDependancy._file_digest_cache = {}
+        bad = []
+        for dep, dep_data in self.deps.items():
+            if not dep_data.verify():
+                bad.append((dep, dep_data))
+        if bad:
+            self.dirty = True
+        while bad:
+            dep, dep_data = bad.pop()
+            del self.deps[dep]
+            if self.pxds.has_key(dep):
+                del self.pxds[dep]
+            for depend in dep_data.dependants:
+                depend_data = self.deps.get(depend)
+                if depend_data is not None and depend_data.valid:
+                    depend_data.valid = False
+                    bad.append((depend, depend_data))
+
+
-
-#------------------------------------------------------------------------
-#

a/Cython/Compiler/ModuleNode.py

@@ -64 +64 @@
-        if self.has_imported_c_functions():
-            self.module_temp_cname = env.allocate_temp_pyobject()
-            env.release_temp(self.module_temp_cname)
-
+ or 1
-            self.generate_dep_file(env, result)
-        self.generate_c_code(env, options, result)
-        self.generate_h_code(env, options, result)

a/Cython/Compiler/PyrexTypes.py

@@ -7 +7 @@
-import copy
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    #
-    #  Base class for all Pyrex types including pseudo-types.
+    
+    def __new__(cls, *args, **kwds):
+        if issubclass(cls, (PyExtensionType, BufferType)):
+            return super(BaseType, cls).__new__(cls, *args, **kwds)
+        else:
+            if len(args) == 1 and isinstance(args[0], BaseTypeKey):
+                key = args[0]
+                args, kwds = key.data[1], dict(key.data[2])
+            else:
+                key = BaseTypeKey(cls, args, kwds)
+            t = _type_cache.get(key)
+            if t is None:
+                t = _type_cache[key] = super(BaseType, cls).__new__(cls, *args, **kwds)
+                t._key = key
+            return t
+        
+    def __getnewargs__(self):
+        if isinstance(self, (PyExtensionType, BufferType)):
+            return ()
+        else:
+            return (self._key,)
-
-    def cast_code(self, expr_code):
-        return "((%s)%s)" % (self.declaration_code(""), expr_code)
@@ -116 +163 @@
-        return self.same_as_resolved_type(other_type.resolve(), **kwds)
-    
-    def same_as_resolved_type(self, other_type):
+        if str(self) == str(other_type) and self != other_type:
+            print self, id(self), type(self)
+            print other_type, id(other_type), type(other_type)   
+            import cPickle as pickle
+            print id(pickle.loads(pickle.dumps(self, 2)))
+            print id(pickle.loads(pickle.dumps(other_type, 2)))
+            print self.has_attributes
+#            print self._key, other_type._key
-        return self == other_type or other_type is error_type
-    
-    def subtype_of(self, other_type):
@@ -156 +211 @@
-    def __init__(self, cname, base_type):
-        self.typedef_cname = cname
-        self.typedef_base_type = base_type
+        
+    def __setstate__(self, state):
+        self.__dict__ = state
-    
-    def resolve(self):
-        return self.typedef_base_type.resolve()
@@ -207 +265 @@
-
-    is_buffer = 1
-    writable = True
+    
-    def __init__(self, base, dtype, ndim, mode, negative_indices, cast):
-        self.base = base
-        self.dtype = dtype
@@ -215 +274 @@
-        self.mode = mode
-        self.negative_indices = negative_indices
-        self.cast = cast
+    
+    def __setstate__(self, state):
+        self.__dict__ = state
-    
-    def as_argument_type(self):
-        return self
@@ -238 +300 @@
-    pymemberdef_typecode = "T_OBJECT"
-    buffer_defaults = None
-    typestring = "O"
-
+
-    def __str__(self):
-        return "Python object"
-    

a/Cython/Includes/python_version.pxd

@@ -28 +28 @@
-    #    0xC (release candidate)
-    #    0xF (final)
-
-[]
-[]
+*
+*