waf: persist implicit dependency information across clean

Implicit dependency scanning takes significant time and, since it doesn't
produce files, it's okay to keep the resulting information across clean
commands as long as the scanner is triggered again if there's need to. This
commit accomplishes that.

The advantage of this approach can be observed by the following timings when
building the group "bin":

 Method                                                         Time
 ------------------------------------------------------------------------
 Fully clean build                                              5m18.633s
 Clean build with scanning result persisted                     4m23.346s
 Clean build with ccache but non-persistent scan results        1m40.125s
 Clean build with scanning results persisted and with ccache      14.843s

While at it, move management of information persisted across clean commands to
a separate module.

Tools/ardupilotwaf/ap_persistent.py

@@ -0,0 +1,59 @@
+# Copyright (C) 2016  Intel Corporation. All rights reserved.
+#
+# This file is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This file is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Module that changes Waf to keep persistent information across clean operations
+in for performance improvement.
+"""
+from waflib import Build, Task
+
+Build.SAVED_ATTRS.append('ap_persistent_task_sigs')
+Build.SAVED_ATTRS.append('ap_persistent_imp_sigs')
+Build.SAVED_ATTRS.append('ap_persistent_node_deps')
+
+_original_signature = Task.Task.signature
+
+_original_sig_implicit_deps = Task.Task.sig_implicit_deps
+if hasattr(_original_sig_implicit_deps, '__func__'):
+    _original_sig_implicit_deps = _original_sig_implicit_deps.__func__
+
+def _signature(self):
+    s = _original_signature(self)
+    real_fn = self.sig_implicit_deps.__func__
+    if not self.scan or _original_sig_implicit_deps != real_fn:
+        return s
+    bld = self.generator.bld
+    bld.ap_persistent_imp_sigs[self.uid()] = bld.imp_sigs[self.uid()]
+    bld.ap_persistent_node_deps[self.uid()] = bld.node_deps[self.uid()]
+    return s
+Task.Task.signature = _signature
+
+class CleanContext(Build.CleanContext):
+    def clean(self):
+        if not self.options.clean_all_sigs:
+            saved_task_sigs = dict(self.ap_persistent_task_sigs)
+            saved_imp_sigs = dict(self.ap_persistent_imp_sigs)
+            saved_node_deps = dict(self.ap_persistent_node_deps)
+
+        super(CleanContext, self).clean()
+
+        if not self.options.clean_all_sigs:
+            self.task_sigs.update(saved_task_sigs)
+            self.ap_persistent_task_sigs.update(saved_task_sigs)
+
+            self.imp_sigs.update(saved_imp_sigs)
+            self.ap_persistent_imp_sigs.update(saved_imp_sigs)
+
+            self.node_deps.update(saved_node_deps)
+            self.ap_persistent_node_deps.update(saved_node_deps)

Tools/ardupilotwaf/ardupilotwaf.py

@@ -8,6 +8,8 @@ from waflib.TaskGen import before_method, feature
 import os.path, os
 from collections import OrderedDict
 
+import ap_persistent
+
 SOURCE_EXTS = [
     '*.S',
     '*.c',
@@ -382,25 +384,13 @@ my board".
     g.add_option('--clean-all-sigs',
         action='store_true',
         help='''
-Clean signatures for all tasks. By default, some tasks that don't produce files
-and are time consuming keep their signatures when clean is called without this
-parameter. One example is the group of tasks that check headers included by
-Ardupilot libraries.
+Clean signatures for all tasks. By default, tasks that scan for implicit
+dependencies (like the compilation tasks) keep the dependency information
+across clean commands, so that that information is changed only when really
+necessary. Also, some tasks that don't really produce files persist their
+signature. This option avoids that behavior when cleaning the build.
 ''')
 
 def build(bld):
     bld.add_pre_fun(_process_build_command)
     bld.add_pre_fun(_select_programs_from_group)
-
-class CleanContext(Build.CleanContext):
-    Build.SAVED_ATTRS.append('ap_persistent_task_sigs')
-
-    def clean(self):
-        if not self.options.clean_all_sigs:
-            saved_sigs = dict(self.ap_persistent_task_sigs)
-
-        super(CleanContext, self).clean()
-
-        if not self.options.clean_all_sigs:
-            self.task_sigs.update(saved_sigs)
-            self.ap_persistent_task_sigs.update(saved_sigs)