diff --git a/dist/tools/compile_test/compile_test.py b/dist/tools/compile_test/compile_test.py
index 97f29f32fa..2f76773743 100755
--- a/dist/tools/compile_test/compile_test.py
+++ b/dist/tools/compile_test/compile_test.py
@@ -21,49 +21,59 @@ from __future__ import print_function
 
 from itertools import groupby
 from os import devnull, environ, listdir
-from os.path import abspath, dirname, isdir, join
-from subprocess import PIPE, Popen
+from os.path import abspath, dirname, isfile, join
+from subprocess import CalledProcessError, check_call, PIPE, Popen
 from sys import exit, stdout
 
 riotbase = environ.get('RIOTBASE') or abspath(join(dirname(abspath(__file__)), '../' * 3))
 
-null = open(devnull, 'w')
+null = open(devnull, 'w', 0)
 
 success = []
 failed = []
 exceptions = []
 
-for folder in ('examples', 'tests'):
-    print('Building all applications in: \033[1;34m{}\033[0m'.format(folder))
-    for application in sorted(listdir(join(riotbase, folder))):
-        if not isdir(join(riotbase, folder, application)):
+def is_tracked(application_folder):
+    if not isfile(join(application_folder, 'Makefile')):
+        return False
+    try:
+        check_call(('git', 'ls-files', '--error-unmatch', 'Makefile'),
+                   stdin=null, stdout=null, stderr=null, cwd=application_folder)
+    except CalledProcessError:
+        return False
+    else:
+        return True
+
+def get_lines(readline, prefix):
+    while 1:
+        result = readline()
+        if not result:
+            break
+        elif not result.startswith(prefix):
             continue
 
-        stdout.write('\tBuilding application: \033[1;34m{}\033[0m '.format(application))
+        result = result[len(prefix):].rstrip().split(' .. ')[::-1]
+        if len(result) == 2:
+            stdout.write('.')
+            stdout.flush()
+            yield result
+
+for folder in ('examples', 'tests'):
+    print('Building all applications in: \033[1;34m{}\033[0m'.format(folder))
+
+    applications = listdir(join(riotbase, folder))
+    applications = filter(lambda app: is_tracked(join(riotbase, folder, app)), applications)
+    applications = sorted(applications)
+
+    for nth, application in enumerate(applications, 1):
+        stdout.write('\tBuilding application: \033[1;34m{}\033[0m ({}/{}) '.format(application, nth, len(applications)))
         stdout.flush()
         try:
             subprocess = Popen(('make', 'buildtest'),
-                               bufsize=1,
-                               stdin=null,
-                               stdout=PIPE,
-                               stderr=null,
+                               bufsize=1, stdin=null, stdout=PIPE, stderr=null,
                                cwd=join(riotbase, folder, application))
 
-            def lines(readline, prefix):
-                while 1:
-                    result = readline()
-                    if not result:
-                        break
-                    elif not result.startswith(prefix):
-                        continue
-
-                    result = result[len(prefix):].rstrip().split(' .. ')[::-1]
-                    if len(result) == 2:
-                        stdout.write('.')
-                        stdout.flush()
-                        yield result
-
-            lines = lines(subprocess.stdout.readline, 'Building for ')
+            lines = get_lines(subprocess.stdout.readline, 'Building for ')
             lines = groupby(sorted(lines), lambda (outcome, board): outcome)
             for group, results in lines:
                 print('\n\t\t{}: {}'.format(group, ', '.join(sorted(board for outcome, board in results))))
@@ -80,7 +90,8 @@ for folder in ('examples', 'tests'):
                 pass
 
 print('Outcome:')
-for color, group, applications in (('2', 'success', success), ('1', 'failed', failed), ('4', 'exceptions', exceptions)):
+for color, group in (('2', 'success'), ('1', 'failed'), ('4', 'exceptions')):
+    applications = locals()[group]
     if applications:
         print('\t\033[1;3{}m{}\033[0m: {}'.format(color, group, ', '.join(applications)))