#! /usr/bin/python2 import os.path import sys import shlex import re import tempfile import copy from headerutils import * requires = { } provides = { } no_remove = [ "system.h", "coretypes.h", "config.h" , "bconfig.h", "backend.h" ] # These targets are the ones which provide "coverage". Typically, if any # target is going to fail compilation, it's one of these. This was determined # during the initial runs of reduce-headers... On a full set of target builds, # every failure which occured was triggered by one of these. # This list is used during target-list construction simply to put any of these # *first* in the candidate list, increasing the probability that a failure is # found quickly. target_priority = [ "aarch64-linux-gnu", "arm-netbsdelf", "c6x-elf", "epiphany-elf", "hppa2.0-hpux10.1", "i686-mingw32crt", "i686-pc-msdosdjgpp", "mipsel-elf", "powerpc-eabisimaltivec", "rs6000-ibm-aix5.1.0", "sh-superh-elf", "sparc64-elf", "spu-elf" ] target_dir = "" build_dir = "" ignore_list = list() target_builds = list() target_dict = { } header_dict = { } search_path = [ ".", "../include", "../libcpp/include" ] remove_count = { } # Given a header name, normalize it. ie. cp/cp-tree.h could be in gcc, while # the same header could be referenced from within the cp subdirectory as # just cp-tree.h # for now, just assume basenames are unique def normalize_header (header): return os.path.basename (header) # Adds a header file and its sub-includes to the global dictionary if they # aren't already there. Specify s_path since different build directories may # append themselves on demand to the global list. # return entry for the specified header, knowing all sub entries are completed def get_header_info (header, s_path): global header_dict global empty_iinfo process_list = list () location = "" bname = "" bname_iinfo = empty_iinfo for path in s_path: if os.path.exists (path + "/" + header): location = path + "/" + header break if location: bname = normalize_header (location) if header_dict.get (bname): bname_iinfo = header_dict[bname] loc2 = ii_path (bname_iinfo)+ "/" + bname if loc2[:2] == "./": loc2 = loc2[2:] if location[:2] == "./": location = location[2:] if loc2 != location: # Don't use the cache if it isnt the right one. bname_iinfo = process_ii_macro (location) return bname_iinfo bname_iinfo = process_ii_macro (location) header_dict[bname] = bname_iinfo # now decend into the include tree for i in ii_include_list (bname_iinfo): get_header_info (i, s_path) else: # if the file isnt in the source directories, look in the build and target # directories. If it is here, then aggregate all the versions. location = build_dir + "/gcc/" + header build_inc = target_inc = False if os.path.exists (location): build_inc = True for x in target_dict: location = target_dict[x] + "/gcc/" + header if os.path.exists (location): target_inc = True break if (build_inc or target_inc): bname = normalize_header(header) defines = set() consumes = set() incl = set() if build_inc: iinfo = process_ii_macro (build_dir + "/gcc/" + header) defines = set (ii_macro_define (iinfo)) consumes = set (ii_macro_consume (iinfo)) incl = set (ii_include_list (iinfo)) if (target_inc): for x in target_dict: location = target_dict[x] + "/gcc/" + header if os.path.exists (location): iinfo = process_ii_macro (location) defines.update (ii_macro_define (iinfo)) consumes.update (ii_macro_consume (iinfo)) incl.update (ii_include_list (iinfo)) bname_iinfo = (header, "build", list(incl), list(), list(consumes), list(defines), list(), list()) header_dict[bname] = bname_iinfo for i in incl: get_header_info (i, s_path) return bname_iinfo # return a list of all headers brought in by this header def all_headers (fname): global header_dict headers_stack = list() headers_list = list() if header_dict.get (fname) == None: return list () for y in ii_include_list (header_dict[fname]): headers_stack.append (y) while headers_stack: h = headers_stack.pop () hn = normalize_header (h) if hn not in headers_list: headers_list.append (hn) if header_dict.get(hn): for y in ii_include_list (header_dict[hn]): if normalize_header (y) not in headers_list: headers_stack.append (y) return headers_list # Search bld_dir for all target tuples, confirm that they have a build path with # bld_dir/target-tuple/gcc, and build a dictionary of build paths indexed by # target tuple.. def build_target_dict (bld_dir, just_these): global target_dict target_doct = { } error = False if os.path.exists (bld_dir): if just_these: ls = just_these else: ls = os.listdir(bld_dir) for t in ls: if t.find("-") != -1: target = t.strip() tpath = bld_dir + "/" + target if not os.path.exists (tpath + "/gcc"): print "Error: gcc build directory for target " + t + " Does not exist: " + tpath + "/gcc" error = True else: target_dict[target] = tpath if error: target_dict = { } def get_obj_name (src_file): if src_file[-2:] == ".c": return src_file.replace (".c", ".o") elif src_file[-3:] == ".cc": return src_file.replace (".cc", ".o") return "" def target_obj_exists (target, obj_name): global target_dict # look in a subdir if src has a subdir, then check gcc base directory. if target_dict.get(target): obj = target_dict[target] + "/gcc/" + obj_name if not os.path.exists (obj): obj = target_dict[target] + "/gcc/" + os.path.basename(obj_name) if os.path.exists (obj): return True return False # Given a src file, return a list of targets which may build this file. def find_targets (src_file): global target_dict targ_list = list() obj_name = get_obj_name (src_file) if not obj_name: print "Error: " + src_file + " - Cannot determine object name." return list() # Put the high priority targets which tend to trigger failures first for target in target_priority: if target_obj_exists (target, obj_name): targ_list.append ((target, target_dict[target])) for target in target_dict: if target not in target_priority and target_obj_exists (target, obj_name): targ_list.append ((target, target_dict[target])) return targ_list def try_to_remove (src_file, h_list, verbose): global target_dict global header_dict global build_dir # build from scratch each time header_dict = { } summary = "" rmcount = 0 because = { } src_info = process_ii_macro_src (src_file) src_data = ii_src (src_info) if src_data: inclist = ii_include_list_non_cond (src_info) # work is done if there are no includes to check if not inclist: return src_file + ": No include files to attempt to remove" # work on the include list in reverse. inclist.reverse() # Get the target list targ_list = list() targ_list = find_targets (src_file) spath = search_path if os.path.dirname (src_file): spath.append (os.path.dirname (src_file)) hostbuild = True if src_file.find("config/") != -1: # config files dont usually build on the host hostbuild = False obn = get_obj_name (os.path.basename (src_file)) if obn and os.path.exists (build_dir + "/gcc/" + obn): hostbuild = True if not target_dict: summary = src_file + ": Target builds are required for config files. None found." print summary return summary if not targ_list: summary =src_file + ": Cannot find any targets which build this file." print summary return summary if hostbuild: # confirm it actually builds before we do anything print "Confirming source file builds" res = get_make_output (build_dir + "/gcc", "all") if res[0] != 0: message = "Error: " + src_file + " does not build currently." summary = src_file + " does not build on host." print message print res[1] if verbose: verbose.write (message + "\n") verbose.write (res[1]+ "\n") return summary src_requires = set (ii_macro_consume (src_info)) for macro in src_requires: because[macro] = src_file header_seen = list () os.rename (src_file, src_file + ".bak") src_orig = copy.deepcopy (src_data) src_tmp = copy.deepcopy (src_data) try: # process the includes from bottom to top. This is because we know that # later includes have are known to be needed, so any dependency from this # header is a true dependency for inc_file in inclist: inc_file_norm = normalize_header (inc_file) if inc_file in no_remove: continue if len (h_list) != 0 and inc_file_norm not in h_list: continue if inc_file_norm[0:3] == "gt-": continue if inc_file_norm[0:6] == "gtype-": continue if inc_file_norm.replace(".h",".c") == os.path.basename(src_file): continue lookfor = ii_src_line(src_info)[inc_file] src_tmp.remove (lookfor) message = "Trying " + src_file + " without " + inc_file print message if verbose: verbose.write (message + "\n") out = open(src_file, "w") for line in src_tmp: out.write (line) out.close() keep = False if hostbuild: res = get_make_output (build_dir + "/gcc", "all") else: res = (0, "") rc = res[0] message = "Passed Host build" if (rc != 0): # host build failed message = "Compilation failed:\n"; keep = True else: if targ_list: objfile = get_obj_name (src_file) t1 = targ_list[0] if objfile and os.path.exists(t1[1] +"/gcc/"+objfile): res = get_make_output_parallel (targ_list, objfile, 0) else: res = get_make_output_parallel (targ_list, "all-gcc", 0) rc = res[0] if rc != 0: message = "Compilation failed on TARGET : " + res[2] keep = True else: message = "Passed host and target builds" if keep: print message + "\n" if (rc != 0): if verbose: verbose.write (message + "\n"); verbose.write (res[1]) verbose.write ("\n"); if os.path.exists (inc_file): ilog = open(inc_file+".log","a") ilog.write (message + " for " + src_file + ":\n\n"); ilog.write ("============================================\n"); ilog.write (res[1]) ilog.write ("\n"); ilog.close() if os.path.exists (src_file): ilog = open(src_file+".log","a") ilog.write (message + " for " +inc_file + ":\n\n"); ilog.write ("============================================\n"); ilog.write (res[1]) ilog.write ("\n"); ilog.close() # Given a sequence where : # #include "tm.h" # #include "target.h" // includes tm.h # target.h was required, and when attempting to remove tm.h we'd see that # all the macro defintions are "required" since they all look like: # #ifndef HAVE_blah # #define HAVE_blah # endif # when target.h was found to be required, tm.h will be tagged as included. # so when we get this far, we know we dont have to check the macros for # tm.h since we know it is already been included. if inc_file_norm not in header_seen: iinfo = get_header_info (inc_file, spath) newlist = all_headers (inc_file_norm) if ii_path(iinfo) == "build" and not target_dict: keep = True text = message + " : Will not remove a build file without some targets." print text ilog = open(src_file+".log","a") ilog.write (text +"\n") ilog.write ("============================================\n"); ilog.close() ilog = open("reduce-headers-kept.log","a") ilog.write (src_file + " " + text +"\n") ilog.close() else: newlist = list() if not keep and inc_file_norm not in header_seen: # now look for any macro requirements. for h in newlist: if not h in header_seen: if header_dict.get(h): defined = ii_macro_define (header_dict[h]) for dep in defined: if dep in src_requires and dep not in ignore_list: keep = True; text = message + ", but must keep " + inc_file + " because it provides " + dep if because.get(dep) != None: text = text + " Possibly required by " + because[dep] print text ilog = open(inc_file+".log","a") ilog.write (because[dep]+": Requires [dep] in "+src_file+"\n") ilog.write ("============================================\n"); ilog.close() ilog = open(src_file+".log","a") ilog.write (text +"\n") ilog.write ("============================================\n"); ilog.close() ilog = open("reduce-headers-kept.log","a") ilog.write (src_file + " " + text +"\n") ilog.close() if verbose: verbose.write (text + "\n") if keep: # add all headers 'consumes' to src_requires list, and mark as seen for h in newlist: if not h in header_seen: header_seen.append (h) if header_dict.get(h): consume = ii_macro_consume (header_dict[h]) for dep in consume: if dep not in src_requires: src_requires.add (dep) if because.get(dep) == None: because[dep] = inc_file src_tmp = copy.deepcopy (src_data) else: print message + " --> removing " + inc_file + "\n" rmcount += 1 if verbose: verbose.write (message + " --> removing " + inc_file + "\n") if remove_count.get(inc_file) == None: remove_count[inc_file] = 1 else: remove_count[inc_file] += 1 src_data = copy.deepcopy (src_tmp) except: print "Interuption: restoring original file" out = open(src_file, "w") for line in src_orig: out.write (line) out.close() raise # copy current version, since it is the "right" one now. out = open(src_file, "w") for line in src_data: out.write (line) out.close() # Try a final host bootstrap build to make sure everything is kosher. if hostbuild: res = get_make_output (build_dir, "all") rc = res[0] if (rc != 0): # host build failed! return to original version print "Error: " + src_file + " Failed to bootstrap at end!!! restoring." print " Bad version at " + src_file + ".bad" os.rename (src_file, src_file + ".bad") out = open(src_file, "w") for line in src_orig: out.write (line) out.close() return src_file + ": failed to build after reduction. Restored original" if src_data == src_orig: summary = src_file + ": No change." else: summary = src_file + ": Reduction performed, "+str(rmcount)+" includes removed." print summary return summary only_h = list () ignore_cond = False usage = False src = list() only_targs = list () for x in sys.argv[1:]: if x[0:2] == "-b": build_dir = x[2:] elif x[0:2] == "-f": fn = normalize_header (x[2:]) if fn not in only_h: only_h.append (fn) elif x[0:2] == "-h": usage = True elif x[0:2] == "-d": ignore_cond = True elif x[0:2] == "-D": ignore_list.append(x[2:]) elif x[0:2] == "-T": only_targs.append(x[2:]) elif x[0:2] == "-t": target_dir = x[2:] elif x[0] == "-": print "Error: Unrecognized option " + x usgae = True else: if not os.path.exists (x): print "Error: specified file " + x + " does not exist." usage = True else: src.append (x) if target_dir: build_target_dict (target_dir, only_targs) if build_dir == "" and target_dir == "": print "Error: Must specify a build directory, and/or a target directory." usage = True if build_dir and not os.path.exists (build_dir): print "Error: specified build directory does not exist : " + build_dir usage = True if target_dir and not os.path.exists (target_dir): print "Error: specified target directory does not exist : " + target_dir usage = True if usage: print "Attempts to remove extraneous include files from source files." print " " print "Should be run from the main gcc source directory, and works on a target" print "directory, as we attempt to make the 'all' target." print " " print "By default, gcc-reorder-includes is run on each file before attempting" print "to remove includes. this removes duplicates and puts some headers in a" print "canonical ordering" print " " print "The build directory should be ready to compile via make. Time is saved" print "if the build is already complete, so that only changes need to be built." print " " print "Usage: [options] file1.c [file2.c] ... [filen.c]" print " -bdir : the root build directory to attempt buiding .o files." print " -tdir : the target build directory" print " -d : Ignore conditional macro dependencies." print " " print " -Dmacro : Ignore a specific macro for dependencies" print " -Ttarget : Only consider target in target directory." print " -fheader : Specifies a specific .h file to be considered." print " " print " -D, -T, and -f can be specified mulitple times and are aggregated." print " " print " The original file will be in filen.bak" print " " sys.exit (0) if only_h: print "Attempting to remove only these files:" for x in only_h: print x print " " logfile = open("reduce-headers.log","w") for x in src: msg = try_to_remove (x, only_h, logfile) ilog = open("reduce-headers.sum","a") ilog.write (msg + "\n") ilog.close() ilog = open("reduce-headers.sum","a") ilog.write ("===============================================================\n") for x in remove_count: msg = x + ": Removed " + str(remove_count[x]) + " times." print msg logfile.write (msg + "\n") ilog.write (msg + "\n")