gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree into a specified directory.
   4 #
   5
   6 import argparse, sys, os, errno, shutil, re, subprocess
   7 import tarfile, gzip
   8
   9 # find self
  10 source_dir = os.path.abspath(os.path.dirname(__file__))
  11 sys.path.append(source_dir)
  12 # and import libraries we have
  13 from lib import kconfig, patch, make
  14 from lib import bpgit as git
  15 from lib import bpgpg as gpg
  16 from lib import bpkup as kup
  17 from lib.tempdir import tempdir
  18 from lib import bpreqs as reqs
  19 from lib import bpversion as gen_version
  20
  21 class Bp_Identity(object):
  22     """
  23     folks considering multiple integrations may want to
  24     consider stuffing versioning info here as well but
  25     that will need thought/design on sharing compat and
  26     module namespaces.
  27
  28     Use the *_resafe when combining on regexps, although we currently
  29     don't support regexps there perhaps later we will and this will
  30     just make things safer for the output regardless. Once and if those
  31     are added, how we actually use the others for regular printing will
  32     need to be considered.
  33     """
  34     def __init__(self, integrate=False, kconfig_prefix='CPTCFG_',
  35                  project_prefix='', project_dir='',
  36                  target_dir='', target_dir_name='',
  37                  kconfig_source_var=None):
  38         self.integrate = integrate
  39         self.kconfig_prefix = kconfig_prefix
  40         self.kconfig_prefix_resafe = re.escape(kconfig_prefix)
  41         self.project_prefix = project_prefix
  42         self.project_prefix_resafe = re.escape(project_prefix)
  43         self.full_prefix = kconfig_prefix + project_prefix
  44         self.full_prefix_resafe = re.escape(self.full_prefix)
  45         self.project_dir = project_dir
  46         self.target_dir = target_dir
  47         self.target_dir_name = target_dir_name
  48         self.kconfig_source_var = kconfig_source_var
  49         if self.kconfig_source_var:
  50             self.kconfig_source_var_resafe = re.escape(self.kconfig_source_var)
  51         else:
  52             self.kconfig_source_var_resafe = None
  53
  54 def read_copy_list(copyfile):
  55     """
  56     Read a copy-list file and return a list of (source, target)
  57     tuples. The source and target are usually the same, but in
  58     the copy-list file there may be a rename included.
  59     """
  60     ret = []
  61     for item in copyfile:
  62         # remove leading/trailing whitespace
  63         item = item.strip()
  64         # comments
  65         if not item or item[0] == '#':
  66             continue
  67         if item[0] == '/':
  68             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  69         if ' -> ' in item:
  70             srcitem, dstitem = item.split(' -> ')
  71             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  72                 raise Exception("Cannot copy file/dir to dir/file")
  73         else:
  74             srcitem = dstitem = item
  75         ret.append((srcitem, dstitem))
  76     return ret
  77
  78
  79 def read_dependencies(depfilename):
  80     """
  81     Read a (the) dependency file and return the list of
  82     dependencies as a dictionary, mapping a Kconfig symbol
  83     to a list of kernel version dependencies.
  84
  85     If a backported feature that an upstream backported driver
  86     depends on had kconfig limitations (ie, debugging feature not
  87     available) a built constaint restriction can be expressed
  88     by using a kconfig expression. The kconfig expressions can
  89     be specified by using the "kconfig: " prefix.
  90
  91     While reading ignore blank or commented lines.
  92     """
  93     ret = {}
  94     depfile = open(depfilename, 'r')
  95     for item in depfile:
  96         kconfig_exp = ""
  97         item = item.strip()
  98         if not item or item[0] == '#':
  99             continue
 100         if "kconfig:" in item:
 101             sym, kconfig_exp = item.split(" ", 1)
 102             if not sym in ret:
 103                 ret[sym] = [kconfig_exp, ]
 104             else:
 105                 ret[sym].append(kconfig_exp)
 106         else:
 107             sym, dep = item.split()
 108             if not sym in ret:
 109                 ret[sym] = [dep, ]
 110             else:
 111                 ret[sym].append(dep)
 112     return ret
 113
 114
 115 def check_output_dir(d, clean):
 116     """
 117     Check that the output directory doesn't exist or is empty,
 118     unless clean is True in which case it's nuked. This helps
 119     sanity check the output when generating a tree, so usually
 120     running with --clean isn't suggested.
 121     """
 122     if clean:
 123         shutil.rmtree(d, ignore_errors=True)
 124     try:
 125         os.rmdir(d)
 126     except OSError as e:
 127         if e.errno != errno.ENOENT:
 128             raise
 129
 130
 131 def copytree(src, dst, symlinks=False, ignore=None):
 132     """
 133     Copy a directory tree. This differs from shutil.copytree()
 134     in that it allows destination directories to already exist.
 135     """
 136     names = os.listdir(src)
 137     if ignore is not None:
 138         ignored_names = ignore(src, names)
 139     else:
 140         ignored_names = set()
 141
 142     if not os.path.isdir(dst):
 143         os.makedirs(dst)
 144     errors = []
 145     for name in names:
 146         if name in ignored_names:
 147             continue
 148         srcname = os.path.join(src, name)
 149         dstname = os.path.join(dst, name)
 150         try:
 151             if symlinks and os.path.islink(srcname):
 152                 linkto = os.readlink(srcname)
 153                 os.symlink(linkto, dstname)
 154             elif os.path.isdir(srcname):
 155                 copytree(srcname, dstname, symlinks, ignore)
 156             else:
 157                 shutil.copy2(srcname, dstname)
 158         except (IOError, os.error) as why:
 159             errors.append((srcname, dstname, str(why)))
 160         # catch the Error from the recursive copytree so that we can
 161         # continue with other files
 162         except shutil.Error as err:
 163             errors.extend(err.args[0])
 164     try:
 165         shutil.copystat(src, dst)
 166     except WindowsError:
 167         # can't copy file access times on Windows
 168         pass
 169     except OSError as why:
 170         errors.extend((src, dst, str(why)))
 171     if errors:
 172         raise shutil.Error(errors)
 173
 174
 175 def copy_files(srcpath, copy_list, outdir):
 176     """
 177     Copy the copy_list files and directories from the srcpath
 178     to the outdir. The copy_list contains source and target
 179     names.
 180
 181     For now, it also ignores any *~ editor backup files, though
 182     this should probably be generalized (maybe using .gitignore?)
 183     Similarly the code that only copies some files (*.c, *.h,
 184     *.awk, Kconfig, Makefile) to avoid any build remnants in the
 185     kernel if they should exist.
 186     """
 187     for srcitem, tgtitem in copy_list:
 188         if tgtitem == '':
 189             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 190         elif tgtitem[-1] == '/':
 191             def copy_ignore(dir, entries):
 192                 r = []
 193                 for i in entries:
 194                     if i[-2:] == '.o' or i[-1] == '~':
 195                         r.append(i)
 196                 return r
 197             copytree(os.path.join(srcpath, srcitem),
 198                      os.path.join(outdir, tgtitem),
 199                      ignore=copy_ignore)
 200         else:
 201             try:
 202                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 203             except OSError as e:
 204                 # ignore dirs we might have created just now
 205                 if e.errno != errno.EEXIST:
 206                     raise
 207             shutil.copy(os.path.join(srcpath, srcitem),
 208                         os.path.join(outdir, tgtitem))
 209
 210
 211 def copy_git_files(srcpath, copy_list, rev, outdir):
 212     """
 213     "Copy" files from a git repository. This really means listing them with
 214     ls-tree and then using git show to obtain all the blobs.
 215     """
 216     for srcitem, tgtitem in copy_list:
 217         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 218             assert t == 'blob'
 219             f = os.path.join(outdir, f.replace(srcitem, tgtitem))
 220             d = os.path.dirname(f)
 221             if not os.path.exists(d):
 222                 os.makedirs(d)
 223             outf = open(f, 'w')
 224             git.get_blob(h, outf, tree=srcpath)
 225             outf.close()
 226             os.chmod(f, int(m, 8))
 227
 228 def automatic_backport_mangle_c_file(name):
 229     return name.replace('/', '-')
 230
 231
 232 def add_automatic_backports(args):
 233     disable_list = []
 234     export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
 235     bpi = kconfig.get_backport_info(os.path.join(args.bpid.target_dir, 'compat', 'Kconfig'))
 236     configtree = kconfig.ConfigTree(os.path.join(args.bpid.target_dir, 'Kconfig'), args.bpid)
 237     ignore=['Kconfig.kernel', 'Kconfig.versions']
 238     configtree.verify_sources(ignore=ignore)
 239     git_debug_snapshot(args, "verify sources for automatic backports")
 240     all_selects = configtree.all_selects()
 241     for sym, vals in bpi.items():
 242         if sym.startswith('BPAUTO_BUILD_'):
 243             if not sym[13:] in all_selects:
 244                 disable_list.append(sym)
 245                 continue
 246         symtype, module_name, c_files, h_files = vals
 247
 248         # first copy files
 249         files = []
 250         for f in c_files:
 251             files.append((f, os.path.join('compat', automatic_backport_mangle_c_file(f))))
 252         for f in h_files:
 253             files.append((os.path.join('include', f),
 254                           os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
 255         if args.git_revision:
 256             copy_git_files(args.kerneldir, files, args.git_revision, args.bpid.target_dir)
 257         else:
 258             copy_files(args.kerneldir, files, args.bpid.target_dir)
 259
 260         # now add the Makefile line
 261         mf = open(os.path.join(args.bpid.target_dir, 'compat', 'Makefile'), 'a+')
 262         o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files]
 263         if symtype == 'tristate':
 264             if not module_name:
 265                 raise Exception('backporting a module requires a #module-name')
 266             for of in o_files:
 267                 mf.write('%s-objs += %s\n' % (module_name, of))
 268             mf.write('obj-$(%s%s) += %s.o\n' % (args.bpid.full_prefix, sym, module_name))
 269         elif symtype == 'bool':
 270             mf.write('compat-$(%s%s) += %s\n' % (args.bpid.full_prefix, sym, ' '.join(o_files)))
 271
 272         # finally create the include file
 273         syms = []
 274         for f in c_files:
 275             for l in open(os.path.join(args.bpid.target_dir, 'compat',
 276                                        automatic_backport_mangle_c_file(f)), 'r'):
 277                 m = export.match(l)
 278                 if m:
 279                     syms.append(m.group('sym'))
 280         for f in h_files:
 281             outf = open(os.path.join(args.bpid.target_dir, 'include', f), 'w')
 282             outf.write('/* Automatically created during backport process */\n')
 283             outf.write('#ifndef %s%s\n' % (args.bpid.full_prefix, sym))
 284             outf.write('#include_next <%s>\n' % f)
 285             outf.write('#else\n');
 286             for s in syms:
 287                 outf.write('#undef %s\n' % s)
 288                 outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
 289             outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
 290             outf.write('#endif /* %s%s */\n' % (args.bpid.full_prefix, sym))
 291     return disable_list
 292
 293 def git_debug_init(args):
 294     """
 295     Initialize a git repository in the output directory and commit the current
 296     code in it. This is only used for debugging the transformations this code
 297     will do to the output later.
 298     """
 299     if not args.gitdebug:
 300         return
 301     # Git supports re-initialization, although not well documented it can
 302     # reset config stuff, lets avoid that if the tree already exists.
 303     if not os.path.exists(os.path.join(args.bpid.project_dir, '.git')):
 304         git.init(tree=args.bpid.project_dir)
 305     git.commit_all("Copied backport", tree=args.bpid.project_dir)
 306
 307
 308 def git_debug_snapshot(args, name):
 309     """
 310     Take a git snapshot for the debugging.
 311     """
 312     if not args.gitdebug:
 313         return
 314     git.commit_all(name, tree=args.bpid.project_dir)
 315
 316 def get_rel_spec_stable(rel):
 317     """
 318     Returns release specs for a linux-stable backports based release.
 319     """
 320     if ("rc" in rel):
 321         m = re.match(r"(?P<VERSION>\d+)\.+" \
 322                      "(?P<PATCHLEVEL>\d+)[.]*" \
 323                      "(?P<SUBLEVEL>\d*)" \
 324                      "[-rc]+(?P<RC_VERSION>\d+)\-+" \
 325                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 326                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 327                      rel)
 328     else:
 329         m = re.match(r"(?P<VERSION>\d+)\.+" \
 330                      "(?P<PATCHLEVEL>\d+)[.]*" \
 331                      "(?P<SUBLEVEL>\d*)\-+" \
 332                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 333                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 334                      rel)
 335     if (not m):
 336         return m
 337     return m.groupdict()
 338
 339 def get_rel_spec_next(rel):
 340     """
 341     Returns release specs for a linux-next backports based release.
 342     """
 343     m = re.match(r"(?P<DATE_VERSION>\d+)[-]*" \
 344                  "(?P<RELMOD_UPDATE>\d*)[-]*" \
 345                  "(?P<RELMOD_TYPE>[usnpc]*)", \
 346                  rel)
 347     if (not m):
 348         return m
 349     return m.groupdict()
 350
 351 def get_rel_prep(rel):
 352     """
 353     Returns a dict with prep work details we need prior to
 354     uploading a backports release to kernel.org
 355     """
 356     rel_specs = get_rel_spec_stable(rel)
 357     is_stable = True
 358     rel_tag = ""
 359     paths = list()
 360     if (not rel_specs):
 361         rel_specs = get_rel_spec_next(rel)
 362         if (not rel_specs):
 363             sys.stdout.write("rel: %s\n" % rel)
 364             return None
 365         if (rel_specs['RELMOD_UPDATE'] == '0' or
 366             rel_specs['RELMOD_UPDATE'] == '1'):
 367             return None
 368         is_stable = False
 369         date = rel_specs['DATE_VERSION']
 370         year = date[0:4]
 371         if (len(year) != 4):
 372             return None
 373         month = date[4:6]
 374         if (len(month) != 2):
 375             return None
 376         day = date[6:8]
 377         if (len(day) != 2):
 378             return None
 379         paths.append(year)
 380         paths.append(month)
 381         paths.append(day)
 382         rel_tag = "backports-" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 383     else:
 384         ignore = "-"
 385         if (not rel_specs['RELMOD_UPDATE']):
 386             return None
 387         if (rel_specs['RELMOD_UPDATE'] == '0'):
 388             return None
 389         ignore += rel_specs['RELMOD_UPDATE']
 390         if (rel_specs['RELMOD_TYPE'] != ''):
 391             ignore += rel_specs['RELMOD_TYPE']
 392         base_rel = rel.replace(ignore, "")
 393         paths.append("v" + base_rel)
 394         rel_tag = "v" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 395
 396     rel_prep = dict(stable = is_stable,
 397                     expected_tag = rel_tag,
 398                     paths_to_create = paths)
 399     return rel_prep
 400
 401 def create_tar_and_gz(tar_name, dir_to_tar):
 402     """
 403     We need both a tar file and gzip for kernel.org, the tar file
 404     gets signed, then we upload the compressed version, kup-server
 405     in the backend decompresses and verifies the tarball against
 406     our signature.
 407     """
 408     basename = os.path.basename(dir_to_tar)
 409     tar = tarfile.open(tar_name, "w")
 410     tar.add(dir_to_tar, basename)
 411     tar.close()
 412
 413     tar_file = open(tar_name, "r")
 414
 415     gz_file = gzip.GzipFile(tar_name + ".gz", 'wb')
 416     gz_file.write(tar_file.read())
 417     gz_file.close()
 418
 419 def upload_release(args, rel_prep, logwrite=lambda x:None):
 420     """
 421     Given a path of a relase make tarball out of it, PGP sign it, and
 422     then upload it to kernel.org using kup.
 423
 424     The linux-next based release do not require a RELMOD_UPDATE
 425     given that typically only one release is made per day. Using
 426     RELMOD_UPDATE for these releases is allowed though and if
 427     present it must be > 1.
 428
 429     The linux-stable based releases require a RELMOD_UPDATE.
 430
 431     RELMOD_UPDATE must be numeric and > 0 just as the RC releases
 432     of the Linux kernel.
 433
 434     The tree must also be tagged with the respective release, without
 435     the RELMOD_TYPE. For linux-next based releases this consists of
 436     backports- followed by DATE_VERSION and if RELMOD_TYPE is present.
 437     For linux-stable releases this consists of v followed by the
 438     full release version except the RELMOD_TYPE.
 439
 440     Uploads will not be allowed if these rules are not followed.
 441     """
 442     korg_path = "/pub/linux/kernel/projects/backports"
 443
 444     if (rel_prep['stable']):
 445         korg_path += "/stable"
 446
 447     parent = os.path.dirname(args.bpid.project_dir)
 448     release = os.path.basename(args.bpid.project_dir)
 449     tar_name = parent + '/' + release + ".tar"
 450     gzip_name = tar_name + ".gz"
 451
 452     create_tar_and_gz(tar_name, args.bpid.project_dir)
 453
 454     logwrite(gpg.sign(tar_name, extra_args=['--armor', '--detach-sign']))
 455
 456     logwrite("------------------------------------------------------")
 457
 458     if (not args.kup_test):
 459         logwrite("About to upload, current target path contents:")
 460     else:
 461         logwrite("kup-test: current target path contents:")
 462
 463     logwrite(kup.ls(path=korg_path))
 464
 465     for path in rel_prep['paths_to_create']:
 466         korg_path += '/' + path
 467         if (not args.kup_test):
 468             logwrite("create directory: %s" % korg_path)
 469             logwrite(kup.mkdir(korg_path))
 470     korg_path += '/'
 471     if (not args.kup_test):
 472         logwrite("upload file %s to %s" % (gzip_name, korg_path))
 473         logwrite(kup.put(gzip_name, tar_name + '.asc', korg_path))
 474         logwrite("\nFinished upload!\n")
 475         logwrite("Target path contents:")
 476         logwrite(kup.ls(path=korg_path))
 477     else:
 478         kup_cmd = "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (gzip_name, tar_name + '.asc', korg_path)
 479         logwrite("kup-test: skipping cmd: %s" % kup_cmd)
 480
 481 def apply_patches(args, desc, source_dir, patch_src, target_dir, logwrite=lambda x:None):
 482     """
 483     Given a path of a directories of patches and SmPL patches apply
 484     them on the target directory. If requested refresh patches, or test
 485     a specific SmPL patch.
 486     """
 487     logwrite('Applying patches from %s to %s ...' % (patch_src, target_dir))
 488     test_cocci = args.test_cocci or args.profile_cocci
 489     test_cocci_found = False
 490     patches = []
 491     sempatches = []
 492     for root, dirs, files in os.walk(os.path.join(source_dir, patch_src)):
 493         for f in files:
 494             if not test_cocci and f.endswith('.patch'):
 495                 patches.append(os.path.join(root, f))
 496             if f.endswith('.cocci'):
 497                 if test_cocci:
 498                     if f not in test_cocci:
 499                         continue
 500                     test_cocci_found = True
 501                     if args.test_cocci:
 502                         logwrite("Testing Coccinelle SmPL patch: %s" % test_cocci)
 503                     elif args.profile_cocci:
 504                         logwrite("Profiling Coccinelle SmPL patch: %s" % test_cocci)
 505                 sempatches.append(os.path.join(root, f))
 506     patches.sort()
 507     prefix_len = len(os.path.join(source_dir, patch_src)) + 1
 508     for pfile in patches:
 509         print_name = pfile[prefix_len:]
 510         # read the patch file
 511         p = patch.fromfile(pfile)
 512         # complain if it's not a patch
 513         if not p:
 514             raise Exception('No patch content found in %s' % print_name)
 515         # leading / seems to be stripped?
 516         if 'dev/null' in p.items[0].source:
 517             raise Exception('Patches creating files are not supported (in %s)' % print_name)
 518         # check if the first file the patch touches exists, if so
 519         # assume the patch needs to be applied -- otherwise continue
 520         patched_file = '/'.join(p.items[0].source.split('/')[1:])
 521         fullfn = os.path.join(target_dir, patched_file)
 522         if not os.path.exists(fullfn):
 523             if args.verbose:
 524                 logwrite("Not applying %s, not needed" % print_name)
 525             continue
 526         if args.verbose:
 527             logwrite("Applying patch %s" % print_name)
 528
 529         if args.refresh:
 530             # but for refresh, of course look at all files the patch touches
 531             for patchitem in p.items:
 532                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 533                 fullfn = os.path.join(target_dir, patched_file)
 534                 shutil.copyfile(fullfn, fullfn + '.orig_file')
 535
 536         process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 537                                    stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 538                                    close_fds=True, universal_newlines=True,
 539                                    cwd=target_dir)
 540         output = process.communicate(input=open(pfile, 'r').read())[0]
 541         output = output.split('\n')
 542         if output[-1] == '':
 543             output = output[:-1]
 544         if args.verbose:
 545             for line in output:
 546                 logwrite('> %s' % line)
 547         if process.returncode != 0:
 548             if not args.verbose:
 549                 logwrite("Failed to apply changes from %s" % print_name)
 550                 for line in output:
 551                     logwrite('> %s' % line)
 552             return 2
 553
 554         if args.refresh:
 555             pfilef = open(pfile + '.tmp', 'a')
 556             pfilef.write(p.top_header)
 557             pfilef.flush()
 558             for patchitem in p.items:
 559                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 560                 fullfn = os.path.join(target_dir, patched_file)
 561                 process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
 562                                             '--label', 'a/' + patched_file,
 563                                             '--label', 'b/' + patched_file],
 564                                            stdout=pfilef, close_fds=True,
 565                                            universal_newlines=True, cwd=target_dir)
 566                 process.wait()
 567                 os.unlink(fullfn + '.orig_file')
 568                 if not process.returncode in (0, 1):
 569                     logwrite("Failed to diff to refresh %s" % print_name)
 570                     pfilef.close()
 571                     os.unlink(pfile + '.tmp')
 572                     return 2
 573             pfilef.close()
 574             os.rename(pfile + '.tmp', pfile)
 575
 576         # remove orig/rej files that patch sometimes creates
 577         for root, dirs, files in os.walk(target_dir):
 578             for f in files:
 579                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 580                     os.unlink(os.path.join(root, f))
 581         git_debug_snapshot(args, "apply %s patch %s" % (desc, print_name))
 582
 583     sempatches.sort()
 584     prefix_len = len(os.path.join(source_dir, patch_src)) + 1
 585
 586     for cocci_file in sempatches:
 587         # Until Coccinelle picks this up
 588         pycocci = os.path.join(source_dir, 'devel/pycocci')
 589         cmd = [pycocci, cocci_file]
 590         extra_spatch_args = []
 591         if args.profile_cocci:
 592             cmd.append('--profile-cocci')
 593         cmd.append(os.path.abspath(target_dir))
 594         print_name = cocci_file[prefix_len:]
 595         if args.verbose:
 596             logwrite("Applying SmPL patch %s" % print_name)
 597             logwrite(" %s" % cmd)
 598         sprocess = subprocess.Popen(cmd,
 599                                     stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
 600                                     close_fds=True, universal_newlines=True,
 601                                     cwd=target_dir)
 602         output = sprocess.communicate()[0]
 603         sprocess.wait()
 604         if sprocess.returncode != 0:
 605             logwrite("Failed to process SmPL patch %s with %i" % (print_name, sprocess.returncode))
 606             return 2
 607         output = output.split('\n')
 608         if output[-1] == '':
 609             output = output[:-1]
 610         if args.verbose:
 611             for line in output:
 612                 logwrite('> %s' % line)
 613
 614         # remove cocci_backup files
 615         for root, dirs, files in os.walk(target_dir):
 616             for f in files:
 617                 if f.endswith('.cocci_backup'):
 618                     os.unlink(os.path.join(root, f))
 619         git_debug_snapshot(args, "apply %s SmPL patch %s" % (desc, print_name))
 620
 621     if test_cocci and test_cocci_found:
 622         logwrite('Done!')
 623         sys.exit(0)
 624
 625 def _main():
 626     # Our binary requirements go here
 627     req = reqs.Req()
 628     req.require('git')
 629     req.coccinelle('1.0.6')
 630     if not req.reqs_match():
 631         sys.exit(1)
 632
 633     # set up and parse arguments
 634     parser = argparse.ArgumentParser(description='generate backport tree')
 635     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 636                         help='Kernel tree to copy drivers from')
 637     parser.add_argument('outdir', metavar='<output directory>', type=str,
 638                         help='Directory to write the generated tree to')
 639     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 640                         default='copy-list',
 641                         help='File containing list of files/directories to copy, default "copy-list"')
 642     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 643                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 644                              'If this is specified, the kernel tree is used as git object storage ' +
 645                              'and we use git ls-tree to get the files.')
 646     parser.add_argument('--clean', const=True, default=False, action="store_const",
 647                         help='Clean output directory instead of erroring if it isn\'t empty')
 648     parser.add_argument('--integrate', const=True, default=False, action="store_const",
 649                         help='Integrate a future backported kernel solution into ' +
 650                              'an older kernel tree source directory.')
 651     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 652                         help='Refresh patches as they are applied, the source dir will be modified!')
 653     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 654                         help='name of base tree, default just "Linux"')
 655     parser.add_argument('--gitdebug', '--git-debug', const=True, default=False, action="store_const",
 656                         help='Use git, in the output tree, to debug the various transformation steps ' +
 657                              'that the tree generation makes (apply patches, ...)')
 658     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 659                         help='Print more verbose information')
 660     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 661                         action='append', default=[], help='Extra driver directory/copy-list.')
 662     parser.add_argument('--kup', const=True, default=False, action="store_const",
 663                         help='For maintainers: upload a release to kernel.org')
 664     parser.add_argument('--kup-test', const=True, default=False, action="store_const",
 665                         help='For maintainers: do all the work as if you were about to ' +
 666                              'upload to kernel.org but do not do the final `kup put` ' +
 667                              'and also do not run any `kup mkdir` commands. This will ' +
 668                              'however run `kup ls` on the target paths so ' +
 669                              'at the very least we test your kup configuration. ' +
 670                              'If this is your first time uploading use this first!')
 671     parser.add_argument('--test-cocci', metavar='<sp_file>', type=str, default=None,
 672                         help='Only use the cocci file passed for Coccinelle, don\'t do anything else, ' +
 673                              'also creates a git repo on the target directory for easy inspection ' +
 674                              'of changes done by Coccinelle.')
 675     parser.add_argument('--profile-cocci', metavar='<sp_file>', type=str, default=None,
 676                         help='Only use the cocci file passed and pass --profile  to Coccinelle, ' +
 677                              'also creates a git repo on the target directory for easy inspection ' +
 678                              'of changes done by Coccinelle.')
 679     args = parser.parse_args()
 680
 681     # When building a package we use CPTCFG as we can rely on the
 682     # fact that kconfig treats CONFIG_ as an environment variable
 683     # requring less changes on code. For kernel integration we use
 684     # the longer CONFIG_BACKPORT given that we'll be sticking to
 685     # the kernel symbol namespace, to address that we do a final
 686     # search / replace. Technically its possible to rely on the
 687     # same prefix for packaging as with kernel integration but
 688     # there are already some users of the CPTCFG prefix.
 689     bpid = None
 690     if args.integrate:
 691         bpid = Bp_Identity(integrate = args.integrate,
 692                            kconfig_prefix = 'CONFIG_',
 693                            project_prefix = 'BACKPORT_',
 694                            project_dir = args.outdir,
 695                            target_dir = os.path.join(args.outdir, 'backports/'),
 696                            target_dir_name = 'backports/',
 697                            kconfig_source_var = '$BACKPORT_DIR',
 698                            )
 699     else:
 700         bpid = Bp_Identity(integrate = args.integrate,
 701                            kconfig_prefix = 'CPTCFG_',
 702                            project_prefix = '',
 703                            project_dir = args.outdir,
 704                            target_dir = args.outdir,
 705                            target_dir_name = '',
 706                            kconfig_source_var = '$BACKPORT_DIR',
 707                            )
 708
 709     def logwrite(msg):
 710         sys.stdout.write(msg)
 711         sys.stdout.write('\n')
 712         sys.stdout.flush()
 713
 714     return process(args.kerneldir, args.copy_list,
 715                    git_revision=args.git_revision,
 716                    bpid=bpid,
 717                    clean=args.clean,
 718                    refresh=args.refresh, base_name=args.base_name,
 719                    gitdebug=args.gitdebug, verbose=args.verbose,
 720                    extra_driver=args.extra_driver,
 721                    kup=args.kup,
 722                    kup_test=args.kup_test,
 723                    test_cocci=args.test_cocci,
 724                    profile_cocci=args.profile_cocci,
 725                    logwrite=logwrite)
 726
 727 def process(kerneldir, copy_list_file, git_revision=None,
 728             bpid=None,
 729             clean=False, refresh=False, base_name="Linux", gitdebug=False,
 730             verbose=False, extra_driver=[], kup=False,
 731             kup_test=False,
 732             test_cocci=None,
 733             profile_cocci=None,
 734             logwrite=lambda x:None,
 735             git_tracked_version=False):
 736     class Args(object):
 737         def __init__(self, kerneldir, copy_list_file,
 738                      git_revision, bpid, clean, refresh, base_name,
 739                      gitdebug, verbose, extra_driver, kup,
 740                      kup_test,
 741                      test_cocci,
 742                      profile_cocci):
 743             self.kerneldir = kerneldir
 744             self.copy_list = copy_list_file
 745             self.git_revision = git_revision
 746             self.bpid = bpid
 747             self.clean = clean
 748             self.refresh = refresh
 749             self.base_name = base_name
 750             self.gitdebug = gitdebug
 751             self.verbose = verbose
 752             self.extra_driver = extra_driver
 753             self.kup = kup
 754             self.kup_test = kup_test
 755             self.test_cocci = test_cocci
 756             self.profile_cocci = profile_cocci
 757             if self.test_cocci or self.profile_cocci:
 758                 self.gitdebug = True
 759     def git_paranoia(tree=None, logwrite=lambda x:None):
 760         data = git.paranoia(tree)
 761         if (data['r'] != 0):
 762             logwrite('Cannot use %s' % tree)
 763             logwrite('%s' % data['output'])
 764             sys.exit(data['r'])
 765         else:
 766             logwrite('Validated tree: %s' % tree)
 767
 768     args = Args(kerneldir, copy_list_file,
 769                 git_revision, bpid, clean, refresh, base_name,
 770                 gitdebug, verbose, extra_driver, kup, kup_test,
 771                 test_cocci, profile_cocci)
 772     rel_prep = None
 773
 774     if bpid.integrate:
 775         if args.kup_test or args.test_cocci or args.profile_cocci or args.refresh:
 776             logwrite('Cannot use integration with:\n\tkup_test\n\ttest_cocci\n\tprofile_cocci\n\trefresh\n');
 777             sys.exit(1)
 778
 779     # start processing ...
 780     if (args.kup or args.kup_test):
 781         git_paranoia(source_dir, logwrite)
 782         git_paranoia(kerneldir, logwrite)
 783
 784         rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
 785         release = os.path.basename(bpid.target_dir)
 786         version = release.replace("backports-", "")
 787
 788         rel_prep = get_rel_prep(version)
 789         if (not rel_prep):
 790             logwrite('Invalid backports release name: %s' % release)
 791             logwrite('For rules on the release name see upload_release()')
 792             sys.exit(1)
 793         rel_type = "linux-stable"
 794         if (not rel_prep['stable']):
 795             rel_type = "linux-next"
 796         if (rel_prep['expected_tag'] != rel_describe):
 797             logwrite('Unexpected %s based backports release tag on' % rel_type)
 798             logwrite('the backports tree tree: %s\n' % rel_describe)
 799             logwrite('You asked to make a release with this ')
 800             logwrite('directory name: %s' % release)
 801             logwrite('The actual expected tag we should find on')
 802             logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag'])
 803             logwrite('For rules on the release name see upload_release()')
 804             sys.exit(1)
 805
 806     copy_list = read_copy_list(args.copy_list)
 807     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 808
 809     # validate output directory
 810     check_output_dir(bpid.target_dir, args.clean)
 811
 812     # do the copy
 813     backport_integrate_files = [
 814             ('Makefile.kernel', 'Makefile'),
 815             ('Kconfig.integrate', 'Kconfig'),
 816             ]
 817     backport_package_files = [(x, x) for x in [
 818         'Makefile',
 819         'kconf/',
 820         'Makefile.real',
 821         'Makefile.kernel',
 822         'Kconfig.package.hacks',
 823         'scripts/',
 824         '.blacklist.map',
 825         '.gitignore',
 826         'Makefile.build'] ]
 827     backport_package_files += [
 828             ('Kconfig.package', 'Kconfig'),
 829             ]
 830     backport_files = [(x, x) for x in [
 831         'Kconfig.sources',
 832         'compat/',
 833         'backport-include/',
 834     ]]
 835
 836     if not bpid.integrate:
 837         backport_files += backport_package_files
 838     else:
 839         backport_files += backport_integrate_files
 840
 841     if not args.git_revision:
 842         logwrite('Copy original source files ...')
 843     else:
 844         logwrite('Get original source files from git ...')
 845
 846     copy_files(os.path.join(source_dir, 'backport'), backport_files, bpid.target_dir)
 847
 848     git_debug_init(args)
 849
 850     if not args.git_revision:
 851         copy_files(args.kerneldir, copy_list, bpid.target_dir)
 852     else:
 853         copy_git_files(args.kerneldir, copy_list, args.git_revision, bpid.target_dir)
 854
 855     # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
 856     for src, copy_list in args.extra_driver:
 857         if (args.kup or args.kup_test):
 858             git_paranoia(src)
 859         copy_files(src, read_copy_list(open(copy_list, 'r')), bpid.target_dir)
 860
 861     git_debug_snapshot(args, 'Add driver sources')
 862
 863     disable_list = add_automatic_backports(args)
 864     if git_tracked_version:
 865         backports_version = "(see git)"
 866         kernel_version = "(see git)"
 867     else:
 868         backports_version = git.describe(tree=source_dir, extra_args=['--long'])
 869         kernel_version = git.describe(rev=args.git_revision or 'HEAD',
 870                                       tree=args.kerneldir,
 871                                       extra_args=['--long'])
 872
 873     if not bpid.integrate:
 874         f = open(os.path.join(bpid.target_dir, 'versions'), 'w')
 875         f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 876         f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
 877         f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
 878         if git_tracked_version:
 879             f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
 880         f.close()
 881         git_debug_snapshot(args, "add versions files")
 882     else:
 883         kconf_regexes = [
 884                 (re.compile(r'.*(?P<key>%%BACKPORT_DIR%%)'), '%%BACKPORT_DIR%%', 'backports/'),
 885                 (re.compile(r'.*(?P<key>%%BACKPORTS_VERSION%%).*'), '%%BACKPORTS_VERSION%%', backports_version),
 886                 (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_VERSION%%).*'), '%%BACKPORTED_KERNEL_VERSION%%', kernel_version),
 887                 (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_NAME%%).*'), '%%BACKPORTED_KERNEL_NAME%%', args.base_name),
 888         ]
 889         out = ''
 890         for l in open(os.path.join(bpid.target_dir, 'Kconfig'), 'r'):
 891             for r in kconf_regexes:
 892                 m = r[0].match(l)
 893                 if m:
 894                     l = re.sub(r'(' + r[1] + ')', r'' + r[2] + '', l)
 895             out += l
 896         outf = open(os.path.join(bpid.target_dir, 'Kconfig'), 'w')
 897         outf.write(out)
 898         outf.close()
 899         git_debug_snapshot(args, "modify top level backports/Kconfig with backports identity")
 900
 901     if disable_list:
 902         # No need to verify_sources() as compat's Kconfig has no 'source' call
 903         bpcfg = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'compat', 'Kconfig'), bpid)
 904         bpcfg.disable_symbols(disable_list)
 905     git_debug_snapshot(args, 'Add automatic backports')
 906
 907     failure = apply_patches(args, "backport", source_dir, 'patches', bpid.target_dir, logwrite)
 908     if failure:
 909         return failure
 910
 911     # Kernel integration requires Kconfig.versions already generated for you,
 912     # we cannot do this for a package as we have no idea what kernel folks
 913     # will be using.
 914     if bpid.integrate:
 915         kver = gen_version.kernelversion(bpid.project_dir)
 916         rel_specs = gen_version.get_rel_spec_stable(kver)
 917         if not rel_specs:
 918             logwrite('Cannot parse source kernel version, update parser')
 919             sys.exit(1)
 920         data = gen_version.genkconfig_versions(rel_specs)
 921         fo = open(os.path.join(bpid.target_dir, 'Kconfig.versions'), 'w')
 922         fo.write(data)
 923         fo.close()
 924         git_debug_snapshot(args, "generate kernel version requirement Kconfig file")
 925
 926     # some post-processing is required
 927     configtree = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'Kconfig'), bpid)
 928     ignore=['Kconfig.kernel', 'Kconfig.versions', 'Kconfig.local']
 929
 930     configtree.verify_sources(ignore=ignore)
 931     git_debug_snapshot(args, "verify sources on top level backports Kconfig")
 932
 933     orig_symbols = configtree.symbols()
 934
 935     logwrite('Modify Kconfig tree ...')
 936     configtree.prune_sources(ignore=ignore)
 937     git_debug_snapshot(args, "prune Kconfig tree")
 938
 939     if not bpid.integrate:
 940         configtree.force_tristate_modular()
 941         git_debug_snapshot(args, "force tristate options modular")
 942
 943     ignore = [os.path.join(bpid.target_dir, x) for x in [
 944                 'Kconfig.package.hacks',
 945                 'Kconfig.versions',
 946                 'Kconfig.local',
 947                 'Kconfig',
 948                 ]
 949             ]
 950     configtree.adjust_backported_configs(ignore=ignore, orig_symbols=orig_symbols)
 951     git_debug_snapshot(args, "adjust backports config symbols we port")
 952
 953     configtree.modify_selects()
 954     git_debug_snapshot(args, "convert select to depends on")
 955
 956     symbols = configtree.symbols()
 957
 958     # write local symbol list -- needed during packaging build
 959     if not bpid.integrate:
 960         f = open(os.path.join(bpid.target_dir, 'local-symbols'), 'w')
 961         for sym in symbols:
 962             f.write('%s=\n' % sym)
 963         f.close()
 964         git_debug_snapshot(args, "add symbols files")
 965     # also write Kconfig.local, representing all local symbols
 966     # with a BACKPORTED_ prefix
 967     f = open(os.path.join(bpid.target_dir, 'Kconfig.local'), 'w')
 968     for sym in symbols:
 969         f.write('config BACKPORTED_%s\n' % sym)
 970         f.write('\ttristate\n')
 971         f.write('\tdefault %s\n' % sym)
 972     f.close()
 973     git_debug_snapshot(args, "add Kconfig.local")
 974
 975     # add defconfigs that we want
 976     defconfigs_dir = os.path.join(source_dir, 'backport', 'defconfigs')
 977     os.mkdir(os.path.join(bpid.target_dir, 'defconfigs'))
 978     for dfbase in os.listdir(defconfigs_dir):
 979         copy_defconfig = True
 980         dfsrc = os.path.join(defconfigs_dir, dfbase)
 981         for line in open(dfsrc, 'r'):
 982             if not '=' in line:
 983                 continue
 984             line_ok = False
 985             for sym in symbols:
 986                 if sym + '=' in line:
 987                     line_ok = True
 988                     break
 989             if not line_ok:
 990                 copy_defconfig = False
 991                 break
 992         if copy_defconfig:
 993             shutil.copy(dfsrc, os.path.join(bpid.target_dir, 'defconfigs', dfbase))
 994
 995     git_debug_snapshot(args, "add (useful) defconfig files")
 996
 997     logwrite('Rewrite Makefiles and Kconfig files ...')
 998
 999     # rewrite Makefile and source symbols
1000
1001     # symbols we know only we can provide under the backport project prefix
1002     # for which we need an exception.
1003     skip_orig_syms = [ bpid.project_prefix + x for x in [
1004             'INTEGRATE',
1005             ]
1006     ]
1007     parse_orig_syms = [x for x in orig_symbols if x not in skip_orig_syms ]
1008     regexes = []
1009     for some_symbols in [parse_orig_syms[i:i + 50] for i in range(0, len(parse_orig_syms), 50)]:
1010         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
1011         regexes.append(re.compile(r, re.MULTILINE))
1012     for root, dirs, files in os.walk(bpid.target_dir):
1013         # don't go into .git dir (possible debug thing)
1014         if '.git' in dirs:
1015             dirs.remove('.git')
1016         for f in files:
1017             data = open(os.path.join(root, f), 'r').read()
1018             for r in regexes:
1019                 data = r.sub(r'' + bpid.full_prefix + '\\1', data)
1020             # we have an absolue path in $(src) since we compile out of tree
1021             data = re.sub(r'\$\(srctree\)/\$\(src\)', '$(src)', data)
1022             data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
1023             data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
1024             if bpid.integrate:
1025                 data = re.sub(r'CPTCFG_', bpid.full_prefix, data)
1026             fo = open(os.path.join(root, f), 'w')
1027             fo.write(data)
1028             fo.close()
1029
1030     git_debug_snapshot(args, "rename config symbol / srctree usage")
1031
1032     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
1033     if bpid.integrate:
1034         maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile'))
1035     else:
1036         maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile.kernel'))
1037     disable_kconfig = []
1038     disable_makefile = []
1039     for sym in maketree.get_impossible_symbols():
1040         disable_kconfig.append(sym[7:])
1041         disable_makefile.append(sym[7:])
1042
1043     configtree.disable_symbols(disable_kconfig)
1044     git_debug_snapshot(args, "disable impossible kconfig symbols")
1045
1046     # add kernel version dependencies to Kconfig, from the dependency list
1047     # we read previously
1048     for sym in tuple(deplist.keys()):
1049         new = []
1050         for dep in deplist[sym]:
1051             if "kconfig:" in dep:
1052                     kconfig_expr = dep.replace('kconfig: ', '')
1053                     new.append(kconfig_expr)
1054             elif (dep == "DISABLE"):
1055                     new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
1056             else:
1057                     new.append('!KERNEL_%s' % dep.replace('.', '_'))
1058         if bpid.integrate:
1059             deplist[sym] = ["BACKPORT_" + x for x in new]
1060         else:
1061             deplist[sym] = new
1062     configtree.add_dependencies(deplist)
1063     git_debug_snapshot(args, "add kernel version dependencies")
1064
1065     # disable things in makefiles that can't be selected and that the
1066     # build shouldn't recurse into because they don't exist -- if we
1067     # don't do that then a symbol from the kernel could cause the build
1068     # to attempt to recurse and fail
1069     #
1070     # Note that we split the regex after 50 symbols, this is because of a
1071     # limitation in the regex implementation (it only supports 100 nested
1072     # groups -- 50 seemed safer and is still fast)
1073     regexes = []
1074     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
1075         r = '^(([^#].*((' + bpid.full_prefix_resafe + '|CONFIG_)(' + '|'.join([s for s in some_symbols]) + ')))\W)'
1076         regexes.append(re.compile(r, re.MULTILINE))
1077     for f in maketree.get_makefiles():
1078         data = open(f, 'r').read()
1079         for r in regexes:
1080             data = r.sub(r'#\1', data)
1081         fo = open(f, 'w')
1082         fo.write(data)
1083         fo.close()
1084     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
1085
1086     if bpid.integrate:
1087         f = open(os.path.join(bpid.project_dir, 'Kconfig'), 'a')
1088         f.write('source "backports/Kconfig"\n')
1089         f.close()
1090         git_debug_snapshot(args, "hooked backport to top level Kconfig")
1091
1092         failure = apply_patches(args, "integration", source_dir, 'integration-patches/',
1093                                 bpid.project_dir, logwrite)
1094         if failure:
1095             return failure
1096
1097     if (args.kup or args.kup_test):
1098         req = reqs.Req()
1099         req.kup()
1100         if not req.reqs_match():
1101             sys.exit(1)
1102         upload_release(args, rel_prep, logwrite=logwrite)
1103
1104     logwrite('Done!')
1105     return 0
1106
1107 if __name__ == '__main__':
1108     ret = _main()
1109     if ret:
1110         sys.exit(ret)