gentree.py

   1 #!/usr/bin/env python
   2 #
   3 # Generate the output tree into a specified directory.
   4 #
   5
   6 import argparse, sys, os, errno, shutil, re, subprocess
   7 import tarfile, gzip
   8
   9 # find self
  10 source_dir = os.path.abspath(os.path.dirname(__file__))
  11 sys.path.append(source_dir)
  12 # and import libraries we have
  13 from lib import kconfig, patch, make
  14 from lib import bpgit as git
  15 from lib import bpgpg as gpg
  16 from lib import bpkup as kup
  17 from lib.tempdir import tempdir
  18 from lib import bpreqs as reqs
  19 from lib import bpversion as gen_version
  20
  21 class Bp_Identity(object):
  22     """
  23     folks considering multiple integrations may want to
  24     consider stuffing versioning info here as well but
  25     that will need thought/design on sharing compat and
  26     module namespaces.
  27
  28     Use the *_resafe when combining on regexps, although we currently
  29     don't support regexps there perhaps later we will and this will
  30     just make things safer for the output regardless. Once and if those
  31     are added, how we actually use the others for regular printing will
  32     need to be considered.
  33     """
  34     def __init__(self, integrate=False, kconfig_prefix='CPTCFG_',
  35                  project_prefix='', project_dir='',
  36                  target_dir='', target_dir_name='',
  37                  kconfig_source_var=None):
  38         self.integrate = integrate
  39         self.kconfig_prefix = kconfig_prefix
  40         self.kconfig_prefix_resafe = re.escape(kconfig_prefix)
  41         self.project_prefix = project_prefix
  42         self.project_prefix_resafe = re.escape(project_prefix)
  43         self.full_prefix = kconfig_prefix + project_prefix
  44         self.full_prefix_resafe = re.escape(self.full_prefix)
  45         self.project_dir = project_dir
  46         self.target_dir = target_dir
  47         self.target_dir_name = target_dir_name
  48         self.kconfig_source_var = kconfig_source_var
  49         if self.kconfig_source_var:
  50             self.kconfig_source_var_resafe = re.escape(self.kconfig_source_var)
  51         else:
  52             self.kconfig_source_var_resafe = None
  53
  54 def read_copy_list(copyfile):
  55     """
  56     Read a copy-list file and return a list of (source, target)
  57     tuples. The source and target are usually the same, but in
  58     the copy-list file there may be a rename included.
  59     """
  60     ret = []
  61     for item in copyfile:
  62         # remove leading/trailing whitespace
  63         item = item.strip()
  64         # comments
  65         if not item or item[0] == '#':
  66             continue
  67         if item[0] == '/':
  68             raise Exception("Input path '%s' is absolute path, this isn't allowed" % (item, ))
  69         if ' -> ' in item:
  70             srcitem, dstitem = item.split(' -> ')
  71             if (srcitem[-1] == '/') != (dstitem[-1] == '/'):
  72                 raise Exception("Cannot copy file/dir to dir/file")
  73         else:
  74             srcitem = dstitem = item
  75         ret.append((srcitem, dstitem))
  76     return ret
  77
  78
  79 def read_dependencies(depfilename):
  80     """
  81     Read a (the) dependency file and return the list of
  82     dependencies as a dictionary, mapping a Kconfig symbol
  83     to a list of kernel version dependencies.
  84
  85     If a backported feature that an upstream backported driver
  86     depends on had kconfig limitations (ie, debugging feature not
  87     available) a built constaint restriction can be expressed
  88     by using a kconfig expression. The kconfig expressions can
  89     be specified by using the "kconfig: " prefix.
  90
  91     While reading ignore blank or commented lines.
  92     """
  93     ret = {}
  94     depfile = open(depfilename, 'r')
  95     for item in depfile:
  96         kconfig_exp = ""
  97         item = item.strip()
  98         if not item or item[0] == '#':
  99             continue
 100         if "kconfig:" in item:
 101             sym, kconfig_exp = item.split(" ", 1)
 102             if not sym in ret:
 103                 ret[sym] = [kconfig_exp, ]
 104             else:
 105                 ret[sym].append(kconfig_exp)
 106         else:
 107             sym, dep = item.split()
 108             if not sym in ret:
 109                 ret[sym] = [dep, ]
 110             else:
 111                 ret[sym].append(dep)
 112     return ret
 113
 114
 115 def check_output_dir(d, clean):
 116     """
 117     Check that the output directory doesn't exist or is empty,
 118     unless clean is True in which case it's nuked. This helps
 119     sanity check the output when generating a tree, so usually
 120     running with --clean isn't suggested.
 121     """
 122     if clean:
 123         shutil.rmtree(d, ignore_errors=True)
 124     try:
 125         os.rmdir(d)
 126     except OSError as e:
 127         if e.errno != errno.ENOENT:
 128             raise
 129
 130
 131 def copytree(src, dst, symlinks=False, ignore=None):
 132     """
 133     Copy a directory tree. This differs from shutil.copytree()
 134     in that it allows destination directories to already exist.
 135     """
 136     names = os.listdir(src)
 137     if ignore is not None:
 138         ignored_names = ignore(src, names)
 139     else:
 140         ignored_names = set()
 141
 142     if not os.path.isdir(dst):
 143         os.makedirs(dst)
 144     errors = []
 145     for name in names:
 146         if name in ignored_names:
 147             continue
 148         srcname = os.path.join(src, name)
 149         dstname = os.path.join(dst, name)
 150         try:
 151             if symlinks and os.path.islink(srcname):
 152                 linkto = os.readlink(srcname)
 153                 os.symlink(linkto, dstname)
 154             elif os.path.isdir(srcname):
 155                 copytree(srcname, dstname, symlinks, ignore)
 156             else:
 157                 shutil.copy2(srcname, dstname)
 158         except (IOError, os.error) as why:
 159             errors.append((srcname, dstname, str(why)))
 160         # catch the Error from the recursive copytree so that we can
 161         # continue with other files
 162         except shutil.Error as err:
 163             errors.extend(err.args[0])
 164     try:
 165         shutil.copystat(src, dst)
 166     except WindowsError:
 167         # can't copy file access times on Windows
 168         pass
 169     except OSError as why:
 170         errors.extend((src, dst, str(why)))
 171     if errors:
 172         raise shutil.Error(errors)
 173
 174
 175 def copy_files(srcpath, copy_list, outdir):
 176     """
 177     Copy the copy_list files and directories from the srcpath
 178     to the outdir. The copy_list contains source and target
 179     names.
 180
 181     For now, it also ignores any *~ editor backup files, though
 182     this should probably be generalized (maybe using .gitignore?)
 183     Similarly the code that only copies some files (*.c, *.h,
 184     *.awk, Kconfig, Makefile) to avoid any build remnants in the
 185     kernel if they should exist.
 186     """
 187     for srcitem, tgtitem in copy_list:
 188         if tgtitem == '':
 189             copytree(srcpath, outdir, ignore=shutil.ignore_patterns('*~'))
 190         elif tgtitem[-1] == '/':
 191             def copy_ignore(dir, entries):
 192                 r = []
 193                 for i in entries:
 194                     if i[-2:] == '.o' or i[-1] == '~':
 195                         r.append(i)
 196                 return r
 197             copytree(os.path.join(srcpath, srcitem),
 198                      os.path.join(outdir, tgtitem),
 199                      ignore=copy_ignore)
 200         else:
 201             try:
 202                 os.makedirs(os.path.join(outdir, os.path.dirname(tgtitem)))
 203             except OSError as e:
 204                 # ignore dirs we might have created just now
 205                 if e.errno != errno.EEXIST:
 206                     raise
 207             shutil.copy(os.path.join(srcpath, srcitem),
 208                         os.path.join(outdir, tgtitem))
 209
 210
 211 def copy_git_files(srcpath, copy_list, rev, outdir):
 212     """
 213     "Copy" files from a git repository. This really means listing them with
 214     ls-tree and then using git show to obtain all the blobs.
 215     """
 216     for srcitem, tgtitem in copy_list:
 217         for m, t, h, f in git.ls_tree(rev=rev, files=(srcitem,), tree=srcpath):
 218             assert t == 'blob'
 219             f = os.path.join(outdir, f.replace(srcitem, tgtitem))
 220             d = os.path.dirname(f)
 221             if not os.path.exists(d):
 222                 os.makedirs(d)
 223             outf = open(f, 'w')
 224             git.get_blob(h, outf, tree=srcpath)
 225             outf.close()
 226             os.chmod(f, int(m, 8))
 227
 228 def automatic_backport_mangle_c_file(name):
 229     return name.replace('/', '-')
 230
 231
 232 def add_automatic_backports(args):
 233     disable_list = []
 234     export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
 235     bpi = kconfig.get_backport_info(os.path.join(args.bpid.target_dir, 'compat', 'Kconfig'))
 236     configtree = kconfig.ConfigTree(os.path.join(args.bpid.target_dir, 'Kconfig'), args.bpid)
 237     ignore=['Kconfig.kernel', 'Kconfig.versions']
 238     configtree.verify_sources(ignore=ignore)
 239     git_debug_snapshot(args, "verify sources for automatic backports")
 240     all_selects = configtree.all_selects()
 241     for sym, vals in bpi.items():
 242         if sym.startswith('BPAUTO_BUILD_'):
 243             if not sym[13:] in all_selects:
 244                 disable_list.append(sym)
 245                 continue
 246         symtype, module_name, c_files, h_files = vals
 247
 248         # first copy files
 249         files = []
 250         for f in c_files:
 251             files.append((f, os.path.join('compat', automatic_backport_mangle_c_file(f))))
 252         for f in h_files:
 253             files.append((os.path.join('include', f),
 254                           os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f))))
 255         if args.git_revision:
 256             copy_git_files(args.kerneldir, files, args.git_revision, args.bpid.target_dir)
 257         else:
 258             copy_files(args.kerneldir, files, args.bpid.target_dir)
 259
 260         # now add the Makefile line
 261         mf = open(os.path.join(args.bpid.target_dir, 'compat', 'Makefile'), 'a+')
 262         o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files]
 263         if symtype == 'tristate':
 264             if not module_name:
 265                 raise Exception('backporting a module requires a #module-name')
 266             for of in o_files:
 267                 mf.write('%s-objs += %s\n' % (module_name, of))
 268             mf.write('obj-$(%s%s) += %s.o\n' % (args.bpid.full_prefix, sym, module_name))
 269         elif symtype == 'bool':
 270             mf.write('compat-$(%s%s) += %s\n' % (args.bpid.full_prefix, sym, ' '.join(o_files)))
 271
 272         # finally create the include file
 273         syms = []
 274         for f in c_files:
 275             for l in open(os.path.join(args.bpid.target_dir, 'compat',
 276                                        automatic_backport_mangle_c_file(f)), 'r'):
 277                 m = export.match(l)
 278                 if m:
 279                     syms.append(m.group('sym'))
 280         for f in h_files:
 281             outf = open(os.path.join(args.bpid.target_dir, 'include', f), 'w')
 282             outf.write('/* Automatically created during backport process */\n')
 283             outf.write('#ifndef %s%s\n' % (args.bpid.full_prefix, sym))
 284             outf.write('#include_next <%s>\n' % f)
 285             outf.write('#else\n');
 286             for s in syms:
 287                 outf.write('#undef %s\n' % s)
 288                 outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s))
 289             outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), ))
 290             outf.write('#endif /* %s%s */\n' % (args.bpid.full_prefix, sym))
 291     return disable_list
 292
 293 def git_debug_init(args):
 294     """
 295     Initialize a git repository in the output directory and commit the current
 296     code in it. This is only used for debugging the transformations this code
 297     will do to the output later.
 298     """
 299     if not args.gitdebug:
 300         return
 301     # Git supports re-initialization, although not well documented it can
 302     # reset config stuff, lets avoid that if the tree already exists.
 303     if not os.path.exists(os.path.join(args.bpid.project_dir, '.git')):
 304         git.init(tree=args.bpid.project_dir)
 305     git.commit_all("Copied backport", tree=args.bpid.project_dir)
 306
 307
 308 def git_debug_snapshot(args, name):
 309     """
 310     Take a git snapshot for the debugging.
 311     """
 312     if not args.gitdebug:
 313         return
 314     git.commit_all(name, tree=args.bpid.project_dir)
 315
 316 def get_rel_spec_stable(rel):
 317     """
 318     Returns release specs for a linux-stable backports based release.
 319     """
 320     if ("rc" in rel):
 321         m = re.match(r"(?P<VERSION>\d+)\.+" \
 322                      "(?P<PATCHLEVEL>\d+)[.]*" \
 323                      "(?P<SUBLEVEL>\d*)" \
 324                      "[-rc]+(?P<RC_VERSION>\d+)\-+" \
 325                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 326                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 327                      rel)
 328     else:
 329         m = re.match(r"(?P<VERSION>\d+)\.+" \
 330                      "(?P<PATCHLEVEL>\d+)[.]*" \
 331                      "(?P<SUBLEVEL>\d*)\-+" \
 332                      "(?P<RELMOD_UPDATE>\d+)[-]*" \
 333                      "(?P<RELMOD_TYPE>[usnpc]*)", \
 334                      rel)
 335     if (not m):
 336         return m
 337     return m.groupdict()
 338
 339 def get_rel_spec_next(rel):
 340     """
 341     Returns release specs for a linux-next backports based release.
 342     """
 343     m = re.match(r"(?P<DATE_VERSION>\d+)[-]*" \
 344                  "(?P<RELMOD_UPDATE>\d*)[-]*" \
 345                  "(?P<RELMOD_TYPE>[usnpc]*)", \
 346                  rel)
 347     if (not m):
 348         return m
 349     return m.groupdict()
 350
 351 def get_rel_prep(rel):
 352     """
 353     Returns a dict with prep work details we need prior to
 354     uploading a backports release to kernel.org
 355     """
 356     rel_specs = get_rel_spec_stable(rel)
 357     is_stable = True
 358     rel_tag = ""
 359     paths = list()
 360     if (not rel_specs):
 361         rel_specs = get_rel_spec_next(rel)
 362         if (not rel_specs):
 363             sys.stdout.write("rel: %s\n" % rel)
 364             return None
 365         if (rel_specs['RELMOD_UPDATE'] == '0' or
 366             rel_specs['RELMOD_UPDATE'] == '1'):
 367             return None
 368         is_stable = False
 369         date = rel_specs['DATE_VERSION']
 370         year = date[0:4]
 371         if (len(year) != 4):
 372             return None
 373         month = date[4:6]
 374         if (len(month) != 2):
 375             return None
 376         day = date[6:8]
 377         if (len(day) != 2):
 378             return None
 379         paths.append(year)
 380         paths.append(month)
 381         paths.append(day)
 382         rel_tag = "backports-" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 383     else:
 384         ignore = "-"
 385         if (not rel_specs['RELMOD_UPDATE']):
 386             return None
 387         if (rel_specs['RELMOD_UPDATE'] == '0'):
 388             return None
 389         ignore += rel_specs['RELMOD_UPDATE']
 390         if (rel_specs['RELMOD_TYPE'] != ''):
 391             ignore += rel_specs['RELMOD_TYPE']
 392         base_rel = rel.replace(ignore, "")
 393         paths.append("v" + base_rel)
 394         rel_tag = "v" + rel.replace(rel_specs['RELMOD_TYPE'], "")
 395
 396     rel_prep = dict(stable = is_stable,
 397                     expected_tag = rel_tag,
 398                     paths_to_create = paths)
 399     return rel_prep
 400
 401 def create_tar_and_gz(tar_name, dir_to_tar):
 402     """
 403     We need both a tar file and gzip for kernel.org, the tar file
 404     gets signed, then we upload the compressed version, kup-server
 405     in the backend decompresses and verifies the tarball against
 406     our signature.
 407     """
 408     basename = os.path.basename(dir_to_tar)
 409     tar = tarfile.open(tar_name, "w")
 410     tar.add(dir_to_tar, basename)
 411     tar.close()
 412
 413     tar_file = open(tar_name, "r")
 414
 415     gz_file = gzip.GzipFile(tar_name + ".gz", 'wb')
 416     gz_file.write(tar_file.read())
 417     gz_file.close()
 418
 419 def upload_release(args, rel_prep, logwrite=lambda x:None):
 420     """
 421     Given a path of a relase make tarball out of it, PGP sign it, and
 422     then upload it to kernel.org using kup.
 423
 424     The linux-next based release do not require a RELMOD_UPDATE
 425     given that typically only one release is made per day. Using
 426     RELMOD_UPDATE for these releases is allowed though and if
 427     present it must be > 1.
 428
 429     The linux-stable based releases require a RELMOD_UPDATE.
 430
 431     RELMOD_UPDATE must be numeric and > 0 just as the RC releases
 432     of the Linux kernel.
 433
 434     The tree must also be tagged with the respective release, without
 435     the RELMOD_TYPE. For linux-next based releases this consists of
 436     backports- followed by DATE_VERSION and if RELMOD_TYPE is present.
 437     For linux-stable releases this consists of v followed by the
 438     full release version except the RELMOD_TYPE.
 439
 440     Uploads will not be allowed if these rules are not followed.
 441     """
 442     korg_path = "/pub/linux/kernel/projects/backports"
 443
 444     if (rel_prep['stable']):
 445         korg_path += "/stable"
 446
 447     parent = os.path.dirname(args.bpid.project_dir)
 448     release = os.path.basename(args.bpid.project_dir)
 449     tar_name = parent + '/' + release + ".tar"
 450     gzip_name = tar_name + ".gz"
 451
 452     create_tar_and_gz(tar_name, args.bpid.project_dir)
 453
 454     logwrite(gpg.sign(tar_name, extra_args=['--armor', '--detach-sign']))
 455
 456     logwrite("------------------------------------------------------")
 457
 458     if (not args.kup_test):
 459         logwrite("About to upload, current target path contents:")
 460     else:
 461         logwrite("kup-test: current target path contents:")
 462
 463     logwrite(kup.ls(path=korg_path))
 464
 465     for path in rel_prep['paths_to_create']:
 466         korg_path += '/' + path
 467         if (not args.kup_test):
 468             logwrite("create directory: %s" % korg_path)
 469             logwrite(kup.mkdir(korg_path))
 470     korg_path += '/'
 471     if (not args.kup_test):
 472         logwrite("upload file %s to %s" % (gzip_name, korg_path))
 473         logwrite(kup.put(gzip_name, tar_name + '.asc', korg_path))
 474         logwrite("\nFinished upload!\n")
 475         logwrite("Target path contents:")
 476         logwrite(kup.ls(path=korg_path))
 477     else:
 478         kup_cmd = "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (gzip_name, tar_name + '.asc', korg_path)
 479         logwrite("kup-test: skipping cmd: %s" % kup_cmd)
 480
 481 def apply_patches(args, desc, source_dir, patch_src, target_dir, logwrite=lambda x:None):
 482     """
 483     Given a path of a directories of patches and SmPL patches apply
 484     them on the target directory. If requested refresh patches, or test
 485     a specific SmPL patch.
 486     """
 487     logwrite('Applying patches from %s to %s ...' % (patch_src, target_dir))
 488     test_cocci = args.test_cocci or args.profile_cocci
 489     test_cocci_found = False
 490     patches = []
 491     sempatches = []
 492     for root, dirs, files in os.walk(os.path.join(source_dir, patch_src)):
 493         for f in files:
 494             if not test_cocci and f.endswith('.patch'):
 495                 patches.append(os.path.join(root, f))
 496             if f.endswith('.cocci'):
 497                 if test_cocci:
 498                     if f not in test_cocci:
 499                         continue
 500                     test_cocci_found = True
 501                     if args.test_cocci:
 502                         logwrite("Testing Coccinelle SmPL patch: %s" % test_cocci)
 503                     elif args.profile_cocci:
 504                         logwrite("Profiling Coccinelle SmPL patch: %s" % test_cocci)
 505                 sempatches.append(os.path.join(root, f))
 506     patches.sort()
 507     prefix_len = len(os.path.join(source_dir, patch_src)) + 1
 508     for pfile in patches:
 509         print_name = pfile[prefix_len:]
 510         # read the patch file
 511         p = patch.fromfile(pfile)
 512         # complain if it's not a patch
 513         if not p:
 514             raise Exception('No patch content found in %s' % print_name)
 515         # leading / seems to be stripped?
 516         if 'dev/null' in p.items[0].source:
 517             raise Exception('Patches creating files are not supported (in %s)' % print_name)
 518         # check if the first file the patch touches exists, if so
 519         # assume the patch needs to be applied -- otherwise continue
 520         patched_file = '/'.join(p.items[0].source.split('/')[1:])
 521         fullfn = os.path.join(target_dir, patched_file)
 522         if not os.path.exists(fullfn):
 523             if args.verbose:
 524                 logwrite("Not applying %s, not needed" % print_name)
 525             continue
 526         if args.verbose:
 527             logwrite("Applying patch %s" % print_name)
 528
 529         if args.refresh:
 530             # but for refresh, of course look at all files the patch touches
 531             for patchitem in p.items:
 532                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 533                 fullfn = os.path.join(target_dir, patched_file)
 534                 shutil.copyfile(fullfn, fullfn + '.orig_file')
 535
 536         process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
 537                                    stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
 538                                    close_fds=True, universal_newlines=True,
 539                                    cwd=target_dir)
 540         output = process.communicate(input=open(pfile, 'r').read())[0]
 541         output = output.split('\n')
 542         if output[-1] == '':
 543             output = output[:-1]
 544         if args.verbose:
 545             for line in output:
 546                 logwrite('> %s' % line)
 547         if process.returncode != 0:
 548             if not args.verbose:
 549                 logwrite("Failed to apply changes from %s" % print_name)
 550                 for line in output:
 551                     logwrite('> %s' % line)
 552             return 2
 553
 554         if args.refresh:
 555             pfilef = open(pfile + '.tmp', 'a')
 556             pfilef.write(p.top_header)
 557             pfilef.flush()
 558             for patchitem in p.items:
 559                 patched_file = '/'.join(patchitem.source.split('/')[1:])
 560                 fullfn = os.path.join(target_dir, patched_file)
 561                 process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
 562                                             '--label', 'a/' + patched_file,
 563                                             '--label', 'b/' + patched_file],
 564                                            stdout=pfilef, close_fds=True,
 565                                            universal_newlines=True, cwd=target_dir)
 566                 process.wait()
 567                 os.unlink(fullfn + '.orig_file')
 568                 if not process.returncode in (0, 1):
 569                     logwrite("Failed to diff to refresh %s" % print_name)
 570                     pfilef.close()
 571                     os.unlink(pfile + '.tmp')
 572                     return 2
 573             pfilef.close()
 574             os.rename(pfile + '.tmp', pfile)
 575
 576         # remove orig/rej files that patch sometimes creates
 577         for root, dirs, files in os.walk(target_dir):
 578             for f in files:
 579                 if f[-5:] == '.orig' or f[-4:] == '.rej':
 580                     os.unlink(os.path.join(root, f))
 581         git_debug_snapshot(args, "apply %s patch %s" % (desc, print_name))
 582
 583     sempatches.sort()
 584     prefix_len = len(os.path.join(source_dir, patch_src)) + 1
 585
 586     for cocci_file in sempatches:
 587         # Until Coccinelle picks this up
 588         pycocci = os.path.join(source_dir, 'devel/pycocci')
 589         cmd = [pycocci, cocci_file]
 590         extra_spatch_args = []
 591         if args.profile_cocci:
 592             cmd.append('--profile-cocci')
 593         cmd.append(os.path.abspath(target_dir))
 594         print_name = cocci_file[prefix_len:]
 595         if args.verbose:
 596             logwrite("Applying SmPL patch %s" % print_name)
 597         sprocess = subprocess.Popen(cmd,
 598                                     stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
 599                                     close_fds=True, universal_newlines=True,
 600                                     cwd=target_dir)
 601         output = sprocess.communicate()[0]
 602         sprocess.wait()
 603         if sprocess.returncode != 0:
 604             logwrite("Failed to process SmPL patch %s with %i" % (print_name, sprocess.returncode))
 605             return 2
 606         output = output.split('\n')
 607         if output[-1] == '':
 608             output = output[:-1]
 609         if args.verbose:
 610             for line in output:
 611                 logwrite('> %s' % line)
 612
 613         # remove cocci_backup files
 614         for root, dirs, files in os.walk(target_dir):
 615             for f in files:
 616                 if f.endswith('.cocci_backup'):
 617                     os.unlink(os.path.join(root, f))
 618         git_debug_snapshot(args, "apply %s SmPL patch %s" % (desc, print_name))
 619
 620     if test_cocci and test_cocci_found:
 621         logwrite('Done!')
 622         sys.exit(0)
 623
 624 def _main():
 625     # Our binary requirements go here
 626     req = reqs.Req()
 627     req.require('git')
 628     req.coccinelle('1.0.0-rc24')
 629     if not req.reqs_match():
 630         sys.exit(1)
 631
 632     # set up and parse arguments
 633     parser = argparse.ArgumentParser(description='generate backport tree')
 634     parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
 635                         help='Kernel tree to copy drivers from')
 636     parser.add_argument('outdir', metavar='<output directory>', type=str,
 637                         help='Directory to write the generated tree to')
 638     parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
 639                         default='copy-list',
 640                         help='File containing list of files/directories to copy, default "copy-list"')
 641     parser.add_argument('--git-revision', metavar='<revision>', type=str,
 642                         help='git commit revision (see gitrevisions(7)) to take objects from.' +
 643                              'If this is specified, the kernel tree is used as git object storage ' +
 644                              'and we use git ls-tree to get the files.')
 645     parser.add_argument('--clean', const=True, default=False, action="store_const",
 646                         help='Clean output directory instead of erroring if it isn\'t empty')
 647     parser.add_argument('--integrate', const=True, default=False, action="store_const",
 648                         help='Integrate a future backported kernel solution into ' +
 649                              'an older kernel tree source directory.')
 650     parser.add_argument('--refresh', const=True, default=False, action="store_const",
 651                         help='Refresh patches as they are applied, the source dir will be modified!')
 652     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
 653                         help='name of base tree, default just "Linux"')
 654     parser.add_argument('--gitdebug', '--git-debug', const=True, default=False, action="store_const",
 655                         help='Use git, in the output tree, to debug the various transformation steps ' +
 656                              'that the tree generation makes (apply patches, ...)')
 657     parser.add_argument('--verbose', const=True, default=False, action="store_const",
 658                         help='Print more verbose information')
 659     parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
 660                         action='append', default=[], help='Extra driver directory/copy-list.')
 661     parser.add_argument('--kup', const=True, default=False, action="store_const",
 662                         help='For maintainers: upload a release to kernel.org')
 663     parser.add_argument('--kup-test', const=True, default=False, action="store_const",
 664                         help='For maintainers: do all the work as if you were about to ' +
 665                              'upload to kernel.org but do not do the final `kup put` ' +
 666                              'and also do not run any `kup mkdir` commands. This will ' +
 667                              'however run `kup ls` on the target paths so ' +
 668                              'at the very least we test your kup configuration. ' +
 669                              'If this is your first time uploading use this first!')
 670     parser.add_argument('--test-cocci', metavar='<sp_file>', type=str, default=None,
 671                         help='Only use the cocci file passed for Coccinelle, don\'t do anything else, ' +
 672                              'also creates a git repo on the target directory for easy inspection ' +
 673                              'of changes done by Coccinelle.')
 674     parser.add_argument('--profile-cocci', metavar='<sp_file>', type=str, default=None,
 675                         help='Only use the cocci file passed and pass --profile  to Coccinelle, ' +
 676                              'also creates a git repo on the target directory for easy inspection ' +
 677                              'of changes done by Coccinelle.')
 678     args = parser.parse_args()
 679
 680     # When building a package we use CPTCFG as we can rely on the
 681     # fact that kconfig treats CONFIG_ as an environment variable
 682     # requring less changes on code. For kernel integration we use
 683     # the longer CONFIG_BACKPORT given that we'll be sticking to
 684     # the kernel symbol namespace, to address that we do a final
 685     # search / replace. Technically its possible to rely on the
 686     # same prefix for packaging as with kernel integration but
 687     # there are already some users of the CPTCFG prefix.
 688     bpid = None
 689     if args.integrate:
 690         bpid = Bp_Identity(integrate = args.integrate,
 691                            kconfig_prefix = 'CONFIG_',
 692                            project_prefix = 'BACKPORT_',
 693                            project_dir = args.outdir,
 694                            target_dir = os.path.join(args.outdir, 'backports/'),
 695                            target_dir_name = 'backports/',
 696                            kconfig_source_var = '$BACKPORT_DIR',
 697                            )
 698     else:
 699         bpid = Bp_Identity(integrate = args.integrate,
 700                            kconfig_prefix = 'CPTCFG_',
 701                            project_prefix = '',
 702                            project_dir = args.outdir,
 703                            target_dir = args.outdir,
 704                            target_dir_name = '',
 705                            kconfig_source_var = '$BACKPORT_DIR',
 706                            )
 707
 708     def logwrite(msg):
 709         sys.stdout.write(msg)
 710         sys.stdout.write('\n')
 711         sys.stdout.flush()
 712
 713     return process(args.kerneldir, args.copy_list,
 714                    git_revision=args.git_revision,
 715                    bpid=bpid,
 716                    clean=args.clean,
 717                    refresh=args.refresh, base_name=args.base_name,
 718                    gitdebug=args.gitdebug, verbose=args.verbose,
 719                    extra_driver=args.extra_driver,
 720                    kup=args.kup,
 721                    kup_test=args.kup_test,
 722                    test_cocci=args.test_cocci,
 723                    profile_cocci=args.profile_cocci,
 724                    logwrite=logwrite)
 725
 726 def process(kerneldir, copy_list_file, git_revision=None,
 727             bpid=None,
 728             clean=False, refresh=False, base_name="Linux", gitdebug=False,
 729             verbose=False, extra_driver=[], kup=False,
 730             kup_test=False,
 731             test_cocci=None,
 732             profile_cocci=None,
 733             logwrite=lambda x:None,
 734             git_tracked_version=False):
 735     class Args(object):
 736         def __init__(self, kerneldir, copy_list_file,
 737                      git_revision, bpid, clean, refresh, base_name,
 738                      gitdebug, verbose, extra_driver, kup,
 739                      kup_test,
 740                      test_cocci,
 741                      profile_cocci):
 742             self.kerneldir = kerneldir
 743             self.copy_list = copy_list_file
 744             self.git_revision = git_revision
 745             self.bpid = bpid
 746             self.clean = clean
 747             self.refresh = refresh
 748             self.base_name = base_name
 749             self.gitdebug = gitdebug
 750             self.verbose = verbose
 751             self.extra_driver = extra_driver
 752             self.kup = kup
 753             self.kup_test = kup_test
 754             self.test_cocci = test_cocci
 755             self.profile_cocci = profile_cocci
 756             if self.test_cocci or self.profile_cocci:
 757                 self.gitdebug = True
 758     def git_paranoia(tree=None, logwrite=lambda x:None):
 759         data = git.paranoia(tree)
 760         if (data['r'] != 0):
 761             logwrite('Cannot use %s' % tree)
 762             logwrite('%s' % data['output'])
 763             sys.exit(data['r'])
 764         else:
 765             logwrite('Validated tree: %s' % tree)
 766
 767     args = Args(kerneldir, copy_list_file,
 768                 git_revision, bpid, clean, refresh, base_name,
 769                 gitdebug, verbose, extra_driver, kup, kup_test,
 770                 test_cocci, profile_cocci)
 771     rel_prep = None
 772
 773     if bpid.integrate:
 774         if args.kup_test or args.test_cocci or args.profile_cocci or args.refresh:
 775             logwrite('Cannot use integration with:\n\tkup_test\n\ttest_cocci\n\tprofile_cocci\n\trefresh\n');
 776             sys.exit(1)
 777
 778     # start processing ...
 779     if (args.kup or args.kup_test):
 780         git_paranoia(source_dir, logwrite)
 781         git_paranoia(kerneldir, logwrite)
 782
 783         rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
 784         release = os.path.basename(bpid.target_dir)
 785         version = release.replace("backports-", "")
 786
 787         rel_prep = get_rel_prep(version)
 788         if (not rel_prep):
 789             logwrite('Invalid backports release name: %s' % release)
 790             logwrite('For rules on the release name see upload_release()')
 791             sys.exit(1)
 792         rel_type = "linux-stable"
 793         if (not rel_prep['stable']):
 794             rel_type = "linux-next"
 795         if (rel_prep['expected_tag'] != rel_describe):
 796             logwrite('Unexpected %s based backports release tag on' % rel_type)
 797             logwrite('the backports tree tree: %s\n' % rel_describe)
 798             logwrite('You asked to make a release with this ')
 799             logwrite('directory name: %s' % release)
 800             logwrite('The actual expected tag we should find on')
 801             logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag'])
 802             logwrite('For rules on the release name see upload_release()')
 803             sys.exit(1)
 804
 805     copy_list = read_copy_list(args.copy_list)
 806     deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
 807
 808     # validate output directory
 809     check_output_dir(bpid.target_dir, args.clean)
 810
 811     # do the copy
 812     backport_integrate_files = [
 813             ('Makefile.kernel', 'Makefile'),
 814             ('Kconfig.integrate', 'Kconfig'),
 815             ]
 816     backport_package_files = [(x, x) for x in [
 817         'Makefile',
 818         'kconf/',
 819         'Makefile.real',
 820         'Makefile.kernel',
 821         'Kconfig.package.hacks',
 822         'scripts/',
 823         '.blacklist.map',
 824         '.gitignore',
 825         'Makefile.build'] ]
 826     backport_package_files += [
 827             ('Kconfig.package', 'Kconfig'),
 828             ]
 829     backport_files = [(x, x) for x in [
 830         'Kconfig.sources',
 831         'compat/',
 832         'backport-include/',
 833     ]]
 834
 835     if not bpid.integrate:
 836         backport_files += backport_package_files
 837     else:
 838         backport_files += backport_integrate_files
 839
 840     if not args.git_revision:
 841         logwrite('Copy original source files ...')
 842     else:
 843         logwrite('Get original source files from git ...')
 844
 845     copy_files(os.path.join(source_dir, 'backport'), backport_files, bpid.target_dir)
 846
 847     git_debug_init(args)
 848
 849     if not args.git_revision:
 850         copy_files(args.kerneldir, copy_list, bpid.target_dir)
 851     else:
 852         copy_git_files(args.kerneldir, copy_list, args.git_revision, bpid.target_dir)
 853
 854     # FIXME: should we add a git version of this (e.g. --git-extra-driver)?
 855     for src, copy_list in args.extra_driver:
 856         if (args.kup or args.kup_test):
 857             git_paranoia(src)
 858         copy_files(src, read_copy_list(open(copy_list, 'r')), bpid.target_dir)
 859
 860     git_debug_snapshot(args, 'Add driver sources')
 861
 862     disable_list = add_automatic_backports(args)
 863     if git_tracked_version:
 864         backports_version = "(see git)"
 865         kernel_version = "(see git)"
 866     else:
 867         backports_version = git.describe(tree=source_dir, extra_args=['--long'])
 868         kernel_version = git.describe(rev=args.git_revision or 'HEAD',
 869                                       tree=args.kerneldir,
 870                                       extra_args=['--long'])
 871
 872     if not bpid.integrate:
 873         f = open(os.path.join(bpid.target_dir, 'versions'), 'w')
 874         f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
 875         f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
 876         f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
 877         if git_tracked_version:
 878             f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
 879         f.close()
 880         git_debug_snapshot(args, "add versions files")
 881     else:
 882         kconf_regexes = [
 883                 (re.compile(r'.*(?P<key>%%BACKPORT_DIR%%)'), '%%BACKPORT_DIR%%', 'backports/'),
 884                 (re.compile(r'.*(?P<key>%%BACKPORTS_VERSION%%).*'), '%%BACKPORTS_VERSION%%', backports_version),
 885                 (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_VERSION%%).*'), '%%BACKPORTED_KERNEL_VERSION%%', kernel_version),
 886                 (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_NAME%%).*'), '%%BACKPORTED_KERNEL_NAME%%', args.base_name),
 887         ]
 888         out = ''
 889         for l in open(os.path.join(bpid.target_dir, 'Kconfig'), 'r'):
 890             for r in kconf_regexes:
 891                 m = r[0].match(l)
 892                 if m:
 893                     l = re.sub(r'(' + r[1] + ')', r'' + r[2] + '', l)
 894             out += l
 895         outf = open(os.path.join(bpid.target_dir, 'Kconfig'), 'w')
 896         outf.write(out)
 897         outf.close()
 898         git_debug_snapshot(args, "modify top level backports/Kconfig with backports identity")
 899
 900     if disable_list:
 901         # No need to verify_sources() as compat's Kconfig has no 'source' call
 902         bpcfg = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'compat', 'Kconfig'), bpid)
 903         bpcfg.disable_symbols(disable_list)
 904     git_debug_snapshot(args, 'Add automatic backports')
 905
 906     failure = apply_patches(args, "backport", source_dir, 'patches', bpid.target_dir, logwrite)
 907     if failure:
 908         return failure
 909
 910     # Kernel integration requires Kconfig.versions already generated for you,
 911     # we cannot do this for a package as we have no idea what kernel folks
 912     # will be using.
 913     if bpid.integrate:
 914         kver = gen_version.kernelversion(bpid.project_dir)
 915         rel_specs = gen_version.get_rel_spec_stable(kver)
 916         if not rel_specs:
 917             logwrite('Cannot parse source kernel version, update parser')
 918             sys.exit(1)
 919         data = gen_version.genkconfig_versions(rel_specs)
 920         fo = open(os.path.join(bpid.target_dir, 'Kconfig.versions'), 'w')
 921         fo.write(data)
 922         fo.close()
 923         git_debug_snapshot(args, "generate kernel version requirement Kconfig file")
 924
 925     # some post-processing is required
 926     configtree = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'Kconfig'), bpid)
 927     ignore=['Kconfig.kernel', 'Kconfig.versions', 'Kconfig.local']
 928
 929     configtree.verify_sources(ignore=ignore)
 930     git_debug_snapshot(args, "verify sources on top level backports Kconfig")
 931
 932     orig_symbols = configtree.symbols()
 933
 934     logwrite('Modify Kconfig tree ...')
 935     configtree.prune_sources(ignore=ignore)
 936     git_debug_snapshot(args, "prune Kconfig tree")
 937
 938     if not bpid.integrate:
 939         configtree.force_tristate_modular()
 940         git_debug_snapshot(args, "force tristate options modular")
 941
 942     ignore = [os.path.join(bpid.target_dir, x) for x in [
 943                 'Kconfig.package.hacks',
 944                 'Kconfig.versions',
 945                 'Kconfig.local',
 946                 'Kconfig',
 947                 ]
 948             ]
 949     configtree.adjust_backported_configs(ignore=ignore, orig_symbols=orig_symbols)
 950     git_debug_snapshot(args, "adjust backports config symbols we port")
 951
 952     configtree.modify_selects()
 953     git_debug_snapshot(args, "convert select to depends on")
 954
 955     symbols = configtree.symbols()
 956
 957     # write local symbol list -- needed during packaging build
 958     if not bpid.integrate:
 959         f = open(os.path.join(bpid.target_dir, 'local-symbols'), 'w')
 960         for sym in symbols:
 961             f.write('%s=\n' % sym)
 962         f.close()
 963         git_debug_snapshot(args, "add symbols files")
 964     # also write Kconfig.local, representing all local symbols
 965     # with a BACKPORTED_ prefix
 966     f = open(os.path.join(bpid.target_dir, 'Kconfig.local'), 'w')
 967     for sym in symbols:
 968         f.write('config BACKPORTED_%s\n' % sym)
 969         f.write('\ttristate\n')
 970         f.write('\tdefault %s\n' % sym)
 971     f.close()
 972     git_debug_snapshot(args, "add Kconfig.local")
 973
 974     # add defconfigs that we want
 975     defconfigs_dir = os.path.join(source_dir, 'backport', 'defconfigs')
 976     os.mkdir(os.path.join(bpid.target_dir, 'defconfigs'))
 977     for dfbase in os.listdir(defconfigs_dir):
 978         copy_defconfig = True
 979         dfsrc = os.path.join(defconfigs_dir, dfbase)
 980         for line in open(dfsrc, 'r'):
 981             if not '=' in line:
 982                 continue
 983             line_ok = False
 984             for sym in symbols:
 985                 if sym + '=' in line:
 986                     line_ok = True
 987                     break
 988             if not line_ok:
 989                 copy_defconfig = False
 990                 break
 991         if copy_defconfig:
 992             shutil.copy(dfsrc, os.path.join(bpid.target_dir, 'defconfigs', dfbase))
 993
 994     git_debug_snapshot(args, "add (useful) defconfig files")
 995
 996     logwrite('Rewrite Makefiles and Kconfig files ...')
 997
 998     # rewrite Makefile and source symbols
 999
1000     # symbols we know only we can provide under the backport project prefix
1001     # for which we need an exception.
1002     skip_orig_syms = [ bpid.project_prefix + x for x in [
1003             'INTEGRATE',
1004             ]
1005     ]
1006     parse_orig_syms = [x for x in orig_symbols if x not in skip_orig_syms ]
1007     regexes = []
1008     for some_symbols in [parse_orig_syms[i:i + 50] for i in range(0, len(parse_orig_syms), 50)]:
1009         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
1010         regexes.append(re.compile(r, re.MULTILINE))
1011     for root, dirs, files in os.walk(bpid.target_dir):
1012         # don't go into .git dir (possible debug thing)
1013         if '.git' in dirs:
1014             dirs.remove('.git')
1015         for f in files:
1016             data = open(os.path.join(root, f), 'r').read()
1017             for r in regexes:
1018                 data = r.sub(r'' + bpid.full_prefix + '\\1', data)
1019             # we have an absolue path in $(src) since we compile out of tree
1020             data = re.sub(r'\$\(srctree\)/\$\(src\)', '$(src)', data)
1021             data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
1022             data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
1023             if bpid.integrate:
1024                 data = re.sub(r'CPTCFG_', bpid.full_prefix, data)
1025             fo = open(os.path.join(root, f), 'w')
1026             fo.write(data)
1027             fo.close()
1028
1029     git_debug_snapshot(args, "rename config symbol / srctree usage")
1030
1031     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
1032     if bpid.integrate:
1033         maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile'))
1034     else:
1035         maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile.kernel'))
1036     disable_kconfig = []
1037     disable_makefile = []
1038     for sym in maketree.get_impossible_symbols():
1039         disable_kconfig.append(sym[7:])
1040         disable_makefile.append(sym[7:])
1041
1042     configtree.disable_symbols(disable_kconfig)
1043     git_debug_snapshot(args, "disable impossible kconfig symbols")
1044
1045     # add kernel version dependencies to Kconfig, from the dependency list
1046     # we read previously
1047     for sym in tuple(deplist.keys()):
1048         new = []
1049         for dep in deplist[sym]:
1050             if "kconfig:" in dep:
1051                     kconfig_expr = dep.replace('kconfig: ', '')
1052                     new.append(kconfig_expr)
1053             elif (dep == "DISABLE"):
1054                     new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
1055             else:
1056                     new.append('!KERNEL_%s' % dep.replace('.', '_'))
1057         if bpid.integrate:
1058             deplist[sym] = ["BACKPORT_" + x for x in new]
1059         else:
1060             deplist[sym] = new
1061     configtree.add_dependencies(deplist)
1062     git_debug_snapshot(args, "add kernel version dependencies")
1063
1064     # disable things in makefiles that can't be selected and that the
1065     # build shouldn't recurse into because they don't exist -- if we
1066     # don't do that then a symbol from the kernel could cause the build
1067     # to attempt to recurse and fail
1068     #
1069     # Note that we split the regex after 50 symbols, this is because of a
1070     # limitation in the regex implementation (it only supports 100 nested
1071     # groups -- 50 seemed safer and is still fast)
1072     regexes = []
1073     for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
1074         r = '^(([^#].*((' + bpid.full_prefix_resafe + '|CONFIG_)(' + '|'.join([s for s in some_symbols]) + ')))\W)'
1075         regexes.append(re.compile(r, re.MULTILINE))
1076     for f in maketree.get_makefiles():
1077         data = open(f, 'r').read()
1078         for r in regexes:
1079             data = r.sub(r'#\1', data)
1080         fo = open(f, 'w')
1081         fo.write(data)
1082         fo.close()
1083     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
1084
1085     if bpid.integrate:
1086         f = open(os.path.join(bpid.project_dir, 'Kconfig'), 'a')
1087         f.write('source "backports/Kconfig"\n')
1088         f.close()
1089         git_debug_snapshot(args, "hooked backport to top level Kconfig")
1090
1091         failure = apply_patches(args, "integration", source_dir, 'integration-patches/',
1092                                 bpid.project_dir, logwrite)
1093         if failure:
1094             return failure
1095
1096     if (args.kup or args.kup_test):
1097         req = reqs.Req()
1098         req.kup()
1099         if not req.reqs_match():
1100             sys.exit(1)
1101         upload_release(args, rel_prep, logwrite=logwrite)
1102
1103     logwrite('Done!')
1104     return 0
1105
1106 if __name__ == '__main__':
1107     ret = _main()
1108     if ret:
1109         sys.exit(ret)