make-changelog.pl

   1 #!/usr/bin/env perl
   2
   3 use strict;
   4 use warnings;
   5 use Text::CSV;
   6
   7 my $range = $ARGV[0];
   8
   9 unless (defined $range) {
  10         printf STDERR "Usage: $0 range\n";
  11         exit 1;
  12 }
  13
  14 my $commit_url = 'https://git.lede-project.org/?p=source.git;a=commitdiff;h=%s';
  15
  16 my @weblinks = (
  17         [ qr'^[^:]+://(git.lede-project.org/)(.+)$' => 'https://%s?p=%s;a=commitdiff;h=%%s' ],
  18         [ qr'^[^:]+://(git.openwrt.org/)(.+)$'      => 'https://%s?p=%s;a=commitdiff;h=%%s' ],
  19         [ qr'^[^:]+://(github.com/.+?)(?:\.git)?$'  => 'https://%s/commit/%%s' ],
  20         [ qr'^[^:]+://git.kernel.org/pub/scm/(.+)$' => 'https://git.kernel.org/cgit/%s/commit/?id=%%s' ],
  21         [ qr'^[^:]+://w1.fi/(?:.+/)?(.+)\.git$'     => 'https://w1.fi/cgit/%s/commit/?id=%%s' ],
  22 );
  23
  24
  25 my %topics;
  26 my %commits;
  27 my %reverts;
  28 my $index = 0;
  29
  30 sub line(*$)
  31 {
  32         my ($fh, $default) = @_;
  33
  34         my $line = readline $fh;
  35
  36         if (defined $line)
  37         {
  38                 chomp $line;
  39                 return $line;
  40         }
  41
  42         return $default;
  43 }
  44
  45 my @topic_paths = (
  46         [ qr'^package/(kernel)/linux',                 'Kernel' ],
  47         [ qr'^(target/linux/generic|include/kernel-version.mk)', 'Kernel' ],
  48         [ qr'^package/kernel/(mac80211)',              'Wireless / Common' ],
  49         [ qr'^package/kernel/(ath10k-ct)',             'Wireless / Ath10k CT' ],
  50         [ qr'^package/kernel/(mt76)',                  'Wireless / MT76' ],
  51         [ qr'^package/(base-files)/',                  'Packages / LEDE base files' ],
  52         [ qr'^package/(boot)/',                        'Packages / Boot Loaders' ],
  53         [ qr'^package/firmware/',                      'Packages / Firmware' ],
  54         [ qr'^package/.+/(uhttpd|usbmode|jsonfilter|ugps|libubox|procd|mountd|ubus|uci|usign|rpcd|fstools|ubox)/', 'Packages / LEDE system userland' ],
  55         [ qr'^package/.+/(iwinfo|umbim|uqmi|relayd|mdns|firewall|netifd|uclient|ustream-ssl|gre|ipip|qos-scripts|swconfig|vti|6in4|6rd|6to4|ds-lite|map|odhcp6c|odhcpd)/', 'Packages / LEDE network userland' ],
  56         [ qr'^package/[^/]+/([^/]+)',                  'Packages / Common' ],
  57         [ qr'^target/sdk/',                            'Build System / SDK' ],
  58         [ qr'^target/imagebuilder/',                   'Build System / Image Builder' ],
  59         [ qr'^target/toolchain/',                      'Build System / Toolchain' ],
  60         [ qr'^target/linux/([^/]+)',                   'Target / $1' ],
  61         [ qr'^(tools)/[^/]+',                          'Build System / Host Utilities' ],
  62         [ qr'^(toolchain)/[^/]+',                      'Build System / Toolchain' ],
  63         [ qr'^(config/|include/|scripts/|target/[^/]+$|Makefile|rules\.mk)', 'Build System / Buildroot' ],
  64         [ qr'^(feeds)\b',                              'Build System / Feeds' ],
  65 );
  66
  67 my @subhistory_matches = (
  68         qr'(?i)^\S+: update to\b',
  69         qr'(?i)^\S+: Upstep to\b',
  70         qr'(?i)^\S+: bump to\b',
  71         qr'(?i)^\S+: fix\b',
  72         qr'(?i)^\S+: backport\b',
  73         qr'(?i)\blatest HEAD\b',
  74 );
  75
  76 sub match_topics(@)
  77 {
  78         my %topics;
  79
  80         foreach my $path (@_)
  81         {
  82                 foreach my $rs (@topic_paths)
  83                 {
  84                         if ($path =~ $rs->[0])
  85                         {
  86                                 my $m = $1;
  87                                 my $s = $rs->[1];
  88
  89                                 $s =~ s!\$1!$m!g;
  90                                 $topics{$s}++;
  91
  92                                 last;
  93                         }
  94                 }
  95         }
  96
  97         my @topics = sort keys %topics;
  98         return (@topics > 0 ? @topics : ('Miscellaneous'));
  99 }
 100
 101 sub parse_history($$)
 102 {
 103         my ($dir, $range) = @_;
 104
 105         my @commits;
 106         my ($max_add, $total_add, $max_del, $total_del) = (0, 0, 0, 0);
 107
 108         if (open GIT, '-|', 'git', "--git-dir=$dir/.git", 'log', '--format=@@%n%H%n%s%n%b%n@@', '--numstat', '--reverse', '--no-merges', $range)
 109         {
 110                 # skip header line
 111                 line(*GIT, undef);
 112
 113                 while (1)
 114                 {
 115                         my $hash = line(GIT, '');
 116                         my $subject = line(GIT, '');
 117
 118                         last unless (length($subject) && $hash =~ m!^!);
 119
 120                         my $line = '';
 121                         my $body = '';
 122                         my @files;
 123                         my ($add, $del) = (0, 0);
 124
 125                         my $is_revert = $subject =~ m!^Revert !;
 126
 127                         $reverts{$hash}++ if $is_revert;
 128
 129                         while ($line ne '@@')
 130                         {
 131                                 $body .= length($line) ? "$line\n" : '';
 132                                 $line = line(*GIT, '@@');
 133
 134                                 if ($is_revert && $line =~ m!\b([0-9a-f]{40})\b!)
 135                                 {
 136                                         $reverts{$1}++;
 137                                 }
 138                         }
 139
 140                         $line = '';
 141
 142                         while ($line ne '@@')
 143                         {
 144                                 if ($line =~ m!^(\d+|-)\s+(\d+|-)\s+(.+)$!)
 145                                 {
 146                                         $add += ($1 eq '-') ? 0 : int($1);
 147                                         $del += ($2 eq '-') ? 0 : int($2);
 148                                         push @files, $3;
 149                                 }
 150
 151                                 $line = line(*GIT, '@@');
 152                         }
 153
 154                         my $commit = [
 155                                 $index++,
 156                                 $hash,
 157                                 $subject,
 158                                 $body,
 159                                 \@files,
 160                                 undef,
 161                                 undef,
 162                                 $add,
 163                                 $del
 164                         ];
 165
 166                         $total_add += $add;
 167                         $total_del += $del;
 168
 169                         $max_add = ($add > $max_add) ? $add : $max_add;
 170                         $max_del = ($del > $max_del) ? $del : $max_del;
 171
 172                         push @commits, $commit;
 173                 }
 174
 175                 close GIT;
 176         }
 177
 178         if (@commits > 0 && $commits[0][2] =~ /\brevert to branch defaults$/)
 179         {
 180                 shift @commits;
 181         }
 182
 183         return wantarray ? @commits : \@commits;
 184 }
 185
 186 sub fetch_subhistory($$$)
 187 {
 188         my ($url, $old, $new) = @_;
 189
 190         (my $path = $url) =~ s![^a-z0-9_-]+!-!g;
 191
 192         unless (-d "/tmp/repos/$path")
 193         {
 194                 mkdir('/tmp/repos');
 195                 system('git', 'clone', '--quiet', $url, "/tmp/repos/$path");
 196         }
 197         else
 198         {
 199                 system('git', "--work-tree=/tmp/repos/$path", "--git-dir=/tmp/repos/$path/.git", 'pull', '--quiet');
 200         }
 201
 202         return parse_history("/tmp/repos/$path", "$old..$new");
 203 }
 204
 205 sub requires_subhistory($$$)
 206 {
 207         my ($subject, $body, $hash) = @_;
 208
 209         foreach my $re (@subhistory_matches)
 210         {
 211                 if ($subject =~ $re || $body =~ $re)
 212                 {
 213                         if (open DIFF, '-|', 'git', 'diff', "$hash^!")
 214                         {
 215                                 my ($url, $old, $new);
 216
 217                                 while (defined(my $line = readline DIFF))
 218                                 {
 219                                         chomp $line;
 220
 221                                         if ($line =~ m!^[ +]PKG_SOURCE_URL\s*:?=\s*(\S+)!)
 222                                         {
 223                                                 $url = $1;
 224                                                 $url =~ s!\$\(LEDE_GIT\)!https://git.lede-project.org!g;
 225                                                 $url =~ s!\$\(OPENWRT_GIT\)!https://git.openwrt.org!g;
 226                                         }
 227                                         elsif ($line =~ m!^-\S+\s*:?=\s*([a-f0-9]{40})\b!)
 228                                         {
 229                                                 $old = $1;
 230                                         }
 231                                         elsif ($line =~ m!^\+\S+\s*:?=\s*([a-f0-9]{40})\b!)
 232                                         {
 233                                                 $new = $1;
 234                                         }
 235
 236                                         if ($url && $old && $new)
 237                                         {
 238                                                 return ($url, $old, $new);
 239                                         }
 240                                 }
 241
 242                                 close DIFF;
 243                         }
 244                 }
 245         }
 246
 247         return ();
 248 }
 249
 250 sub find_weblink_template($)
 251 {
 252         my ($url) = @_;
 253
 254         foreach my $rt (@weblinks)
 255         {
 256                 my @m = $url =~ $rt->[0];
 257                 if (@m > 0)
 258                 {
 259                         return sprintf $rt->[1], @m;
 260                 }
 261         }
 262
 263         warn "No web link template for <$url>\n";
 264         return undef;
 265 }
 266
 267 sub format_stat($)
 268 {
 269         my ($commit) = @_;
 270
 271         my $s = '';
 272         my $c = '<color #ccc>%s</color>';
 273         my $g = '<color #282>%s</color>';
 274         my $r = '<color #f00>%s</color>';
 275
 276         if ($commit->[7] > 1000)
 277         {
 278                 $s .= sprintf $g, sprintf '+%.1fK', $commit->[7] / 1000;
 279         }
 280         elsif ($commit->[7] > 0)
 281         {
 282                 $s .= sprintf $g, sprintf '+%d', $commit->[7];
 283         }
 284
 285         if ($commit->[8] > 1000)
 286         {
 287                 $s .= $s ? sprintf($c, ',') : '';
 288                 $s .= sprintf $r, sprintf '-%.1fK', $commit->[8] / 1000;
 289         }
 290         elsif ($commit->[8] > 0)
 291         {
 292                 $s .= $s ? sprintf($c, ',') : '';
 293                 $s .= sprintf $r, sprintf '-%d', $commit->[8];
 294         }
 295
 296         return sprintf($c, '(') . $s . sprintf($c, ')');
 297 }
 298
 299 sub format_subject($$)
 300 {
 301         my ($subject, $body) = @_;
 302
 303         if (length($subject) > 80)
 304         {
 305                 $subject = substr($subject, 0, 77) . '...';
 306         }
 307
 308         $subject =~ s!^([^\s:]+):\s*!</nowiki>**<nowiki>$1:</nowiki>** <nowiki>!g;
 309
 310         $subject = sprintf '<nowiki>%s</nowiki>', $subject;
 311         $subject =~ s!<nowiki></nowiki>!!g;
 312
 313         return $subject;
 314 }
 315
 316 sub format_change($)
 317 {
 318         my ($change) = @_;
 319
 320         printf "''[[%s|%s]]'' %s //%s//\\\\\n",
 321                 sprintf($commit_url, $change->[1]),
 322                 substr($change->[1], 0, 7),
 323                 format_subject($change->[2], $change->[3]),
 324                 format_stat($change);
 325
 326         if ($change->[6])
 327         {
 328                 my $n = 0;
 329                 foreach my $subchange (@{$change->[6]})
 330                 {
 331                         if ($change->[5])
 332                         {
 333                                 printf " => ''[[%s|%s]]'' %s //%s//\\\\\n",
 334                                         sprintf($change->[5], $subchange->[1]),
 335                                         substr($subchange->[1], 0, 7),
 336                                         format_subject($subchange->[2], $subchange->[3]),
 337                                         format_stat($subchange);
 338                         }
 339                         else
 340                         {
 341                                 printf " => ''%s'' %s //%s//\\\\\n",
 342                                         substr($subchange->[1], 0, 7),
 343                                         format_subject($subchange->[2], $subchange->[3]),
 344                                         format_stat($subchange);
 345                         }
 346
 347                         if (++$n > 15 && @{$change->[6]} > $n)
 348                         {
 349                                 printf " => + //%u more...//\\\\\n", @{$change->[6]} - $n;
 350                                 last;
 351                         }
 352                 }
 353         }
 354 }
 355
 356 sub fetch_cve_info()
 357 {
 358         unless (-f '/tmp/cveinfo.csv')
 359         {
 360                 system('wget', '-O', '/tmp/cveinfo.csv.gz', 'https://cve.mitre.org/data/downloads/allitems.csv.gz') && return 0;
 361                 system('gunzip', '-f', '/tmp/cveinfo.csv.gz') && return 0;
 362         }
 363
 364         return 1;
 365 }
 366
 367 sub parse_cves(@)
 368 {
 369         my $csv = Text::CSV->new({ binary => 1 });
 370         my %cves;
 371
 372         if (fetch_cve_info() && $csv)
 373         {
 374                 if (open CVE, '<', '/tmp/cveinfo.csv')
 375                 {
 376                         while (defined(my $row = $csv->getline(*CVE)))
 377                         {
 378                                 foreach my $cve_id (@_)
 379                                 {
 380                                         if ($row->[0] eq $cve_id)
 381                                         {
 382                                                 $cves{$cve_id} = [$row->[2], $row->[6]];
 383                                                 last;
 384                                         }
 385                                 }
 386                         }
 387
 388                         close CVE;
 389                 }
 390         }
 391
 392         return \%cves;
 393 }
 394
 395 sub fetch_bug_info()
 396 {
 397         unless (-f '/tmp/buginfo.csv')
 398         {
 399                 system('wget', '-O', '/tmp/buginfo.csv', 'https://bugs.lede-project.org/index.php?string=&project=2&do=index&export_list=Export+Tasklist&advancedsearch=on&type%5B%5D=&sev%5B%5D=&pri%5B%5D=&due%5B%5D=&reported%5B%5D=&cat%5B%5D=&status%5B%5D=&percent%5B%5D=&opened=&dev=&closed=&duedatefrom=&duedateto=&changedfrom=&changedto=&openedfrom=&openedto=&closedfrom=&closedto=') && return 0;
 400         }
 401
 402         return 1;
 403 }
 404
 405 sub parse_bugs(@)
 406 {
 407         my $csv = Text::CSV->new({ binary => 1, allow_loose_quotes => 1, eol => "\012" });
 408         my %bugs;
 409
 410         if (fetch_bug_info() && $csv)
 411         {
 412                 if (open BUG, '<', '/tmp/buginfo.csv')
 413                 {
 414                         while (defined(my $row = $csv->getline(*BUG)))
 415                         {
 416                                 foreach my $bug_id (@_)
 417                                 {
 418                                         if ($row->[0] eq $bug_id)
 419                                         {
 420                                                 $bugs{$bug_id} = [$row->[4], $row->[5]];
 421                                                 last;
 422                                         }
 423                                 }
 424                         }
 425
 426                         $csv->error_diag;
 427
 428                         close BUG;
 429                 }
 430         }
 431
 432         return \%bugs;
 433 }
 434
 435
 436 my @commits = parse_history('.', $range);
 437 my (%bugs, %cves);
 438
 439 foreach my $commit (@commits)
 440 {
 441         my @topics = match_topics(@{$commit->[4]});
 442
 443         unless ($commit->[5])
 444         {
 445                 my ($su, $so, $sn) = requires_subhistory($commit->[2], $commit->[3], $commit->[1]);
 446                 if ($su) {
 447                         $commit->[5] = find_weblink_template($su);
 448                         $commit->[6] = fetch_subhistory($su, $so, $sn);
 449                 }
 450         }
 451
 452         foreach my $topic (@topics)
 453         {
 454                 $topics{$topic} ||= [ ];
 455                 push @{$topics{$topic}}, $commit;
 456         }
 457
 458         my (%bug_ids, %cve_ids);
 459
 460         foreach my $bug ($commit->[2] =~ m!([A-Z]*#\d+)\b!g,
 461                          $commit->[3] =~ m!([A-Z]*#\d+)\b!g)
 462         {
 463                 if ($bug =~ m!^(?:FS|GH|)#(\d+)$!)
 464                 {
 465                         $bug_ids{$1}++;
 466                 }
 467         }
 468
 469         foreach my $cve ($commit->[2] =~ m!\b(CVE-\d+-\d+|\d+-CVE-\d+)\b!g,
 470                          $commit->[3] =~ m!\b(CVE-\d+-\d+|\d+-CVE-\d+)\b!g)
 471         {
 472                 # fix misspelled CVE IDs
 473                 $cve =~ s!^(\d+)-CVE-!CVE-$1-!;
 474                 $cve_ids{$cve}++;
 475         }
 476
 477         foreach my $bug (keys %bug_ids)
 478         {
 479                 $bugs{$bug} ||= [ ];
 480                 push @{$bugs{$bug}}, $commit;
 481         }
 482
 483         foreach my $cve (keys %cve_ids)
 484         {
 485                 $cves{$cve} ||= [ ];
 486                 push @{$cves{$cve}}, $commit;
 487         }
 488 }
 489
 490
 491 my @topics = sort { (($a eq 'Miscellaneous') <=> ($b eq 'Miscellaneous')) || $a cmp $b } keys %topics;
 492
 493 foreach my $topic (@topics)
 494 {
 495         my @commits = grep { !$reverts{$_->[1]} } @{$topics{$topic}};
 496
 497         printf "==== %s (%d change%s) ====\n", $topic, 0 + @commits, @commits > 1 ? 's' : '';
 498
 499         foreach my $change (sort { $a->[0] <=> $b->[0] } @commits)
 500         {
 501                 format_change($change);
 502         }
 503
 504         print "\n";
 505 }
 506
 507 my @bugs = sort { int($a) <=> int($b) } keys %bugs;
 508 my $bug_info = parse_bugs(@bugs);
 509
 510 @bugs = grep { $bug_info->{$_} && $bug_info->{$_}[0] } @bugs;
 511
 512 if (@bugs > 0)
 513 {
 514         printf "===== Addressed bugs =====\n";
 515
 516         foreach my $bug (@bugs)
 517         {
 518                 printf "=== #%s ===\n", $bug;
 519                 printf "**Description:** <nowiki>%s</nowiki>\\\\\n", $bug_info->{$bug}[0];
 520                 printf "**Link:** [[https://bugs.lede-project.org/index.php?do=details&task_id=%s]]\\\\\n", $bug;
 521                 printf "**Commits:**\\\\\n";
 522
 523                 foreach my $commit (@{$bugs{$bug}})
 524                 {
 525                         format_change($commit);
 526                 }
 527
 528                 printf "\\\\\n";
 529         }
 530
 531         printf "\n";
 532 }
 533
 534 my @cves =
 535         map { $_->[1] }
 536         sort { ($a->[0] <=> $b->[0]) || ($a->[1] cmp $b->[1]) }
 537         map { $_ =~ m!^CVE-(\d+)-(\d+)$! ? [ $1 * 10000000 + $2, $_ ] : [ 0, $_ ] }
 538         keys %cves;
 539
 540 my $cve_info = parse_cves(@cves);
 541
 542 if (@cves > 0)
 543 {
 544         printf "===== Security fixes ====\n";
 545
 546         foreach my $cve (@cves)
 547         {
 548                 printf "=== %s ===\n", $cve;
 549
 550                 if ($cve_info->{$cve} && $cve_info->{$cve}[0])
 551                 {
 552                         printf "**Description:** <nowiki>%s</nowiki>\n\n", $cve_info->{$cve}[0];
 553                 }
 554
 555                 printf "**Link:** [[https://cve.mitre.org/cgi-bin/cvename.cgi?name=%s]]\\\\\n", $cve;
 556                 printf "**Commits:**\\\\\n";
 557
 558                 foreach my $commit (@{$cves{$cve}})
 559                 {
 560                         format_change($commit);
 561                 }
 562
 563                 printf "\\\\\n";
 564         }
 565
 566         printf "\n";
 567 }