make-changelog.pl

   1 #!/usr/bin/env perl
   2
   3 use strict;
   4 use warnings;
   5 use Text::CSV;
   6
   7 my $range = $ARGV[0];
   8
   9 unless (defined $range) {
  10         printf STDERR "Usage: $0 range\n";
  11         exit 1;
  12 }
  13
  14 my $commit_url = 'https://git.openwrt.org/?p=openwrt/openwrt.git;a=commitdiff;h=%s';
  15
  16 my @weblinks = (
  17         [ qr'^[^:]+://(git.lede-project.org/)(.+)$' => 'https://%s?p=%s;a=commitdiff;h=%%s' ],
  18         [ qr'^[^:]+://(git.openwrt.org/)(.+)$'      => 'https://%s?p=%s;a=commitdiff;h=%%s' ],
  19         [ qr'^[^:]+://(github.com/.+?)(?:\.git)?$'  => 'https://%s/commit/%%s' ],
  20         [ qr'^[^:]+://git.kernel.org/pub/scm/(.+)$' => 'https://git.kernel.org/cgit/%s/commit/?id=%%s' ],
  21         [ qr'^[^:]+://w1.fi/(?:.+/)?(.+)\.git$'     => 'https://w1.fi/cgit/%s/commit/?id=%%s' ],
  22 );
  23
  24
  25 my %topics;
  26 my %commits;
  27 my %reverts;
  28 my $index = 0;
  29
  30 sub line(*$)
  31 {
  32         my ($fh, $default) = @_;
  33
  34         my $line = readline $fh;
  35
  36         if (defined $line)
  37         {
  38                 chomp $line;
  39                 return $line;
  40         }
  41
  42         return $default;
  43 }
  44
  45 my @topic_paths = (
  46         [ qr'^package/(kernel)/linux',                 'Kernel' ],
  47         [ qr'^(target/linux/generic|include/kernel-version.mk)', 'Kernel' ],
  48         [ qr'^package/kernel/(mac80211)',              'Wireless / Common' ],
  49         [ qr'^package/kernel/(ath10k-ct)',             'Wireless / Ath10k CT' ],
  50         [ qr'^package/kernel/(mt76)',                  'Wireless / MT76' ],
  51         [ qr'^package/(base-files)/',                  'Packages / LEDE base files' ],
  52         [ qr'^package/(boot)/',                        'Packages / Boot Loaders' ],
  53         [ qr'^package/firmware/',                      'Packages / Firmware' ],
  54         [ qr'^package/.+/(uhttpd|usbmode|jsonfilter|ugps|libubox|procd|mountd|ubus|uci|usign|rpcd|fstools|ubox)/', 'Packages / LEDE system userland' ],
  55         [ qr'^package/.+/(iwinfo|umbim|uqmi|relayd|mdns|firewall|netifd|uclient|ustream-ssl|gre|ipip|qos-scripts|swconfig|vti|6in4|6rd|6to4|ds-lite|map|odhcp6c|odhcpd)/', 'Packages / LEDE network userland' ],
  56         [ qr'^package/[^/]+/([^/]+)',                  'Packages / Common' ],
  57         [ qr'^target/sdk/',                            'Build System / SDK' ],
  58         [ qr'^target/imagebuilder/',                   'Build System / Image Builder' ],
  59         [ qr'^target/toolchain/',                      'Build System / Toolchain' ],
  60         [ qr'^target/linux/([^/]+)',                   'Target / $1' ],
  61         [ qr'^(tools)/[^/]+',                          'Build System / Host Utilities' ],
  62         [ qr'^(toolchain)/[^/]+',                      'Build System / Toolchain' ],
  63         [ qr'^(config/|include/|scripts/|target/[^/]+$|Makefile|rules\.mk)', 'Build System / Buildroot' ],
  64         [ qr'^(feeds)\b',                              'Build System / Feeds' ],
  65 );
  66
  67 my @subhistory_matches = (
  68         qr'(?i)^\S+: update to\b',
  69         qr'(?i)^\S+: Upstep to\b',
  70         qr'(?i)^\S+: bump to\b',
  71         qr'(?i)^\S+: fix\b',
  72         qr'(?i)^\S+: backport\b',
  73         qr'(?i)\blatest HEAD\b',
  74 );
  75
  76 sub match_topics(@)
  77 {
  78         my %topics;
  79
  80         foreach my $path (@_)
  81         {
  82                 foreach my $rs (@topic_paths)
  83                 {
  84                         if ($path =~ $rs->[0])
  85                         {
  86                                 my $m = $1;
  87                                 my $s = $rs->[1];
  88
  89                                 $s =~ s!\$1!$m!g;
  90                                 $topics{$s}++;
  91
  92                                 last;
  93                         }
  94                 }
  95         }
  96
  97         my @topics = sort keys %topics;
  98         return (@topics > 0 ? @topics : ('Miscellaneous'));
  99 }
 100
 101 sub parse_history($$)
 102 {
 103         my ($dir, $range) = @_;
 104
 105         my @commits;
 106         my ($max_add, $total_add, $max_del, $total_del) = (0, 0, 0, 0);
 107
 108         if (open GIT, '-|', 'git', "--git-dir=$dir/.git", 'log', '--format=@@%n%H%n%s%n%b%n@@', '--numstat', '--reverse', '--no-merges', $range)
 109         {
 110                 # skip header line
 111                 line(*GIT, undef);
 112
 113                 while (1)
 114                 {
 115                         my $hash = line(GIT, '');
 116                         my $subject = line(GIT, '');
 117
 118                         last unless (length($subject) && $hash =~ m!^!);
 119
 120                         my $line = '';
 121                         my $body = '';
 122                         my @files;
 123                         my ($add, $del) = (0, 0);
 124
 125                         my $is_revert = $subject =~ m!^Revert !;
 126
 127                         $reverts{$hash}++ if $is_revert;
 128
 129                         while ($line ne '@@')
 130                         {
 131                                 $body .= length($line) ? "$line\n" : '';
 132                                 $line = line(*GIT, '@@');
 133
 134                                 if ($is_revert && $line =~ m!\b([0-9a-f]{40})\b!)
 135                                 {
 136                                         $reverts{$1}++;
 137                                 }
 138                         }
 139
 140                         $line = '';
 141
 142                         while ($line ne '@@')
 143                         {
 144                                 if ($line =~ m!^(\d+|-)\s+(\d+|-)\s+(.+)$!)
 145                                 {
 146                                         $add += ($1 eq '-') ? 0 : int($1);
 147                                         $del += ($2 eq '-') ? 0 : int($2);
 148                                         push @files, $3;
 149                                 }
 150
 151                                 $line = line(*GIT, '@@');
 152                         }
 153
 154                         my $commit = [
 155                                 $index++,
 156                                 $hash,
 157                                 $subject,
 158                                 $body,
 159                                 \@files,
 160                                 undef,
 161                                 undef,
 162                                 $add,
 163                                 $del
 164                         ];
 165
 166                         $total_add += $add;
 167                         $total_del += $del;
 168
 169                         $max_add = ($add > $max_add) ? $add : $max_add;
 170                         $max_del = ($del > $max_del) ? $del : $max_del;
 171
 172                         push @commits, $commit;
 173                 }
 174
 175                 close GIT;
 176         }
 177
 178         if (@commits > 0 && $commits[0][2] =~ /\brevert to branch defaults$/)
 179         {
 180                 shift @commits;
 181         }
 182
 183         return wantarray ? @commits : \@commits;
 184 }
 185
 186 sub fetch_subhistory($$$)
 187 {
 188         my ($url, $old, $new) = @_;
 189
 190         (my $path = $url) =~ s![^a-z0-9_-]+!-!g;
 191
 192         unless (-d "/tmp/repos/$path")
 193         {
 194                 mkdir('/tmp/repos');
 195                 system('git', 'clone', '--quiet', $url, "/tmp/repos/$path");
 196         }
 197         else
 198         {
 199                 system('git', "--work-tree=/tmp/repos/$path", "--git-dir=/tmp/repos/$path/.git", 'pull', '--quiet');
 200         }
 201
 202         return parse_history("/tmp/repos/$path", "$old..$new");
 203 }
 204
 205 sub requires_subhistory($$$)
 206 {
 207         my ($subject, $body, $hash) = @_;
 208
 209         foreach my $re (@subhistory_matches)
 210         {
 211                 if ($subject =~ $re || $body =~ $re)
 212                 {
 213                         if (open DIFF, '-|', 'git', 'diff', "$hash^!")
 214                         {
 215                                 my ($url, $old, $new);
 216
 217                                 while (defined(my $line = readline DIFF))
 218                                 {
 219                                         chomp $line;
 220
 221                                         if ($line =~ m!^[ +]PKG_SOURCE_URL\s*:?=\s*(\S+)!)
 222                                         {
 223                                                 $url = $1;
 224                                                 $url =~ s!\$\(LEDE_GIT\)!https://git.lede-project.org!g;
 225                                                 $url =~ s!\$\(OPENWRT_GIT\)!https://git.openwrt.org!g;
 226                                                 $url =~ s!\$\(PROJECT_GIT\)!https://git.openwrt.org!g;
 227                                         }
 228                                         elsif ($line =~ m!^-\S+\s*:?=\s*([a-f0-9]{40})\b!)
 229                                         {
 230                                                 $old = $1;
 231                                         }
 232                                         elsif ($line =~ m!^\+\S+\s*:?=\s*([a-f0-9]{40})\b!)
 233                                         {
 234                                                 $new = $1;
 235                                         }
 236
 237                                         if ($url && $old && $new)
 238                                         {
 239                                                 return ($url, $old, $new);
 240                                         }
 241                                 }
 242
 243                                 close DIFF;
 244                         }
 245                 }
 246         }
 247
 248         return ();
 249 }
 250
 251 sub find_weblink_template($)
 252 {
 253         my ($url) = @_;
 254
 255         foreach my $rt (@weblinks)
 256         {
 257                 my @m = $url =~ $rt->[0];
 258                 if (@m > 0)
 259                 {
 260                         return sprintf $rt->[1], @m;
 261                 }
 262         }
 263
 264         warn "No web link template for <$url>\n";
 265         return undef;
 266 }
 267
 268 sub format_stat($)
 269 {
 270         my ($commit) = @_;
 271
 272         my $s = '';
 273         my $c = '<color #ccc>%s</color>';
 274         my $g = '<color #282>%s</color>';
 275         my $r = '<color #f00>%s</color>';
 276
 277         if ($commit->[7] > 1000)
 278         {
 279                 $s .= sprintf $g, sprintf '+%.1fK', $commit->[7] / 1000;
 280         }
 281         elsif ($commit->[7] > 0)
 282         {
 283                 $s .= sprintf $g, sprintf '+%d', $commit->[7];
 284         }
 285
 286         if ($commit->[8] > 1000)
 287         {
 288                 $s .= $s ? sprintf($c, ',') : '';
 289                 $s .= sprintf $r, sprintf '-%.1fK', $commit->[8] / 1000;
 290         }
 291         elsif ($commit->[8] > 0)
 292         {
 293                 $s .= $s ? sprintf($c, ',') : '';
 294                 $s .= sprintf $r, sprintf '-%d', $commit->[8];
 295         }
 296
 297         return sprintf($c, '(') . $s . sprintf($c, ')');
 298 }
 299
 300 sub format_subject($$)
 301 {
 302         my ($subject, $body) = @_;
 303
 304         if (length($subject) > 80)
 305         {
 306                 $subject = substr($subject, 0, 77) . '...';
 307         }
 308
 309         $subject =~ s!^([^\s:]+):\s*!</nowiki>**<nowiki>$1:</nowiki>** <nowiki>!g;
 310
 311         $subject = sprintf '<nowiki>%s</nowiki>', $subject;
 312         $subject =~ s!<nowiki></nowiki>!!g;
 313
 314         return $subject;
 315 }
 316
 317 sub format_change($)
 318 {
 319         my ($change) = @_;
 320
 321         printf "''[[%s|%s]]'' %s //%s//\\\\\n",
 322                 sprintf($commit_url, $change->[1]),
 323                 substr($change->[1], 0, 7),
 324                 format_subject($change->[2], $change->[3]),
 325                 format_stat($change);
 326
 327         if ($change->[6])
 328         {
 329                 my $n = 0;
 330                 foreach my $subchange (@{$change->[6]})
 331                 {
 332                         if ($change->[5])
 333                         {
 334                                 printf " => ''[[%s|%s]]'' %s //%s//\\\\\n",
 335                                         sprintf($change->[5], $subchange->[1]),
 336                                         substr($subchange->[1], 0, 7),
 337                                         format_subject($subchange->[2], $subchange->[3]),
 338                                         format_stat($subchange);
 339                         }
 340                         else
 341                         {
 342                                 printf " => ''%s'' %s //%s//\\\\\n",
 343                                         substr($subchange->[1], 0, 7),
 344                                         format_subject($subchange->[2], $subchange->[3]),
 345                                         format_stat($subchange);
 346                         }
 347
 348                         if (++$n > 15 && @{$change->[6]} > $n)
 349                         {
 350                                 printf " => + //%u more...//\\\\\n", @{$change->[6]} - $n;
 351                                 last;
 352                         }
 353                 }
 354         }
 355 }
 356
 357 sub fetch_cve_info()
 358 {
 359         unless (-f '/tmp/cveinfo.csv')
 360         {
 361                 system('wget', '-O', '/tmp/cveinfo.csv.gz', 'https://cve.mitre.org/data/downloads/allitems.csv.gz') && return 0;
 362                 system('gunzip', '-f', '/tmp/cveinfo.csv.gz') && return 0;
 363         }
 364
 365         return 1;
 366 }
 367
 368 sub parse_cves(@)
 369 {
 370         my $csv = Text::CSV->new({ binary => 1 });
 371         my %cves;
 372
 373         if (fetch_cve_info() && $csv)
 374         {
 375                 if (open CVE, '<', '/tmp/cveinfo.csv')
 376                 {
 377                         while (defined(my $row = $csv->getline(*CVE)))
 378                         {
 379                                 foreach my $cve_id (@_)
 380                                 {
 381                                         if ($row->[0] eq $cve_id)
 382                                         {
 383                                                 $cves{$cve_id} = [$row->[2], $row->[6]];
 384                                                 last;
 385                                         }
 386                                 }
 387                         }
 388
 389                         close CVE;
 390                 }
 391         }
 392
 393         return \%cves;
 394 }
 395
 396 sub fetch_bug_info()
 397 {
 398         unless (-f '/tmp/buginfo.csv')
 399         {
 400                 system('wget', '-O', '/tmp/buginfo.csv', 'https://bugs.openwrt.org/index.php?string=&project=2&do=index&export_list=Export+Tasklist&advancedsearch=on&type%5B%5D=&sev%5B%5D=&pri%5B%5D=&due%5B%5D=&reported%5B%5D=&cat%5B%5D=&status%5B%5D=&percent%5B%5D=&opened=&dev=&closed=&duedatefrom=&duedateto=&changedfrom=&changedto=&openedfrom=&openedto=&closedfrom=&closedto=') && return 0;
 401         }
 402
 403         return 1;
 404 }
 405
 406 sub parse_bugs(@)
 407 {
 408         my $csv = Text::CSV->new({ binary => 1, allow_loose_quotes => 1, eol => "\012" });
 409         my %bugs;
 410
 411         if (fetch_bug_info() && $csv)
 412         {
 413                 if (open BUG, '<', '/tmp/buginfo.csv')
 414                 {
 415                         while (defined(my $row = $csv->getline(*BUG)))
 416                         {
 417                                 foreach my $bug_id (@_)
 418                                 {
 419                                         if ($row->[0] eq $bug_id)
 420                                         {
 421                                                 $bugs{$bug_id} = [$row->[4], $row->[5]];
 422                                                 last;
 423                                         }
 424                                 }
 425                         }
 426
 427                         $csv->error_diag;
 428
 429                         close BUG;
 430                 }
 431         }
 432
 433         return \%bugs;
 434 }
 435
 436
 437 my @commits = parse_history('.', $range);
 438 my (%bugs, %cves);
 439
 440 foreach my $commit (@commits)
 441 {
 442         my @topics = match_topics(@{$commit->[4]});
 443
 444         unless ($commit->[5])
 445         {
 446                 my ($su, $so, $sn) = requires_subhistory($commit->[2], $commit->[3], $commit->[1]);
 447                 if ($su) {
 448                         $commit->[5] = find_weblink_template($su);
 449                         $commit->[6] = fetch_subhistory($su, $so, $sn);
 450                 }
 451         }
 452
 453         foreach my $topic (@topics)
 454         {
 455                 $topics{$topic} ||= [ ];
 456                 push @{$topics{$topic}}, $commit;
 457         }
 458
 459         my (%bug_ids, %cve_ids);
 460
 461         foreach my $bug ($commit->[2] =~ m!([A-Z]*#\d+)\b!g,
 462                          $commit->[3] =~ m!([A-Z]*#\d+)\b!g)
 463         {
 464                 if ($bug =~ m!^(?:FS|GH|)#(\d+)$!)
 465                 {
 466                         $bug_ids{$1}++;
 467                 }
 468         }
 469
 470         foreach my $cve ($commit->[2] =~ m!\b(CVE-\d+-\d+|\d+-CVE-\d+)\b!g,
 471                          $commit->[3] =~ m!\b(CVE-\d+-\d+|\d+-CVE-\d+)\b!g)
 472         {
 473                 # fix misspelled CVE IDs
 474                 $cve =~ s!^(\d+)-CVE-!CVE-$1-!;
 475                 $cve_ids{$cve}++;
 476         }
 477
 478         foreach my $bug (keys %bug_ids)
 479         {
 480                 $bugs{$bug} ||= [ ];
 481                 push @{$bugs{$bug}}, $commit;
 482         }
 483
 484         foreach my $cve (keys %cve_ids)
 485         {
 486                 $cves{$cve} ||= [ ];
 487                 push @{$cves{$cve}}, $commit;
 488         }
 489 }
 490
 491
 492 my @topics = sort { (($a eq 'Miscellaneous') <=> ($b eq 'Miscellaneous')) || $a cmp $b } keys %topics;
 493
 494 foreach my $topic (@topics)
 495 {
 496         my @commits = grep { !$reverts{$_->[1]} } @{$topics{$topic}};
 497
 498         printf "==== %s (%d change%s) ====\n", $topic, 0 + @commits, @commits > 1 ? 's' : '';
 499
 500         foreach my $change (sort { $a->[0] <=> $b->[0] } @commits)
 501         {
 502                 format_change($change);
 503         }
 504
 505         print "\n";
 506 }
 507
 508 my @bugs = sort { int($a) <=> int($b) } keys %bugs;
 509 my $bug_info = parse_bugs(@bugs);
 510
 511 @bugs = grep { $bug_info->{$_} && $bug_info->{$_}[0] } @bugs;
 512
 513 if (@bugs > 0)
 514 {
 515         printf "===== Addressed bugs =====\n";
 516
 517         foreach my $bug (@bugs)
 518         {
 519                 printf "=== #%s ===\n", $bug;
 520                 printf "**Description:** <nowiki>%s</nowiki>\\\\\n", $bug_info->{$bug}[0];
 521                 printf "**Link:** [[https://bugs.openwrt.org/index.php?do=details&task_id=%s]]\\\\\n", $bug;
 522                 printf "**Commits:**\\\\\n";
 523
 524                 foreach my $commit (@{$bugs{$bug}})
 525                 {
 526                         format_change($commit);
 527                 }
 528
 529                 printf "\\\\\n";
 530         }
 531
 532         printf "\n";
 533 }
 534
 535 my @cves =
 536         map { $_->[1] }
 537         sort { ($a->[0] <=> $b->[0]) || ($a->[1] cmp $b->[1]) }
 538         map { $_ =~ m!^CVE-(\d+)-(\d+)$! ? [ $1 * 10000000 + $2, $_ ] : [ 0, $_ ] }
 539         keys %cves;
 540
 541 my $cve_info = parse_cves(@cves);
 542
 543 if (@cves > 0)
 544 {
 545         printf "===== Security fixes ====\n";
 546
 547         foreach my $cve (@cves)
 548         {
 549                 printf "=== %s ===\n", $cve;
 550
 551                 if ($cve_info->{$cve} && $cve_info->{$cve}[0])
 552                 {
 553                         printf "**Description:** <nowiki>%s</nowiki>\n\n", $cve_info->{$cve}[0];
 554                 }
 555
 556                 printf "**Link:** [[https://cve.mitre.org/cgi-bin/cvename.cgi?name=%s]]\\\\\n", $cve;
 557                 printf "**Commits:**\\\\\n";
 558
 559                 foreach my $commit (@{$cves{$cve}})
 560                 {
 561                         format_change($commit);
 562                 }
 563
 564                 printf "\\\\\n";
 565         }
 566
 567         printf "\n";
 568 }