make-changelog.pl

   1 #!/usr/bin/env perl
   2
   3 use strict;
   4 use warnings;
   5 use Text::CSV;
   6 use HTML::TreeBuilder;
   7
   8 my $range = $ARGV[0];
   9 our $workdir = './openwrt-changelog-data';
  10
  11 unless (defined $range) {
  12         printf STDERR "Usage: $0 range\n";
  13         exit 1;
  14 }
  15
  16 unless (-d $workdir) {
  17         unless (system('mkdir', '-p', $workdir) == 0) {
  18                 printf STDERR "Unable to create work directory!\n";
  19                 exit 1;
  20         }
  21 }
  22
  23 my %topics;
  24
  25 sub format_stat($)
  26 {
  27         my ($commit) = @_;
  28
  29         my $s = '';
  30         my $c = '<color #ccc>%s</color>';
  31         my $g = '<color #282>%s</color>';
  32         my $r = '<color #f00>%s</color>';
  33
  34         if ($commit->added > 1000)
  35         {
  36                 $s .= sprintf $g, sprintf '+%.1fK', $commit->added / 1000;
  37         }
  38         elsif ($commit->added > 0)
  39         {
  40                 $s .= sprintf $g, sprintf '+%d', $commit->added;
  41         }
  42
  43         if ($commit->deleted > 1000)
  44         {
  45                 $s .= $s ? sprintf($c, ',') : '';
  46                 $s .= sprintf $r, sprintf '-%.1fK', $commit->deleted / 1000;
  47         }
  48         elsif ($commit->deleted > 0)
  49         {
  50                 $s .= $s ? sprintf($c, ',') : '';
  51                 $s .= sprintf $r, sprintf '-%d', $commit->deleted;
  52         }
  53
  54         return sprintf($c, '(') . $s . sprintf($c, ')');
  55 }
  56
  57 sub format_subject($$)
  58 {
  59         my ($subject, $body) = @_;
  60
  61         if (length($subject) > 80)
  62         {
  63                 $subject = substr($subject, 0, 77) . '...';
  64         }
  65
  66         $subject =~ s!^([^\s:]+):\s*!</nowiki>**<nowiki>$1:</nowiki>** <nowiki>!g;
  67
  68         $subject = sprintf '<nowiki>%s</nowiki>', $subject;
  69         $subject =~ s!<nowiki></nowiki>!!g;
  70
  71         return $subject;
  72 }
  73
  74 sub format_change($)
  75 {
  76         my ($change) = @_;
  77
  78         printf "''[[%s|%s]]'' %s //%s//\\\\\n",
  79                 sprintf($change->repository->commit_link_template, $change->sha1),
  80                 substr($change->sha1, 0, 7),
  81                 format_subject($change->subject, $change->body),
  82                 format_stat($change);
  83
  84         my @subhistory = $change->subhistory;
  85
  86         if (@subhistory > 0) {
  87                 my $n = 0;
  88                 my $link_tpl;
  89
  90                 foreach my $subchange (@subhistory) {
  91                         if ($n == 0) {
  92                                 $link_tpl = $subchange->repository->commit_link_template;
  93                         }
  94
  95                         if ($link_tpl) {
  96                                 printf " => ''[[%s|%s]]'' %s //%s//\\\\\n",
  97                                         sprintf($link_tpl, $subchange->sha1),
  98                                         substr($subchange->sha1, 0, 7),
  99                                         format_subject($subchange->subject, $subchange->body),
 100                                         format_stat($subchange);
 101                         }
 102                         else {
 103                                 printf " => ''%s'' %s //%s//\\\\\n",
 104                                         substr($subchange->sha1, 0, 7),
 105                                         format_subject($subchange->subject, $subchange->body),
 106                                         format_stat($subchange);
 107                         }
 108
 109                         if (++$n > 15 && @subhistory > $n) {
 110                                 printf " => + //%u more...//\\\\\n", @subhistory - $n;
 111                                 last;
 112                         }
 113                 }
 114         }
 115 }
 116
 117 sub fetch_cve_info()
 118 {
 119         unless (-f "$workdir/cveinfo.csv")
 120         {
 121                 system('wget', '-O', "$workdir/cveinfo.csv.gz", 'https://cve.mitre.org/data/downloads/allitems.csv.gz') && return 0;
 122                 system('gunzip', '-f', "$workdir/cveinfo.csv.gz") && return 0;
 123         }
 124
 125         return 1;
 126 }
 127
 128 sub parse_cves(@)
 129 {
 130         my $csv = Text::CSV->new({ binary => 1 });
 131         my %cves;
 132
 133         if (fetch_cve_info() && $csv)
 134         {
 135                 if (open CVE, '<', "$workdir/cveinfo.csv")
 136                 {
 137                         while (defined(my $row = $csv->getline(*CVE)))
 138                         {
 139                                 foreach my $cve_id (@_)
 140                                 {
 141                                         if ($row->[0] eq $cve_id)
 142                                         {
 143                                                 $cves{$cve_id} = [$row->[2], $row->[6]];
 144                                                 last;
 145                                         }
 146                                 }
 147                         }
 148
 149                         close CVE;
 150                 }
 151         }
 152
 153         return \%cves;
 154 }
 155
 156
 157 my $repository = Repository->new('https://git.openwrt.org/openwrt/openwrt.git');
 158 my $bugtracker = BugTracker->new;
 159
 160 my @commits = $repository->parse_history($range);
 161 my (%bugs, %cves, %sha1s);
 162
 163 foreach my $commit (@commits)
 164 {
 165         if ($commit->subject =~ m!\b(?:LEDE|OpenWrt) v\d\d\.\d\d\.\d+(?:-rc\d+)?: (?:adjust config|revert to branch) defaults\b!) {
 166                 Log::info("Skipping maintenance commit %s (%s)", $commit->sha1, $commit->subject);
 167                 next;
 168         }
 169
 170         my @topics = $commit->topics;
 171
 172         foreach my $topic (@topics)
 173         {
 174                 $topics{$topic} ||= [ ];
 175                 push @{$topics{$topic}}, $commit;
 176         }
 177
 178         foreach my $bug ($commit->bugs) {
 179                 if ($bug->status ne 'closed') {
 180                         Log::warn("Commit %s closes bug #%d", $commit->sha1, $bug->id);
 181                 }
 182
 183                 $bugs{ $bug->id } ||= [ ];
 184                 push @{$bugs{ $bug->id }}, $commit;
 185         }
 186
 187         foreach my $cve_id ($commit->cve_ids) {
 188                 $cves{$cve_id} ||= [ ];
 189                 push @{$cves{$cve_id}}, $commit;
 190         }
 191
 192         $sha1s{$commit->[1]}++;
 193 }
 194
 195 Log::info("Finding commit references in bugs...");
 196
 197 foreach my $bug ($bugtracker->bugs)
 198 {
 199         next if exists $bugs{ $bug->id };
 200
 201         foreach my $hash ($bug->refs) {
 202                 my $commit = $repository->find_commit($hash);
 203                 next unless defined $commit;
 204
 205                 if ($bug->status ne 'closed') {
 206                         Log::warn("Bug #%d closed by commit %s", $bug->id, $commit->sha1);
 207                 }
 208
 209                 $bugs{ $bug->id } ||= [ ];
 210                 push @{$bugs{ $bug->id }}, $commit;
 211         }
 212 }
 213
 214
 215 my @topics = sort { (($a eq 'Miscellaneous') <=> ($b eq 'Miscellaneous')) || $a cmp $b } keys %topics;
 216
 217 foreach my $topic (@topics)
 218 {
 219         my @commits = @{$topics{$topic}};
 220
 221         printf "==== %s (%d change%s) ====\n", $topic, 0 + @commits, @commits > 1 ? 's' : '';
 222
 223         foreach my $change (sort { $a->pos <=> $b->pos } @commits)
 224         {
 225                 format_change($change);
 226         }
 227
 228         print "\n";
 229 }
 230
 231 my @bugs = map { $bugtracker->get($_) } sort { int($a) <=> int($b) } keys %bugs;
 232
 233 if (@bugs > 0) {
 234         printf "===== Addressed bugs =====\n";
 235
 236         foreach my $bug (@bugs)
 237         {
 238                 printf "=== #%d ===\n", $bug->id;
 239                 printf "**Description:** <nowiki>%s</nowiki>\\\\\n", $bug->summary;
 240                 printf "**Link:** [[https://bugs.openwrt.org/index.php?do=details&task_id=%d]]\\\\\n", $bug->id;
 241                 printf "**Commits:**\\\\\n";
 242
 243                 foreach my $commit (@{$bugs{ $bug->id }})
 244                 {
 245                         format_change($commit);
 246                 }
 247
 248                 printf "\\\\\n";
 249         }
 250
 251         printf "\n";
 252 }
 253
 254 my @cves =
 255         map { $_->[1] }
 256         sort { ($a->[0] <=> $b->[0]) || ($a->[1] cmp $b->[1]) }
 257         map { $_ =~ m!^CVE-(\d+)-(\d+)$! ? [ $1 * 10000000 + $2, $_ ] : [ 0, $_ ] }
 258         keys %cves;
 259
 260 my $cve_info = parse_cves(@cves);
 261
 262 if (@cves > 0)
 263 {
 264         printf "===== Security fixes ====\n";
 265
 266         foreach my $cve (@cves)
 267         {
 268                 printf "=== %s ===\n", $cve;
 269
 270                 if ($cve_info->{$cve} && $cve_info->{$cve}[0])
 271                 {
 272                         printf "**Description:** <nowiki>%s</nowiki>\n\n", $cve_info->{$cve}[0];
 273                 }
 274
 275                 printf "**Link:** [[https://cve.mitre.org/cgi-bin/cvename.cgi?name=%s]]\\\\\n", $cve;
 276                 printf "**Commits:**\\\\\n";
 277
 278                 foreach my $commit (@{$cves{$cve}})
 279                 {
 280                         format_change($commit);
 281                 }
 282
 283                 printf "\\\\\n";
 284         }
 285
 286         printf "\n";
 287 }
 288
 289
 290 package Log;
 291
 292 sub info {
 293         my ($fmt, @args) = @_;
 294         printf STDERR "[I] %s\n", sprintf $fmt, @args;
 295         return 0;
 296 }
 297
 298 sub warn {
 299         my ($fmt, @args) = @_;
 300         printf STDERR "[W] %s\n", sprintf $fmt, @args;
 301         return 1;
 302 }
 303
 304 sub err {
 305         my ($fmt, @args) = @_;
 306         printf STDERR "[E] %s\n", sprintf $fmt, @args;
 307         return 1;
 308 }
 309
 310
 311 package BugTracker;
 312
 313 our $inst;
 314
 315 sub _date {
 316         my ($self, $ts) = @_;
 317         my @loc = gmtime $ts;
 318         return sprintf '%04d-%02d-%02d', $loc[5] + 1900, $loc[4] + 1, $loc[3];
 319 }
 320
 321 sub _fetch {
 322         my ($self) = @_;
 323
 324         return 0 if $self->{'fetched'};
 325
 326         my @stat = stat "$main::workdir/buginfo.csv";
 327         my $since = defined($stat[9]) ? $stat[9] : 86400; $since -= ($since % 86400);
 328         my $sdate = $self->_date($since - 86400);
 329
 330         Log::info("Updating bug database...");
 331
 332         if (system('wget', '-qO', "$main::workdir/buginfo-delta.csv",
 333                    "https://bugs.openwrt.org/index.php?string=&project=2&do=index&export_list=Export+Tasklist&advancedsearch=on&type%5B%5D=&sev%5B%5D=&pri%5B%5D=&due%5B%5D=&reported%5B%5D=&cat%5B%5D=&status%5B%5D=&percent%5B%5D=&opened=&dev=&closed=&duedatefrom=&duedateto=&changedfrom=$sdate&changedto=&openedfrom=&openedto=&closedfrom=&closedto=")) {
 334                 return Log::err('Unable to fetch database changes!');
 335         }
 336
 337         $self->_update($since);
 338
 339         $self->{'fetched'}++;
 340
 341         return 0;
 342 }
 343
 344 sub _update {
 345         my ($self) = @_;
 346         my %records;
 347
 348         my $csv = Text::CSV->new({
 349                 'binary' => 1,
 350                 'allow_loose_quotes' => 1
 351         });
 352
 353         if (open my $file, '<', "$main::workdir/buginfo.csv") {
 354                 while (defined(my $row = $csv->getline($file))) {
 355                         next if $row->[0] eq 'ID';
 356                         $row->[13] = 0 unless defined $row->[13];
 357                         $records{$row->[0]} = $row;
 358                 }
 359
 360                 close $file;
 361         }
 362
 363         if (open my $file, '<', "$main::workdir/buginfo-delta.csv") {
 364                 my $changed = 0;
 365                 my $now = time();
 366
 367                 while (defined(my $row = $csv->getline($file))) {
 368                         next if $row->[0] eq 'ID';
 369                         $changed++;
 370                         $row->[13] = $now;
 371                         $records{$row->[0]} = $row;
 372                 }
 373
 374                 close $file;
 375
 376                 if ($changed) {
 377                         if (open $file, '>:utf8', "$main::workdir/buginfo.csv") {
 378                                 foreach my $id (sort { $a <=> $b } keys %records) {
 379                                         $csv->print($file, $records{$id});
 380                                         print $file "\n";
 381                                 }
 382                                 close $file;
 383                         }
 384
 385                         if (!utime($now, $now, "$main::workdir/buginfo.csv")) {
 386                                 Log::warn("Unable to change modification time: $!");
 387                         }
 388
 389                         Log::info("Found %d updated bugs", $changed);
 390                 }
 391         }
 392 }
 393
 394 sub _parse {
 395         my ($self) = @_;
 396
 397         return 0 if $self->{'bugs'};
 398         return 1 if $self->_fetch;
 399
 400         $self->{'bugs'} = { };
 401
 402         my $csv = Text::CSV->new({
 403                 'binary' => 1,
 404                 'allow_loose_quotes' => 1
 405         });
 406
 407         if (open my $file, '<', "$main::workdir/buginfo.csv") {
 408                 while (defined(my $row = $csv->getline($file))) {
 409                         next if $row->[0] eq 'ID';
 410
 411                         my ($date_opened, $date_closed, $date_modified) = (0, 0, 0);
 412
 413                         if (defined($row->[7]) && $row->[7] =~ m!^(\d+)$!) {
 414                                 $date_opened = int($1);
 415                         }
 416
 417                         if (defined($row->[8]) && $row->[8] =~ m!^(\d+)$!) {
 418                                 $date_closed = int($1);
 419                         }
 420
 421                         if (defined($row->[13]) && $row->[13] =~ m!^(\d+)$!) {
 422                                 $date_modified = int($1);
 423                         }
 424
 425                         my $bug = Bug->new(
 426                                 $row->[0],
 427                                 $row->[4],
 428                                 lc(($date_closed > $date_opened) ? 'Closed' : $row->[5]),
 429                                 $date_opened,
 430                                 $date_closed,
 431                                 $date_modified
 432                         );
 433
 434                         $self->{'bugs'}{ $bug->id } = $bug;
 435                 }
 436
 437                 close $file;
 438         }
 439
 440         return 0;
 441 }
 442
 443 sub new {
 444         my ($pack) = @_;
 445
 446         unless ($inst) {
 447                 $inst = bless {}, $pack;
 448         }
 449
 450         return $inst;
 451 }
 452
 453 sub get($$) {
 454         my ($self, $id) = @_;
 455
 456         return undef if $self->_parse;
 457         return $self->{'bugs'}{$id};
 458 }
 459
 460 sub bugs($) {
 461         my ($self) = @_;
 462         return undef if $self->_parse;
 463
 464         my @bugs = map { $self->{'bugs'}{$_} } sort { $a <=> $b } keys %{$self->{'bugs'}};
 465         return wantarray ? @bugs : \@bugs;
 466 }
 467
 468
 469 package Bug;
 470
 471 use File::Basename;
 472 use constant {
 473         '_ID'     => 0,
 474         '_SUM'    => 1,
 475         '_STAT'   => 2,
 476         '_OPEN'   => 3,
 477         '_CLOSE'  => 4,
 478         '_CHANGE' => 5,
 479         '_REFS'   => 6
 480 };
 481
 482 sub new
 483 {
 484         my ($pack, $id, $summary, $status, $opened, $closed, $modified) = @_;
 485         return bless [
 486                 $id,
 487                 $summary,
 488                 $status,
 489                 $opened,
 490                 $closed,
 491                 $modified
 492         ], $pack;
 493 }
 494
 495 sub id { shift->[_ID] }
 496 sub url { sprintf 'https://bugs.openwrt.org/index.php?do=details&task_id=%d', shift->id }
 497 sub file { sprintf '%s/ticket/%d.html', $main::workdir, shift->id }
 498 sub summary { shift->[_SUM] }
 499 sub status { shift->[_STAT] }
 500
 501 sub _fetch()
 502 {
 503         my ($self) = @_;
 504         my @stat = stat $self->file;
 505
 506         if (defined($stat[9]) && ($stat[9] >= $self->[_CHANGE])) {
 507                 return 0;
 508         }
 509
 510         Log::info("Fetching details for Bug #%d ...", $self->id);
 511
 512         if (system('mkdir', '-p', File::Basename::dirname($self->file))) {
 513                 return Log::err("Unable to create directory!");
 514         }
 515         elsif (system('wget', '-q', '-O', $self->file, $self->url)) {
 516                 return Log::err("Unable to fetch bug details!");
 517         }
 518         elsif (!utime($self->[_CHANGE], $self->[_CHANGE], $self->file)) {
 519                 return LOG::warn("Unable to change modification time: $!");
 520         }
 521
 522         return 0;
 523 }
 524
 525 sub _find_commit_references()
 526 {
 527         my ($self) = @_;
 528
 529         return undef if $self->_fetch;
 530
 531         eval {
 532                 my $tree = HTML::TreeBuilder->new_from_file($self->file);
 533
 534                 my $closed = $tree->look_down('id' => 'taskclosed');
 535                 if ($closed) {
 536                         my $str = $closed->as_HTML;
 537                         if ($str =~ m!<strong>Reason for closing:</strong>[^\n]+\bFixed\b!) {
 538                                 $str =~ s!\n!!g;
 539                                 $str =~ s!<! <!g;
 540
 541                                 my @refs = $str =~ m!\b (
 542                                         https?://git\.(?:openwrt|lede-project)\.org/\?p=[\w/]+\.git\S*;h=[a-fA-F0-9]{4,40} |
 543                                         https?://git\.(?:openwrt|lede-project)\.org/[a-fA-F0-9]{4,40} |
 544                                         https?://github\.com/[^/]+/commit/[a-fA-F0-9]{4,40} |
 545                                         [a-fA-F0-9]{7,40}
 546                                 ) \b!x;
 547
 548                                 return @refs if @refs > 0;
 549                         }
 550                 }
 551
 552                 foreach my $comment (reverse $tree->look_down('class' => 'commenttext')) {
 553                         my $str = $comment->as_HTML;
 554                         my @refs = $str =~ m!
 555                                 (?:
 556                                         Fixed \s+ with \s+ |
 557                                         Fixed \s+ in \s+ |
 558                                         fix \s+ (?: in | into ) \s+ (?: \w+ \s+ )*
 559                                 )
 560                                 (?: <a \s+ href=" )?  # "
 561                                 \b (
 562                                         https?://git\.(?:openwrt|lede-project)\.org/\?p=[\w/]+\.git\S*;h=[a-fA-F0-9]{4,40} |
 563                                         https?://git\.(?:openwrt|lede-project)\.org/[a-fA-F0-9]{4,40} |
 564                                         https?://github\.com/[^/]+/commit/[a-fA-F0-9]{4,40} |
 565                                         [a-fA-F0-9]{7,40}
 566                                 ) \b
 567                         !ixg;
 568
 569                         return @refs if @refs > 0;
 570                 }
 571         };
 572 }
 573
 574 sub refs ($) {
 575         my ($self) = @_;
 576
 577         unless (defined $self->[_REFS]) {
 578                 my %sha1;
 579
 580                 foreach my $ref ($self->_find_commit_references) {
 581                         if ($ref =~ m!\b([a-fA-F0-9]{4,40})$!) {
 582                                 $sha1{lc $1}++;
 583                         }
 584                 }
 585
 586                 $self->[_REFS] = [ sort keys %sha1 ];
 587         }
 588
 589         return wantarray ? @{$self->[_REFS]} : $self->[_REFS];
 590 }
 591
 592
 593 package Repository;
 594
 595 use File::Basename;
 596
 597 our %repositories;
 598 our %commits;
 599 our @index;
 600
 601 sub new($$) {
 602         my ($pack, $url) = @_;
 603
 604         my $id = $url;
 605            $id =~ s!\bgit\.lede-project\.org\b!git.openwrt.org!;
 606            $id =~ s![^a-z0-9_-]+!-!g;
 607
 608         unless (exists $repositories{$id}) {
 609                 $repositories{$id} = bless {
 610                         'id' => $id,
 611                         'url' => $url,
 612                         'cache' => { }
 613                 }, $pack;
 614
 615                 $repositories{$id}->_fetch;
 616         }
 617
 618         return $repositories{$id};
 619 }
 620
 621 sub id { shift->{'id'} }
 622 sub url { shift->{'url'} }
 623 sub directory { sprintf '%s/repos/%s', $main::workdir, shift->id }
 624
 625 sub _fetch($) {
 626         my ($self) = @_;
 627
 628         if (-d $self->directory) {
 629                 Log::info("Updating repository %s ...", $self->url);
 630
 631                 my $tree = $self->directory;
 632                 my $git  = $tree . '/.git';
 633
 634                 if (system('git', "--work-tree=$tree", "--git-dir=$git", 'fetch', '--all', '--quiet')) {
 635                         return Log::err("Unable to pull repository!");
 636                 }
 637
 638                 return 0;
 639         }
 640
 641         Log::info("Cloning repository %s ...", $self->url);
 642
 643         if (system('mkdir', '-p', $self->directory)) {
 644                 return Log::err("Unable to create directory!");
 645         }
 646         elsif (system('git', 'clone', '--quiet', $self->url, $self->directory)) {
 647                 return Log::err("Unable to clone repository!");
 648         }
 649
 650         return 0;
 651 }
 652
 653 sub _readline($*$) {
 654         my ($self, $fh, $default) = @_;
 655
 656         my $line = readline $fh;
 657
 658         if (defined $line)
 659         {
 660                 chomp $line;
 661                 return $line;
 662         }
 663
 664         return $default;
 665 }
 666
 667 sub _parse($*)
 668 {
 669         my ($self, $fh) = @_;
 670         my @commits;
 671         my $num = 0;
 672
 673         # skip header line
 674         $self->_readline($fh, undef);
 675
 676         while (1) {
 677                 my $hash = $self->_readline($fh, '');
 678                 my $subject = $self->_readline($fh, '');
 679
 680                 last unless (length($subject) && $hash =~ m!^[a-f0-9]{40}$!);
 681
 682                 my $line = '';
 683
 684                 # commit already cached, skip lines and use cached object
 685                 if (exists $Repository::commits{$hash}) {
 686                         for ($line = ''; $line ne '@@'; $line = $self->_readline($fh, '@@')) { next; }
 687                         for ($line = ''; $line ne '@@'; $line = $self->_readline($fh, '@@')) { next; }
 688
 689                         push @commits, $Repository::commits{$hash};
 690                         next;
 691                 }
 692
 693                 my $body = '';
 694                 my @files;
 695                 my ($add, $del) = (0, 0);
 696
 697                 while ($line ne '@@') {
 698                         $body .= length($line) ? "$line\n" : '';
 699                         $line = $self->_readline($fh, '@@');
 700                 }
 701
 702                 $line = '';
 703
 704                 my $reading_diff = 0;
 705                 my ($subhistory, $subhistory_url, $subhistory_start, $subhistory_end);
 706
 707                 while ($line ne '@@') {
 708                         if ($line =~ m!^diff --git a/!) {
 709                                 $reading_diff = 1;
 710                                 undef $subhistory_url;
 711                                 undef $subhistory_start;
 712                                 undef $subhistory_end;
 713                         }
 714                         elsif ($reading_diff) {
 715                                 if ($line =~ m!^[ +]PKG_SOURCE_URL\s*:?=\s*(\S+)!) {
 716                                         $subhistory_url = $1;
 717                                         $subhistory_url =~ s!\$\(LEDE_GIT\)!https://git.lede-project.org!g;
 718                                         $subhistory_url =~ s!\$\(OPENWRT_GIT\)!https://git.openwrt.org!g;
 719                                         $subhistory_url =~ s!\$\(PROJECT_GIT\)!https://git.openwrt.org!g;
 720                                 }
 721                                 elsif ($line =~ m!^-\S+\s*:?=\s*([a-f0-9]{40})\b!) {
 722                                         $subhistory_start = $1;
 723                                 }
 724                                 elsif ($line =~ m!^\+\S+\s*:?=\s*([a-f0-9]{40})\b!) {
 725                                         $subhistory_end = $1;
 726
 727                                         if ($subhistory_url && $subhistory_start && $subhistory_end) {
 728                                                 $subhistory = Repository->new($subhistory_url)->parse_history("$subhistory_start..$subhistory_end");
 729                                         }
 730                                 }
 731                         }
 732                         elsif ($line =~ m!^(\d+|-)\s+(\d+|-)\s+(.+)$!) {
 733                                 $add += ($1 eq '-') ? 0 : int($1);
 734                                 $del += ($2 eq '-') ? 0 : int($2);
 735                                 push @files, $3;
 736                         }
 737
 738                         $line = $self->_readline($fh, '@@');
 739                 }
 740
 741                 my $commit = Commit->new($self, $num++, $hash, $subject, $body, $add, $del, $subhistory, @files);
 742
 743                 push @commits, $commit;
 744                 push @Repository::index, $commit;
 745
 746                 $Repository::commits{ $commit->sha1 } = $commit;
 747         }
 748
 749         @Repository::index = sort { $a->sha1 cmp $b->sha1 } @Repository::index;
 750
 751         return wantarray ? @commits : \@commits;
 752 }
 753
 754 sub parse_history($$) {
 755         my ($self, $range) = @_;
 756         my $gitdir = sprintf '%s/.git', $self->directory;
 757         my @commits;
 758
 759         if (open my $git, '-|', 'git', "--git-dir=$gitdir", 'log', '-p', '--format=@@%n%H%n%s%n%b%n@@', '--numstat', '--reverse', '--no-merges', $range) {
 760                 @commits = $self->_parse($git);
 761                 close $git;
 762         }
 763
 764         return wantarray ? @commits : \@commits;
 765 }
 766
 767 sub find_commit($$) {
 768         my ($self, $hash) = @_;
 769
 770         if (exists $Repository::commits{$hash}) {
 771                 return $Repository::commits{$hash};
 772         }
 773         else {
 774                 my ($l, $r) = (0, @Repository::index - 1);
 775
 776                 while ($l <= $r) {
 777                         my $m = $l + int(($r - $l) / 2);
 778
 779                         if (index($Repository::index[$m]->sha1, $hash) == 0) {
 780                                 return $Repository::index[$m];
 781                         }
 782                         elsif ($Repository::index[$m]->sha1 gt $hash) {
 783                                 $r = $m - 1;
 784                         }
 785                         else {
 786                                 $l = $m + 1;
 787                         }
 788                 }
 789         }
 790
 791         return undef;
 792 }
 793
 794 sub _weblinks { (
 795         [ qr'^[^:]+://(git.lede-project.org/)(.+)$' => 'https://%s?p=%s;a=commitdiff;h=%%s' ],
 796         [ qr'^[^:]+://(git.openwrt.org/)(.+)$'      => 'https://%s?p=%s;a=commitdiff;h=%%s' ],
 797         [ qr'^[^:]+://(github.com/.+?)(?:\.git)?$'  => 'https://%s/commit/%%s' ],
 798         [ qr'^[^:]+://git.kernel.org/pub/scm/(.+)$' => 'https://git.kernel.org/cgit/%s/commit/?id=%%s' ],
 799         [ qr'^[^:]+://w1.fi/(?:.+/)?(.+)\.git$'     => 'https://w1.fi/cgit/%s/commit/?id=%%s' ],
 800         [ qr'^[^:]+://git.netfilter.org/(.+)'       => 'https://git.netfilter.org/%s/commit/?id=%%s' ],
 801         [ qr'^[^:]+://git.musl-libc.org/(.+)'       => 'https://git.musl-libc.org/cgit/%s/commit/?id=%%s' ],
 802         [ qr'^[^:]+://git.zx2c4.com/(.+)'           => 'https://git.zx2c4.com/%s/commit/?id=%%s' ],
 803         [ qr'^[^:]+://sourceware.org/git/(.+)'      => 'https://sourceware.org/git/?p=%s;a=commitdiff;h=%%s' ]
 804 ) }
 805
 806 sub commit_link_template($) {
 807         my ($self) = @_;
 808
 809         foreach my $lnk ($self->_weblinks) {
 810                 my @matches = $self->url =~ $lnk->[0];
 811                 if (@matches > 0) {
 812                         return sprintf $lnk->[1], @matches;
 813                 }
 814         }
 815
 816         Log::warn("No web link template available for %s", $self->url);
 817         return undef;
 818 }
 819
 820 sub log($) {
 821         my ($self) = @_;
 822         return wantarray ? @{$self->{'log'}} : $self->{'log'};
 823 }
 824
 825
 826 package Commit;
 827
 828 use constant {
 829         '_REPO'  => 0,
 830         '_POS'   => 1,
 831         '_SHA1'  => 2,
 832         '_SUBJ'  => 3,
 833         '_BODY'  => 4,
 834         '_FILES' => 5,
 835         '_SHIST' => 6,
 836         '_NADD'  => 7,
 837         '_NDEL'  => 8
 838 };
 839
 840 sub _topic_map { (
 841         [ qr'^package/(kernel)/linux',                 'Kernel' ],
 842         [ qr'^(target/linux/generic|include/kernel-version.mk)', 'Kernel' ],
 843         [ qr'^package/kernel/(mac80211)',              'Wireless / Common' ],
 844         [ qr'^package/kernel/(ath10k-ct)',             'Wireless / Ath10k CT' ],
 845         [ qr'^package/kernel/(mt76)',                  'Wireless / MT76' ],
 846         [ qr'^package/kernel/(mwlwifi)',               'Wireless / Mwlwifi' ],
 847         [ qr'^package/(base-files)/',                  'Packages / OpenWrt base files' ],
 848         [ qr'^package/(boot)/',                        'Packages / Boot Loaders' ],
 849         [ qr'^package/firmware/',                      'Packages / Firmware' ],
 850         [ qr'^package/.+/(uhttpd|usbmode|jsonfilter|ugps|libubox|procd|mountd|ubus|uci|usign|rpcd|fstools|ubox)/', 'Packages / OpenWrt system userland' ],
 851         [ qr'^package/.+/(iwinfo|umbim|uqmi|relayd|mdns|firewall|netifd|uclient|ustream-ssl|gre|ipip|qos-scripts|swconfig|vti|6in4|6rd|6to4|ds-lite|map|odhcp6c|odhcpd)/', 'Packages / OpenWrt network userland' ],
 852         [ qr'^package/[^/]+/([^/]+)',                  'Packages / Common' ],
 853         [ qr'^target/sdk/',                            'Build System / SDK' ],
 854         [ qr'^target/imagebuilder/',                   'Build System / Image Builder' ],
 855         [ qr'^target/toolchain/',                      'Build System / Toolchain' ],
 856         [ qr'^target/linux/([^/]+)',                   'Target / $1' ],
 857         [ qr'^(tools)/[^/]+',                          'Build System / Host Utilities' ],
 858         [ qr'^(toolchain)/[^/]+',                      'Build System / Toolchain' ],
 859         [ qr'^(config/|include/|scripts/|target/[^/]+$|Makefile|rules\.mk)', 'Build System / Buildroot' ],
 860         [ qr'^(feeds)\b',                              'Build System / Feeds' ],
 861 ) }
 862
 863 sub new ($$$$$$$$@) {
 864         my ($pack, $repo, $pos, $hash, $subject, $body, $add, $del, $shist, @files) = @_;
 865         my @commit;
 866
 867         $commit[_REPO] = $repo;
 868         $commit[_POS]  = $pos;
 869         $commit[_SHA1] = $hash;
 870         $commit[_SUBJ] = $subject;
 871         $commit[_BODY] = $body;
 872         $commit[_NADD] = $add;
 873         $commit[_NDEL] = $del;
 874         $commit[_SHIST] = $shist;
 875         $commit[_FILES] = \@files;
 876
 877         return bless \@commit, $pack;
 878 }
 879
 880 sub repository { shift->[_REPO] }
 881 sub pos { shift->[_POS] }
 882 sub sha1 { shift->[_SHA1] }
 883 sub subject { shift->[_SUBJ] }
 884 sub body { shift->[_BODY] }
 885 sub added { shift->[_NADD] }
 886 sub deleted { shift->[_NDEL] }
 887 sub files { wantarray ? @{shift->[_FILES] || []} : shift->[_FILES] }
 888 sub subhistory { wantarray ? @{shift->[_SHIST] || []} : shift->[_SHIST] }
 889
 890 sub topics($) {
 891         my ($self) = @_;
 892         my %topics;
 893         my %paths;
 894
 895         foreach my $path ($self->files)
 896         {
 897                 if ($path =~ m!^(.+)/\{(.+?) => (.+?)\}$!)
 898                 {
 899                         $paths{"$1/$2"}++;
 900                         $paths{"$1/$3"}++;
 901                 }
 902                 else
 903                 {
 904                         $paths{$path}++;
 905                 }
 906         }
 907
 908         foreach my $path (sort keys %paths)
 909         {
 910                 foreach my $rs ($self->_topic_map)
 911                 {
 912                         if ($path =~ $rs->[0])
 913                         {
 914                                 my $m = $1;
 915                                 my $s = $rs->[1];
 916
 917                                 $s =~ s!\$1!$m!g;
 918                                 $topics{$s}++;
 919
 920                                 last;
 921                         }
 922                 }
 923         }
 924
 925         my @topics = sort keys %topics;
 926         return (@topics > 0 ? @topics : ('Miscellaneous'));
 927 }
 928
 929 sub bugs($) {
 930         my ($self) = @_;
 931
 932         my $bugtracker = BugTracker->new;
 933         my $candidates = qr'\b((?:[Pp]ull [Rr]equest |[Bb]ug |[Ii]ssue |PR |FS |GH |PR|FS|GH)#\d+)\b';
 934         my $issue = qr'(?i)^(?:Bug |Issue |FS |GH |FS|GH)#(\d+)$';
 935         my %bugs;
 936
 937         foreach my $match ($self->subject =~ /$candidates/g, $self->body =~ /$candidates/g) {
 938                 if ($match =~ $issue) {
 939                         my $bug = $bugtracker->get($1);
 940                         if ($bug) {
 941                                 $bugs{ $bug->id } = $bug;
 942                         }
 943                 }
 944         }
 945
 946         return map { $bugs{$_} } sort { $a <=> $b } keys %bugs;
 947 }
 948
 949 sub cve_ids($) {
 950         my ($self) = @_;
 951         my $candidates = qr'\b(CVE-\d+-\d+|\d+-CVE-\d+)\b';
 952         my %cves;
 953
 954         foreach my $match ($self->subject =~ /$candidates/g, $self->body =~ /$candidates/g) {
 955                 # fix misspelled CVE IDs
 956                 $match =~ s!^(\d+)-CVE-!CVE-$1-!;
 957                 $cves{$match}++;
 958         }
 959
 960         return sort { $a cmp $b } keys %cves;
 961 }