make-changelog.pl

   1 #!/usr/bin/env perl
   2
   3 use strict;
   4 use warnings;
   5 use JSON;
   6 use Time::Local;
   7 use Text::CSV;
   8
   9 my $range = $ARGV[0];
  10 our $workdir = './openwrt-changelog-data';
  11
  12 unless (defined $range) {
  13         printf STDERR "Usage: $0 range\n";
  14         exit 1;
  15 }
  16
  17 unless (-d $workdir) {
  18         unless (system('mkdir', '-p', $workdir) == 0) {
  19                 printf STDERR "Unable to create work directory!\n";
  20                 exit 1;
  21         }
  22 }
  23
  24 my %topics;
  25
  26 sub format_stat($)
  27 {
  28         my ($commit) = @_;
  29
  30         my $s = '';
  31         my $c = '<color #ccc>%s</color>';
  32         my $g = '<color #282>%s</color>';
  33         my $r = '<color #f00>%s</color>';
  34
  35         if ($commit->added > 1000)
  36         {
  37                 $s .= sprintf $g, sprintf '+%.1fK', $commit->added / 1000;
  38         }
  39         elsif ($commit->added > 0)
  40         {
  41                 $s .= sprintf $g, sprintf '+%d', $commit->added;
  42         }
  43
  44         if ($commit->deleted > 1000)
  45         {
  46                 $s .= $s ? sprintf($c, ',') : '';
  47                 $s .= sprintf $r, sprintf '-%.1fK', $commit->deleted / 1000;
  48         }
  49         elsif ($commit->deleted > 0)
  50         {
  51                 $s .= $s ? sprintf($c, ',') : '';
  52                 $s .= sprintf $r, sprintf '-%d', $commit->deleted;
  53         }
  54
  55         return sprintf($c, '(') . $s . sprintf($c, ')');
  56 }
  57
  58 sub format_subject($$)
  59 {
  60         my ($subject, $body) = @_;
  61
  62         if (length($subject) > 80)
  63         {
  64                 $subject = substr($subject, 0, 77) . '...';
  65         }
  66
  67         $subject =~ s!^([^\s:]+):\s*!</nowiki>**<nowiki>$1:</nowiki>** <nowiki>!g;
  68
  69         $subject = sprintf '<nowiki>%s</nowiki>', $subject;
  70         $subject =~ s!<nowiki></nowiki>!!g;
  71
  72         return $subject;
  73 }
  74
  75 sub format_change($)
  76 {
  77         my ($change) = @_;
  78
  79         printf "''[[%s|%s]]'' %s //%s//\\\\\n",
  80                 sprintf($change->repository->commit_link_template, $change->sha1),
  81                 substr($change->sha1, 0, 7),
  82                 format_subject($change->subject, $change->body),
  83                 format_stat($change);
  84
  85         my @subhistory = $change->subhistory;
  86
  87         if (@subhistory > 0) {
  88                 my $n = 0;
  89                 my $link_tpl;
  90
  91                 foreach my $subchange (@subhistory) {
  92                         if ($n == 0) {
  93                                 $link_tpl = $subchange->repository->commit_link_template;
  94                         }
  95
  96                         if ($link_tpl) {
  97                                 printf " => ''[[%s|%s]]'' %s //%s//\\\\\n",
  98                                         sprintf($link_tpl, $subchange->sha1),
  99                                         substr($subchange->sha1, 0, 7),
 100                                         format_subject($subchange->subject, $subchange->body),
 101                                         format_stat($subchange);
 102                         }
 103                         else {
 104                                 printf " => ''%s'' %s //%s//\\\\\n",
 105                                         substr($subchange->sha1, 0, 7),
 106                                         format_subject($subchange->subject, $subchange->body),
 107                                         format_stat($subchange);
 108                         }
 109
 110                         if (++$n > 15 && @subhistory > $n) {
 111                                 printf " => + //%u more...//\\\\\n", @subhistory - $n;
 112                                 last;
 113                         }
 114                 }
 115         }
 116 }
 117
 118 sub fetch_cve_info()
 119 {
 120         unless (-f "$workdir/cveinfo.csv")
 121         {
 122                 system('wget', '-O', "$workdir/cveinfo.csv.gz", 'https://cve.mitre.org/data/downloads/allitems.csv.gz') && return 0;
 123                 system('gunzip', '-f', "$workdir/cveinfo.csv.gz") && return 0;
 124         }
 125
 126         return 1;
 127 }
 128
 129 sub parse_cves(@)
 130 {
 131         my $csv = Text::CSV->new({ binary => 1 });
 132         my %cves;
 133
 134         if (fetch_cve_info() && $csv)
 135         {
 136                 if (open CVE, '<', "$workdir/cveinfo.csv")
 137                 {
 138                         while (defined(my $row = $csv->getline(*CVE)))
 139                         {
 140                                 foreach my $cve_id (@_)
 141                                 {
 142                                         if ($row->[0] eq $cve_id)
 143                                         {
 144                                                 $cves{$cve_id} = [$row->[2], $row->[6]];
 145                                                 last;
 146                                         }
 147                                 }
 148                         }
 149
 150                         close CVE;
 151                 }
 152         }
 153
 154         return \%cves;
 155 }
 156
 157
 158 my $repository = Repository->new('https://git.openwrt.org/openwrt/openwrt.git');
 159 my $bugtracker = BugTracker->new;
 160
 161 my @commits = $repository->parse_history($range);
 162 my (%bugs, %cves, %sha1s);
 163
 164 foreach my $commit (@commits)
 165 {
 166         if ($commit->subject =~ m!\b(?:LEDE|OpenWrt) v\d\d\.\d\d\.\d+(?:-rc\d+)?: (?:adjust config|revert to branch) defaults\b!) {
 167                 Log::info("Skipping maintenance commit %s (%s)", $commit->sha1, $commit->subject);
 168                 next;
 169         }
 170
 171         my @topics = $commit->topics;
 172
 173         foreach my $topic (@topics)
 174         {
 175                 $topics{$topic} ||= [ ];
 176                 push @{$topics{$topic}}, $commit;
 177         }
 178
 179         foreach my $bug ($commit->bugs) {
 180                 if ($bug->status ne 'closed') {
 181                         Log::warn("Commit %s closes bug #%d", $commit->sha1, $bug->id);
 182                 }
 183
 184                 $bugs{ $bug->id } ||= [ ];
 185                 push @{$bugs{ $bug->id }}, $commit;
 186         }
 187
 188         foreach my $cve_id ($commit->cve_ids) {
 189                 $cves{$cve_id} ||= [ ];
 190                 push @{$cves{$cve_id}}, $commit;
 191         }
 192
 193         $sha1s{$commit->[1]}++;
 194 }
 195
 196 Log::info("Finding commit references in bugs...");
 197
 198 foreach my $bug ($bugtracker->bugs)
 199 {
 200         next if exists $bugs{ $bug->id };
 201
 202         foreach my $hash ($bug->refs) {
 203                 my $commit = $repository->find_commit($hash);
 204                 next unless defined $commit;
 205
 206                 if ($bug->status ne 'closed') {
 207                         Log::warn("Bug #%d closed by commit %s", $bug->id, $commit->sha1);
 208                 }
 209
 210                 $bugs{ $bug->id } ||= [ ];
 211                 push @{$bugs{ $bug->id }}, $commit;
 212         }
 213 }
 214
 215
 216 my @topics = sort { (($a eq 'Miscellaneous') <=> ($b eq 'Miscellaneous')) || $a cmp $b } keys %topics;
 217
 218 foreach my $topic (@topics)
 219 {
 220         my @commits = @{$topics{$topic}};
 221
 222         printf "==== %s (%d change%s) ====\n", $topic, 0 + @commits, @commits > 1 ? 's' : '';
 223
 224         foreach my $change (sort { $a->pos <=> $b->pos } @commits)
 225         {
 226                 format_change($change);
 227         }
 228
 229         print "\n";
 230 }
 231
 232 my @bugs = map { $bugtracker->get($_) } sort { int($a) <=> int($b) } keys %bugs;
 233
 234 if (@bugs > 0) {
 235         printf "===== Addressed bugs =====\n";
 236
 237         foreach my $bug (@bugs)
 238         {
 239                 if ($bug->fsid) {
 240                         printf "=== FS#%d (#%d) ===\n", $bug->fsid, $bug->id;
 241                 }
 242                 else {
 243                         printf "=== #%d ===\n", $bug->id;
 244                 }
 245
 246                 printf "**Description:** <nowiki>%s</nowiki>\\\\\n", $bug->summary;
 247                 printf "**Link:** [[https://github.com/openwrt/openwrt/issues/%d]]\\\\\n", $bug->id;
 248                 printf "**Commits:**\\\\\n";
 249
 250                 foreach my $commit (@{$bugs{ $bug->id }})
 251                 {
 252                         format_change($commit);
 253                 }
 254
 255                 printf "\\\\\n";
 256         }
 257
 258         printf "\n";
 259 }
 260
 261 my @cves =
 262         map { $_->[1] }
 263         sort { ($a->[0] <=> $b->[0]) || ($a->[1] cmp $b->[1]) }
 264         map { $_ =~ m!^CVE-(\d+)-(\d+)$! ? [ $1 * 10000000 + $2, $_ ] : [ 0, $_ ] }
 265         keys %cves;
 266
 267 my $cve_info = parse_cves(@cves);
 268
 269 if (@cves > 0)
 270 {
 271         printf "===== Security fixes ====\n";
 272
 273         foreach my $cve (@cves)
 274         {
 275                 printf "=== %s ===\n", $cve;
 276
 277                 if ($cve_info->{$cve} && $cve_info->{$cve}[0])
 278                 {
 279                         printf "**Description:** <nowiki>%s</nowiki>\n\n", $cve_info->{$cve}[0];
 280                 }
 281
 282                 printf "**Link:** [[https://cve.mitre.org/cgi-bin/cvename.cgi?name=%s]]\\\\\n", $cve;
 283                 printf "**Commits:**\\\\\n";
 284
 285                 foreach my $commit (@{$cves{$cve}})
 286                 {
 287                         format_change($commit);
 288                 }
 289
 290                 printf "\\\\\n";
 291         }
 292
 293         printf "\n";
 294 }
 295
 296
 297 package Log;
 298
 299 sub info {
 300         my ($fmt, @args) = @_;
 301         printf STDERR "[I] %s\n", sprintf $fmt, @args;
 302         return 0;
 303 }
 304
 305 sub warn {
 306         my ($fmt, @args) = @_;
 307         printf STDERR "[W] %s\n", sprintf $fmt, @args;
 308         return 1;
 309 }
 310
 311 sub err {
 312         my ($fmt, @args) = @_;
 313         printf STDERR "[E] %s\n", sprintf $fmt, @args;
 314         return 1;
 315 }
 316
 317
 318 package GitHubQuery;
 319
 320 sub _date {
 321         my ($self, $ts) = @_;
 322         my @loc = gmtime $ts;
 323         return sprintf '%04d-%02d-%02dT%02d:%02d:%02dZ',
 324                 $loc[5] + 1900, $loc[4] + 1, $loc[3],
 325                 $loc[2], $loc[1], $loc[0];
 326 }
 327
 328 sub _ts {
 329         my ($self, $date) = @_;
 330         return 0 unless $date;
 331
 332         my ($year, $mon, $mday, $hour, $min, $sec) = $date =~ m!^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})Z$!;
 333         return Time::Local::timegm_posix($sec, $min, $hour, $mday, $mon - 1, $year - 1900);
 334 }
 335
 336 sub _read_cache {
 337         my ($self, $path, $records) = @_;
 338
 339         if (open my $file, '<:utf8', $path) {
 340                 local $/;
 341
 342                 eval {
 343                         push @$records, @{ JSON::decode_json(readline $file) };
 344                 };
 345
 346                 close $file;
 347
 348                 if ($@) {
 349                         return Log::err("Unable to read $path: $@");
 350                 }
 351         }
 352
 353         return 0;
 354 }
 355
 356 sub _fetch_one_page {
 357         my ($self, $since, $page) = @_;
 358         my $url = $self->{'url'};
 359         my $sep = ($url =~ m!\?!) ? '&' : '?';
 360         my $res;
 361
 362         if ($since) {
 363                 $url .= $sep . 'since=' . $self->_date($since);
 364                 $sep = '&';
 365         }
 366
 367         if ($page) {
 368                 $url .= $sep . 'per_page=100&page=' . $page;
 369                 $sep = '&';
 370         }
 371
 372         if (open my $wget, '-|', 'wget', '--auth-no-challenge', '-q', '-O', '-', $url) {
 373                 local $/;
 374
 375                 eval {
 376                         $res = JSON::decode_json(readline $wget);
 377                 };
 378
 379                 if ($@) {
 380                         Log::err("Failed to parse result from $url: $@");
 381                 }
 382
 383                 close $wget;
 384         }
 385         else {
 386                 Log::err("Failed to fetch $url via wget: $!");
 387         }
 388
 389         return $res;
 390 }
 391
 392 sub _fetch {
 393         my ($self) = @_;
 394
 395         my $cache = "$main::workdir/" . $self->{'cachefile'};
 396         my @stat = stat $cache;
 397         my $since = defined($stat[9]) ? $stat[9] : $self->{'since'}; $since -= ($since % 86400);
 398         my @new_records;
 399         my @old_records;
 400         my $page = 1;
 401
 402         Log::info("Updating " . $self->{'cachefile'} . " database...");
 403
 404         while (1) {
 405                 my $res = $self->_fetch_one_page($since, $page);
 406
 407                 if (ref($res) ne 'ARRAY') {
 408                         return Log::err("Aborting update due to invalid response");
 409                 }
 410
 411                 push @new_records, @$res;
 412
 413                 Log::info("  Fetched " . @new_records . " records...");
 414
 415                 last if @$res < 100;
 416
 417                 $page++;
 418         }
 419
 420         if ($self->_read_cache($cache, \@old_records)) {
 421                 return 1;
 422         }
 423
 424         my $updated = 0;
 425         my %index;
 426
 427         foreach my $record (@old_records) {
 428                 if (ref($record) ne 'HASH' || !exists($record->{ $self->{'idprop'} })) {
 429                         next;
 430                 }
 431
 432                 $index{ $record->{ $self->{'idprop'} } } = $record;
 433         }
 434
 435         foreach my $record (@new_records) {
 436                 if (ref($record) ne 'HASH' || !exists($record->{ $self->{'idprop'} })) {
 437                         next;
 438                 }
 439
 440                 my $old = $index{ $record->{ $self->{'idprop'} } };
 441
 442                 if (!$old || $self->_ts($record->{'updated_at'}) != $self->_ts($old->{'updated_at'})) {
 443                         $index{ $record->{ $self->{'idprop'} } } = $record;
 444                         $updated++;
 445                 }
 446         }
 447
 448         if (!defined($stat[9]) || $updated) {
 449                 Log::info("  Found " . $updated . " updated records...");
 450
 451                 if (open my $file, '>:utf8', $cache) {
 452                         print $file JSON::encode_json([ values %index ]);
 453                         close $file;
 454                 }
 455                 else {
 456                         return Log::err("Unable to update $cache: $!");
 457                 }
 458
 459                 my $now = time();
 460
 461                 if (!utime($now, $now, $cache)) {
 462                         Log::warn("Unable to change $cache modification time: $!");
 463                 }
 464         }
 465
 466         return 0;
 467 }
 468
 469 sub fetch {
 470         my ($self, $force_update) = @_;
 471         my $cache = "$main::workdir/" . $self->{'cachefile'};
 472         my @records;
 473
 474         if ($force_update) {
 475                 $self->_fetch();
 476         }
 477
 478         if (-f $cache && $self->_read_cache($cache, \@records)) {
 479                 return undef;
 480         }
 481
 482         return wantarray ? @records : \@records;
 483 }
 484
 485 sub new {
 486         my ($pack, $baseurl, $cachefile, $idprop, $since) = @_;
 487
 488         return bless {
 489                 url       => $baseurl,
 490                 cachefile => $cachefile,
 491                 idprop    => $idprop,
 492                 since     => $since
 493         }, $pack;
 494 }
 495
 496
 497 package BugTracker;
 498
 499 our $inst;
 500
 501 sub _parse {
 502         my ($self) = @_;
 503
 504         return 0 if $self->{'bugs'};
 505
 506         my $issues = GitHubQuery->new(
 507                 "https://api.github.com/repos/openwrt/openwrt/issues?state=all&sort=updated&direction=desc",
 508                 "issues.json",
 509                 "number",
 510                 1640995200
 511         )->fetch(1);
 512
 513         return 1 unless $issues;
 514
 515         $self->{'bugs'} = { };
 516         $self->{'fsbugs'} = { };
 517
 518         foreach my $issue (@$issues) {
 519                 my ($date_opened, $date_closed, $date_modified) = (0, 0, 0);
 520
 521                 if (exists($issue->{'created_at'})) {
 522                         $date_opened = GitHubQuery->_ts($issue->{'created_at'});
 523                 }
 524
 525                 if (exists($issue->{'updated_at'})) {
 526                         $date_modified = GitHubQuery->_ts($issue->{'updated_at'});
 527                 }
 528
 529                 if (exists($issue->{'closed_at'})) {
 530                         $date_closed = GitHubQuery->_ts($issue->{'closed_at'});
 531                 }
 532
 533                 my $bug = Bug->new(
 534                         $issue->{'number'},
 535                         $issue->{'title'},
 536                         $issue->{'state'},
 537                         $date_opened,
 538                         $date_closed,
 539                         $date_modified
 540                 );
 541
 542                 $self->{'bugs'}{ $bug->id } = $bug;
 543
 544                 if ($issue->{'title'} =~ /^FS#(\d+) - /) {
 545                         $self->{'fsbugs'}{$1} = $bug;
 546                 }
 547         }
 548
 549         return 0;
 550 }
 551
 552 sub new {
 553         my ($pack) = @_;
 554
 555         unless ($inst) {
 556                 $inst = bless {}, $pack;
 557         }
 558
 559         return $inst;
 560 }
 561
 562 sub get($$) {
 563         my ($self, $id) = @_;
 564
 565         return undef if $self->_parse;
 566         return $self->{'bugs'}{$id};
 567 }
 568
 569 sub get_fs($$) {
 570         my ($self, $id) = @_;
 571
 572         return undef if $self->_parse;
 573         return $self->{'fsbugs'}{$id};
 574 }
 575
 576 sub bugs($) {
 577         my ($self) = @_;
 578         return undef if $self->_parse;
 579
 580         my @bugs = map { $self->{'bugs'}{$_} } sort { $a <=> $b } keys %{$self->{'bugs'}};
 581         return wantarray ? @bugs : \@bugs;
 582 }
 583
 584
 585 package Bug;
 586
 587 use File::Basename;
 588 use constant {
 589         '_ID'     => 0,
 590         '_SUM'    => 1,
 591         '_STAT'   => 2,
 592         '_OPEN'   => 3,
 593         '_CLOSE'  => 4,
 594         '_CHANGE' => 5,
 595         '_FSID'   => 6,
 596         '_REFS'   => 7
 597 };
 598
 599 sub new
 600 {
 601         my ($pack, $id, $summary, $status, $opened, $closed, $modified) = @_;
 602         my $fsid = undef;
 603
 604         if ($summary =~ s/^FS#(\d+) - //) {
 605                 $fsid = $1;
 606         }
 607
 608         return bless [
 609                 $id,
 610                 $summary,
 611                 $status,
 612                 $opened,
 613                 $closed,
 614                 $modified,
 615                 $fsid
 616         ], $pack;
 617 }
 618
 619 sub id { shift->[_ID] }
 620 sub fsid { shift->[_FSID] }
 621 sub url { sprintf 'https://api.github.com/repos/openwrt/openwrt/issues/%d/comments', shift->id }
 622 sub file { sprintf '%s/issue/%d.json', $main::workdir, shift->id }
 623 sub summary { shift->[_SUM] }
 624 sub status { shift->[_STAT] }
 625
 626 sub _fetch()
 627 {
 628         my ($self) = @_;
 629         my @stat = stat $self->file;
 630         my $refresh = 0;
 631
 632         if (!defined($stat[9]) || ($stat[9] < $self->[_CHANGE])) {
 633                 $refresh = 1;
 634         }
 635
 636         #Log::info("Fetching details for Bug #%d ...", $self->id);
 637
 638         if (system('mkdir', '-p', "$main::workdir/issue")) {
 639                 return Log::err("Unable to create directory!");
 640         }
 641
 642         my $comments = GitHubQuery->new(
 643                 $self->url,
 644                 sprintf('issue/%d.json', $self->id),
 645                 'id',
 646                 0
 647         )->fetch($refresh);
 648
 649         if (!$comments) {
 650                 Log::err("Unable to fetch bug details!");
 651
 652                 return undef;
 653         }
 654
 655         return wantarray ? @$comments : $comments;
 656 }
 657
 658 sub _find_commit_references()
 659 {
 660         my ($self) = @_;
 661         my $comments = $self->_fetch;
 662
 663         return undef unless $comments;
 664
 665         foreach my $comment (@$comments) {
 666                 my $str = $comment->{'body'};
 667                 my @refs = $str =~ m!
 668                         (?:
 669                                 Fixed \s+ with \s+ |
 670                                 Fixed \s+ in \s+ |
 671                                 Fixed \s+ by \s+ |
 672                                 fix \s+ (?: in | into ) \s+ (?: \w+ \s+ )*
 673                         )
 674                         (?: <a \s+ href=" )?  # "
 675                         \b (
 676                                 https?://git\.(?:openwrt|lede-project)\.org/\?p=[\w/]+\.git\S*;h=[a-fA-F0-9]{4,40} |
 677                                 https?://git\.(?:openwrt|lede-project)\.org/[a-fA-F0-9]{4,40} |
 678                                 https?://github\.com/[^/]+/commit/[a-fA-F0-9]{4,40} |
 679                                 [a-fA-F0-9]{7,40}
 680                         ) \b
 681                 !ixg;
 682
 683                 return @refs if @refs > 0;
 684         }
 685 }
 686
 687 sub refs ($) {
 688         my ($self) = @_;
 689
 690         unless (defined $self->[_REFS]) {
 691                 my %sha1;
 692
 693                 foreach my $ref ($self->_find_commit_references) {
 694                         if ($ref =~ m!\b([a-fA-F0-9]{4,40})$!) {
 695                                 $sha1{lc $1}++;
 696                         }
 697                 }
 698
 699                 $self->[_REFS] = [ sort keys %sha1 ];
 700         }
 701
 702         return wantarray ? @{$self->[_REFS]} : $self->[_REFS];
 703 }
 704
 705
 706 package Repository;
 707
 708 use File::Basename;
 709
 710 our %repositories;
 711 our %commits;
 712 our @index;
 713
 714 sub new($$) {
 715         my ($pack, $url) = @_;
 716
 717         my $id = $url;
 718            $id =~ s!\bgit\.lede-project\.org\b!git.openwrt.org!;
 719            $id =~ s![^a-z0-9_-]+!-!g;
 720
 721         unless (exists $repositories{$id}) {
 722                 $repositories{$id} = bless {
 723                         'id' => $id,
 724                         'url' => $url,
 725                         'cache' => { }
 726                 }, $pack;
 727
 728                 $repositories{$id}->_fetch;
 729         }
 730
 731         return $repositories{$id};
 732 }
 733
 734 sub id { shift->{'id'} }
 735 sub url { shift->{'url'} }
 736 sub directory { sprintf '%s/repos/%s', $main::workdir, shift->id }
 737
 738 sub _fetch($) {
 739         my ($self) = @_;
 740
 741         if (-d $self->directory) {
 742                 Log::info("Updating repository %s ...", $self->url);
 743
 744                 my $tree = $self->directory;
 745                 my $git  = $tree . '/.git';
 746
 747                 if (system('git', "--work-tree=$tree", "--git-dir=$git", 'fetch', '--all', '--quiet')) {
 748                         return Log::err("Unable to pull repository!");
 749                 }
 750
 751                 return 0;
 752         }
 753
 754         Log::info("Cloning repository %s ...", $self->url);
 755
 756         if (system('mkdir', '-p', $self->directory)) {
 757                 return Log::err("Unable to create directory!");
 758         }
 759         elsif (system('git', 'clone', '--quiet', $self->url, $self->directory)) {
 760                 return Log::err("Unable to clone repository!");
 761         }
 762
 763         return 0;
 764 }
 765
 766 sub _readline($*$) {
 767         my ($self, $fh, $default) = @_;
 768
 769         my $line = readline $fh;
 770
 771         if (defined $line)
 772         {
 773                 chomp $line;
 774                 return $line;
 775         }
 776
 777         return $default;
 778 }
 779
 780 sub _parse($*)
 781 {
 782         my ($self, $fh) = @_;
 783         my @commits;
 784         my $num = 0;
 785
 786         # skip header line
 787         $self->_readline($fh, undef);
 788
 789         while (1) {
 790                 my $hash = $self->_readline($fh, '');
 791                 my $subject = $self->_readline($fh, '');
 792
 793                 last unless (length($subject) && $hash =~ m!^[a-f0-9]{40}$!);
 794
 795                 my $line = '';
 796
 797                 # commit already cached, skip lines and use cached object
 798                 if (exists $Repository::commits{$hash}) {
 799                         for ($line = ''; $line ne '@@'; $line = $self->_readline($fh, '@@')) { next; }
 800                         for ($line = ''; $line ne '@@'; $line = $self->_readline($fh, '@@')) { next; }
 801
 802                         push @commits, $Repository::commits{$hash};
 803                         next;
 804                 }
 805
 806                 my $body = '';
 807                 my @files;
 808                 my ($add, $del) = (0, 0);
 809
 810                 while ($line ne '@@') {
 811                         $body .= length($line) ? "$line\n" : '';
 812                         $line = $self->_readline($fh, '@@');
 813                 }
 814
 815                 $line = '';
 816
 817                 my $reading_diff = 0;
 818                 my ($subhistory, $subhistory_url, $subhistory_start, $subhistory_end);
 819
 820                 while ($line ne '@@') {
 821                         if ($line =~ m!^diff --git a/!) {
 822                                 $reading_diff = 1;
 823                                 undef $subhistory_url;
 824                                 undef $subhistory_start;
 825                                 undef $subhistory_end;
 826                         }
 827                         elsif ($reading_diff) {
 828                                 if ($line =~ m!^[ +]PKG_SOURCE_URL\s*:?=\s*(\S+)!) {
 829                                         $subhistory_url = $1;
 830                                         $subhistory_url =~ s!\$\(LEDE_GIT\)!https://git.lede-project.org!g;
 831                                         $subhistory_url =~ s!\$\(OPENWRT_GIT\)!https://git.openwrt.org!g;
 832                                         $subhistory_url =~ s!\$\(PROJECT_GIT\)!https://git.openwrt.org!g;
 833                                 }
 834                                 elsif ($line =~ m!^-\S+\s*:?=\s*([a-f0-9]{40})\b!) {
 835                                         $subhistory_start = $1;
 836                                 }
 837                                 elsif ($line =~ m!^\+\S+\s*:?=\s*([a-f0-9]{40})\b!) {
 838                                         $subhistory_end = $1;
 839
 840                                         if ($subhistory_url && $subhistory_start && $subhistory_end) {
 841                                                 $subhistory = Repository->new($subhistory_url)->parse_history("$subhistory_start..$subhistory_end");
 842                                         }
 843                                 }
 844                         }
 845                         elsif ($line =~ m!^(\d+|-)\s+(\d+|-)\s+(.+)$!) {
 846                                 $add += ($1 eq '-') ? 0 : int($1);
 847                                 $del += ($2 eq '-') ? 0 : int($2);
 848                                 push @files, $3;
 849                         }
 850
 851                         $line = $self->_readline($fh, '@@');
 852                 }
 853
 854                 my $commit = Commit->new($self, $num++, $hash, $subject, $body, $add, $del, $subhistory, @files);
 855
 856                 push @commits, $commit;
 857                 push @Repository::index, $commit;
 858
 859                 $Repository::commits{ $commit->sha1 } = $commit;
 860         }
 861
 862         @Repository::index = sort { $a->sha1 cmp $b->sha1 } @Repository::index;
 863
 864         return wantarray ? @commits : \@commits;
 865 }
 866
 867 sub parse_history($$) {
 868         my ($self, $range) = @_;
 869         my $gitdir = sprintf '%s/.git', $self->directory;
 870         my @commits;
 871
 872         if (open my $git, '-|', 'git', "--git-dir=$gitdir", 'log', '-p', '--format=@@%n%H%n%s%n%b%n@@', '--numstat', '--reverse', '--no-merges', $range) {
 873                 @commits = $self->_parse($git);
 874                 close $git;
 875         }
 876
 877         return wantarray ? @commits : \@commits;
 878 }
 879
 880 sub find_commit($$) {
 881         my ($self, $hash) = @_;
 882
 883         if (exists $Repository::commits{$hash}) {
 884                 return $Repository::commits{$hash};
 885         }
 886         else {
 887                 my ($l, $r) = (0, @Repository::index - 1);
 888
 889                 while ($l <= $r) {
 890                         my $m = $l + int(($r - $l) / 2);
 891
 892                         if (index($Repository::index[$m]->sha1, $hash) == 0) {
 893                                 return $Repository::index[$m];
 894                         }
 895                         elsif ($Repository::index[$m]->sha1 gt $hash) {
 896                                 $r = $m - 1;
 897                         }
 898                         else {
 899                                 $l = $m + 1;
 900                         }
 901                 }
 902         }
 903
 904         return undef;
 905 }
 906
 907 sub _weblinks { (
 908         [ qr'^[^:]+://(git.lede-project.org/)(.+)$' => 'https://%s?p=%s;a=commitdiff;h=%%s' ],
 909         [ qr'^[^:]+://(git.openwrt.org/)(.+)$'      => 'https://%s?p=%s;a=commitdiff;h=%%s' ],
 910         [ qr'^[^:]+://(github.com/.+?)(?:\.git)?$'  => 'https://%s/commit/%%s' ],
 911         [ qr'^[^:]+://git.kernel.org/pub/scm/(.+)$' => 'https://git.kernel.org/cgit/%s/commit/?id=%%s' ],
 912         [ qr'^[^:]+://w1.fi/(?:.+/)?(.+)\.git$'     => 'https://w1.fi/cgit/%s/commit/?id=%%s' ],
 913         [ qr'^[^:]+://git.netfilter.org/(.+)'       => 'https://git.netfilter.org/%s/commit/?id=%%s' ],
 914         [ qr'^[^:]+://git.musl-libc.org/(.+)'       => 'https://git.musl-libc.org/cgit/%s/commit/?id=%%s' ],
 915         [ qr'^[^:]+://git.zx2c4.com/(.+)'           => 'https://git.zx2c4.com/%s/commit/?id=%%s' ],
 916         [ qr'^[^:]+://sourceware.org/git/(.+)'      => 'https://sourceware.org/git/?p=%s;a=commitdiff;h=%%s' ]
 917 ) }
 918
 919 sub commit_link_template($) {
 920         my ($self) = @_;
 921
 922         foreach my $lnk ($self->_weblinks) {
 923                 my @matches = $self->url =~ $lnk->[0];
 924                 if (@matches > 0) {
 925                         return sprintf $lnk->[1], @matches;
 926                 }
 927         }
 928
 929         Log::warn("No web link template available for %s", $self->url);
 930         return undef;
 931 }
 932
 933 sub log($) {
 934         my ($self) = @_;
 935         return wantarray ? @{$self->{'log'}} : $self->{'log'};
 936 }
 937
 938
 939 package Commit;
 940
 941 use constant {
 942         '_REPO'  => 0,
 943         '_POS'   => 1,
 944         '_SHA1'  => 2,
 945         '_SUBJ'  => 3,
 946         '_BODY'  => 4,
 947         '_FILES' => 5,
 948         '_SHIST' => 6,
 949         '_NADD'  => 7,
 950         '_NDEL'  => 8
 951 };
 952
 953 sub _topic_map { (
 954         [ qr'^package/(kernel)/linux',                 'Kernel' ],
 955         [ qr'^(target/linux/generic|include/kernel-version.mk)', 'Kernel' ],
 956         [ qr'^package/kernel/(mac80211)',              'Wireless / Common' ],
 957         [ qr'^package/kernel/(ath10k-ct)',             'Wireless / Ath10k CT' ],
 958         [ qr'^package/kernel/(mt76)',                  'Wireless / MT76' ],
 959         [ qr'^package/kernel/(mwlwifi)',               'Wireless / Mwlwifi' ],
 960         [ qr'^package/(base-files)/',                  'Packages / OpenWrt base files' ],
 961         [ qr'^package/(boot)/',                        'Packages / Boot Loaders' ],
 962         [ qr'^package/firmware/',                      'Packages / Firmware' ],
 963         [ qr'^package/.+/(uhttpd|usbmode|jsonfilter|ugps|libubox|procd|mountd|ubus|uci|usign|rpcd|fstools|ubox)/', 'Packages / OpenWrt system userland' ],
 964         [ qr'^package/.+/(iwinfo|umbim|uqmi|relayd|mdns|firewall|netifd|uclient|ustream-ssl|gre|ipip|qos-scripts|swconfig|vti|6in4|6rd|6to4|ds-lite|map|odhcp6c|odhcpd)/', 'Packages / OpenWrt network userland' ],
 965         [ qr'^package/[^/]+/([^/]+)',                  'Packages / Common' ],
 966         [ qr'^target/sdk/',                            'Build System / SDK' ],
 967         [ qr'^target/imagebuilder/',                   'Build System / Image Builder' ],
 968         [ qr'^target/toolchain/',                      'Build System / Toolchain' ],
 969         [ qr'^target/linux/([^/]+)',                   'Target / $1' ],
 970         [ qr'^(tools)/[^/]+',                          'Build System / Host Utilities' ],
 971         [ qr'^(toolchain)/[^/]+',                      'Build System / Toolchain' ],
 972         [ qr'^(config/|include/|scripts/|target/[^/]+$|Makefile|rules\.mk)', 'Build System / Buildroot' ],
 973         [ qr'^(feeds)\b',                              'Build System / Feeds' ],
 974 ) }
 975
 976 sub new ($$$$$$$$@) {
 977         my ($pack, $repo, $pos, $hash, $subject, $body, $add, $del, $shist, @files) = @_;
 978         my @commit;
 979
 980         $commit[_REPO] = $repo;
 981         $commit[_POS]  = $pos;
 982         $commit[_SHA1] = $hash;
 983         $commit[_SUBJ] = $subject;
 984         $commit[_BODY] = $body;
 985         $commit[_NADD] = $add;
 986         $commit[_NDEL] = $del;
 987         $commit[_SHIST] = $shist;
 988         $commit[_FILES] = \@files;
 989
 990         return bless \@commit, $pack;
 991 }
 992
 993 sub repository { shift->[_REPO] }
 994 sub pos { shift->[_POS] }
 995 sub sha1 { shift->[_SHA1] }
 996 sub subject { shift->[_SUBJ] }
 997 sub body { shift->[_BODY] }
 998 sub added { shift->[_NADD] }
 999 sub deleted { shift->[_NDEL] }
1000 sub files { wantarray ? @{shift->[_FILES] || []} : shift->[_FILES] }
1001 sub subhistory { wantarray ? @{shift->[_SHIST] || []} : shift->[_SHIST] }
1002
1003 sub topics($) {
1004         my ($self) = @_;
1005         my %topics;
1006         my %paths;
1007
1008         foreach my $path ($self->files)
1009         {
1010                 if ($path =~ m!^(.+)/\{(.+?) => (.+?)\}$!)
1011                 {
1012                         $paths{"$1/$2"}++;
1013                         $paths{"$1/$3"}++;
1014                 }
1015                 else
1016                 {
1017                         $paths{$path}++;
1018                 }
1019         }
1020
1021         foreach my $path (sort keys %paths)
1022         {
1023                 foreach my $rs ($self->_topic_map)
1024                 {
1025                         if ($path =~ $rs->[0])
1026                         {
1027                                 my $m = $1;
1028                                 my $s = $rs->[1];
1029
1030                                 $s =~ s!\$1!$m!g;
1031                                 $topics{$s}++;
1032
1033                                 last;
1034                         }
1035                 }
1036         }
1037
1038         my @topics = sort keys %topics;
1039         return (@topics > 0 ? @topics : ('Miscellaneous'));
1040 }
1041
1042 sub bugs($) {
1043         my ($self) = @_;
1044
1045         my $bugtracker = BugTracker->new;
1046         my $candidates = qr'\b((?:[Pp]ull [Rr]equest |[Bb]ug |[Ii]ssue |PR |FS |GH |PR|FS|GH)#\d+)\b';
1047         my %bugs;
1048
1049         foreach my $match ($self->subject =~ /$candidates/g, $self->body =~ /$candidates/g) {
1050                 my $bug;
1051
1052                 if ($match =~ /^FS ?#(\d+)$/) {
1053                         $bug = $bugtracker->get_fs($1);
1054                 }
1055                 elsif ($match =~ /^(GH|PR|[Pp]ull [Rr]equest) ?#(\d+)$/i) {
1056                         $bug = $bugtracker->get($1);
1057                 }
1058                 elsif ($match =~ /^#(\d+)$/) {
1059                         $bug = $bugtracker->get_fs($1) || $bugtracker->get($1);
1060                 }
1061
1062                 if ($bug) {
1063                         $bugs{ $bug->id } = $bug;
1064                 }
1065         }
1066
1067         foreach my $tag (qw(Fixes Closes Supersedes)) {
1068                 my ($ids) = $self->body =~ /\b$tag: *((?:GH|PR|FS|)#\d+(?:[, ]+#\d+)*)/;
1069
1070                 foreach my $id (split /[, ]+/, ($ids || '')) {
1071                         my $bug;
1072
1073                         if ($id =~ /^FS#(\d+)$/) {
1074                                 $bug = $bugtracker->get_fs($1);
1075                         }
1076                         elsif ($id =~ /^(GH|PR)#(\d+)$/) {
1077                                 $bug = $bugtracker->get($1);
1078                         }
1079                         elsif ($id =~ /^#(\d+)$/) {
1080                                 $bug = $bugtracker->get_fs($1) || $bugtracker->get($1);
1081                         }
1082
1083                         if ($bug) {
1084                                 $bugs{ $bug->id } = $bug;
1085                         }
1086                 }
1087         }
1088
1089         return map { $bugs{$_} } sort { $a <=> $b } keys %bugs;
1090 }
1091
1092 sub cve_ids($) {
1093         my ($self) = @_;
1094         my $candidates = qr'\b(CVE-\d+-\d+|\d+-CVE-\d+)\b';
1095         my %cves;
1096
1097         foreach my $match ($self->subject =~ /$candidates/g, $self->body =~ /$candidates/g) {
1098                 # fix misspelled CVE IDs
1099                 $match =~ s!^(\d+)-CVE-!CVE-$1-!;
1100                 $cves{$match}++;
1101         }
1102
1103         return sort { $a cmp $b } keys %cves;
1104 }