X-Git-Url: http://git.openwrt.org/?a=blobdiff_plain;f=scripts%2Fpackage-metadata.pl;h=261490b868160115e9e42a575b360f277f1db8d7;hb=HEAD;hp=d4100c1726d12c2675634edbf10a5186367c6a43;hpb=80d9ec5d3dbc1e393f377ede51e16d5c267475bd;p=openwrt%2Fstaging%2Fjow.git diff --git a/scripts/package-metadata.pl b/scripts/package-metadata.pl index d4100c1726..9e0e6dd9e5 100755 --- a/scripts/package-metadata.pl +++ b/scripts/package-metadata.pl @@ -4,6 +4,8 @@ use lib "$FindBin::Bin"; use strict; use metadata; use Getopt::Long; +use Time::Piece; +use JSON::PP; my %board; @@ -101,14 +103,16 @@ my %dep_check; sub __find_package_dep($$) { my $pkg = shift; my $name = shift; - my $deps = ($pkg->{vdepends} or $pkg->{depends}); + my $deps = $pkg->{depends}; return 0 unless defined $deps; - foreach my $dep (@{$deps}) { - next if $dep_check{$dep}; - $dep_check{$dep} = 1; - return 1 if $dep eq $name; - return 1 if ($package{$dep} and (__find_package_dep($package{$dep},$name) == 1)); + foreach my $vpkg (@{$deps}) { + foreach my $dep (@{$vpackage{$vpkg}}) { + next if $dep_check{$dep->{name}}; + $dep_check{$dep->{name}} = 1; + return 1 if $dep->{name} eq $name; + return 1 if (__find_package_dep($dep, $name) == 1); + } } return 0; } @@ -156,9 +160,10 @@ sub mconf_depends { my $m = "depends on"; my $flags = ""; $depend =~ s/^([@\+]+)// and $flags = $1; - my $vdep; my $condition = $parent_condition; + $depend = $2 if $depend =~ /^(.+):(.+)$/ and $dep->{$1} eq 'select'; + next if $condition eq $depend; next if $seen->{"$parent_condition:$depend"}; next if $seen->{":$depend"}; @@ -173,28 +178,26 @@ sub mconf_depends { } $depend = $2; } - next if $package{$depend} and $package{$depend}->{buildonly}; if ($flags =~ /\+/) { - if ($vdep = $package{$depend}->{vdepends}) { + my $vdep = $vpackage{$depend}; + if ($vdep) { my @vdeps; - $depend = undef; foreach my $v (@$vdep) { - if ($package{$v} && $package{$v}->{variant_default}) { - $depend = $v; + next if $v->{buildonly}; + if ($v->{variant_default}) { + unshift @vdeps, $v->{name}; } else { - push @vdeps, $v; + push @vdeps, $v->{name}; } } - if (!$depend) { - $depend = shift @vdeps; - } + $depend = shift @vdeps; if (@vdeps > 1) { - $condition = ($condition ? "$condition && " : '') . '!('.join("||", map { "PACKAGE_".$_ } @vdeps).')'; + $condition = ($condition ? "$condition && " : '') . join("&&", map { "PACKAGE_$_ 0) { - $condition = ($condition ? "$condition && " : '') . '!PACKAGE_'.$vdeps[0]; + $condition = ($condition ? "$condition && " : '') . "PACKAGE_${vdeps[0]}{vdepends}) { - $depend = join("||", map { "PACKAGE_".$_ } @$vdep); + my $vdep = $vpackage{$depend}; + if ($vdep && @$vdep > 0) { + $depend = join("||", map { "PACKAGE_".$_->{name} } @$vdep); } else { $flags =~ /@/ or $depend = "PACKAGE_$depend"; } @@ -232,7 +236,7 @@ sub mconf_depends { mconf_depends($pkgname, $tdep->[0], 1, $dep, $seen, $tdep->[1]); } - foreach my $depend (keys %$dep) { + foreach my $depend (sort keys %$dep) { my $m = $dep->{$depend}; $res .= "\t\t$m $depend\n"; } @@ -290,7 +294,7 @@ sub print_package_config_category($) { print "menu \"$menu\"\n"; } foreach my $pkg (@pkgs) { - next if $pkg->{ignore}; + next if $pkg->{src}{ignore}; my $title = $pkg->{name}; my $c = (72 - length($pkg->{name}) - length($pkg->{title})); if ($c > 0) { @@ -337,31 +341,6 @@ sub print_package_config_category($) { undef $category{$cat}; } -sub print_package_features() { - keys %features > 0 or return; - print "menu \"Package features\"\n"; - foreach my $n (keys %features) { - my @features = sort { $b->{priority} <=> $a->{priority} or $a->{title} cmp $b->{title} } @{$features{$n}}; - print <{target_title}" - default FEATURE_$features[0]->{name} -EOF - - foreach my $feature (@features) { - print <{name} - bool "$feature->{title}" -EOF - $feature->{description} =~ /\w/ and do { - print "\t\thelp\n".$feature->{description}."\n"; - }; - } - print "endchoice\n" - } - print "endmenu\n\n"; -} - sub print_package_overrides() { keys %overrides > 0 or return; print "\tconfig OVERRIDE_PKGS\n"; @@ -372,24 +351,10 @@ sub print_package_overrides() { sub gen_package_config() { parse_package_metadata($ARGV[0]) or exit 1; print "menuconfig IMAGEOPT\n\tbool \"Image configuration\"\n\tdefault n\n"; - foreach my $preconfig (keys %preconfig) { - foreach my $cfg (keys %{$preconfig{$preconfig}}) { - my $conf = $preconfig{$preconfig}->{$cfg}->{id}; - $conf =~ tr/\.-/__/; - print <{$cfg}->{label}" if IMAGEOPT - depends on PACKAGE_$preconfig - default "$preconfig{$preconfig}->{$cfg}->{default}" - -EOF - } - } print "source \"package/*/image-config.in\"\n"; if (scalar glob "package/feeds/*/*/image-config.in") { print "source \"package/feeds/*/*/image-config.in\"\n"; } - print_package_features(); print_package_config_category 'Base system'; foreach my $cat (sort {uc($a) cmp uc($b)} keys %category) { print_package_config_category $cat; @@ -397,14 +362,30 @@ EOF print_package_overrides(); } +sub and_condition($) { + my $condition = shift; + my @spl_and = split('\&\&', $condition); + if (@spl_and == 1) { + return "\$(CONFIG_$spl_and[0])"; + } + return "\$(and " . join (',', map("\$(CONFIG_$_)", @spl_and)) . ")"; +} + +sub gen_condition ($) { + my $condition = shift; + # remove '!()', just as include/package-ipkg.mk does + $condition =~ s/[()!]//g; + return join("", map(and_condition($_), split('\|\|', $condition))); +} + sub get_conditional_dep($$) { my $condition = shift; my $depstr = shift; if ($condition) { if ($condition =~ /^!(.+)/) { - return "\$(if \$(CONFIG_$1),,$depstr)"; + return "\$(if " . gen_condition($1) . ",,$depstr)"; } else { - return "\$(if \$(CONFIG_$condition),$depstr)"; + return "\$(if " . gen_condition($condition) . ",$depstr)"; } } else { return $depstr; @@ -412,175 +393,129 @@ sub get_conditional_dep($$) { } sub gen_package_mk() { - my %conf; - my %dep; - my %done; my $line; parse_package_metadata($ARGV[0]) or exit 1; - foreach my $name (sort {uc($a) cmp uc($b)} keys %package) { - my $config; - my $pkg = $package{$name}; - my @srcdeps; + foreach my $srcname (sort {uc($a) cmp uc($b)} keys %srcpackage) { + my $src = $srcpackage{$srcname}; + my $variant_default; + my %deplines = ('' => {}); + + foreach my $pkg (@{$src->{packages}}) { + foreach my $dep (@{$pkg->{depends}}) { + next if ($dep =~ /@/); + + my $condition; + + $dep =~ s/\+//g; + if ($dep =~ /^(.+):(.+)/) { + $condition = $1; + $dep = $2; + } + + my $vpkg_dep = $vpackage{$dep}; + unless (defined $vpkg_dep) { + warn sprintf "WARNING: Makefile '%s' has a dependency on '%s', which does not exist\n", + $src->{makefile}, $dep; + next; + } - next if defined $pkg->{vdepends}; + # Filter out self-depends + my @vdeps = grep { $srcname ne $_->{src}{name} } @{$vpkg_dep}; + + foreach my $vdep (@vdeps) { + my $depstr = sprintf '$(curdir)/%s/compile', $vdep->{src}{path}; + if (@vdeps > 1) { + $depstr = sprintf '$(if $(CONFIG_PACKAGE_%s),%s)', $vdep->{name}, $depstr; + } + my $depline = get_conditional_dep($condition, $depstr); + if ($depline) { + $deplines{''}{$depline}++; + } + } + } + + my $config = ''; + $config = sprintf '$(CONFIG_PACKAGE_%s)', $pkg->{name} unless $pkg->{buildonly}; + + $pkg->{prereq} and printf "prereq-%s += %s\n", $config, $src->{path}; + + next if $pkg->{buildonly}; + + printf "package-%s += %s\n", $config, $src->{path}; - $config = "\$(CONFIG_PACKAGE_$name)"; - if ($config) { - $pkg->{buildonly} and $config = ""; - print "package-$config += $pkg->{subdir}$pkg->{src}\n"; if ($pkg->{variant}) { - if (!defined($done{$pkg->{src}}) or $pkg->{variant_default}) { - print "\$(curdir)/$pkg->{subdir}$pkg->{src}/default-variant := $pkg->{variant}\n"; + if (!defined($variant_default) or $pkg->{variant_default}) { + $variant_default = $pkg->{variant}; } - print "\$(curdir)/$pkg->{subdir}$pkg->{src}/variants += \$(if $config,$pkg->{variant})\n" + printf "\$(curdir)/%s/variants += \$(if %s,%s)\n", $src->{path}, $config, $pkg->{variant}; } - $pkg->{prereq} and print "prereq-$config += $pkg->{subdir}$pkg->{src}\n"; } - next if $done{$pkg->{src}}; - $done{$pkg->{src}} = 1; + if (defined($variant_default)) { + printf "\$(curdir)/%s/default-variant := %s\n", $src->{path}, $variant_default; + } - if (@{$pkg->{buildtypes}} > 0) { - print "buildtypes-$pkg->{subdir}$pkg->{src} = ".join(' ', @{$pkg->{buildtypes}})."\n"; + unless (grep {!$_->{buildonly}} @{$src->{packages}}) { + printf "package- += %s\n", $src->{path}; } - foreach my $spkg (@{$srcpackage{$pkg->{src}}}) { - foreach my $dep (@{$spkg->{depends}}, @{$spkg->{builddepends}}) { - $dep =~ /@/ or do { - $dep =~ s/\+//g; - push @srcdeps, $dep; - }; - } + if (@{$src->{buildtypes}} > 0) { + printf "buildtypes-%s = %s\n", $src->{path}, join(' ', @{$src->{buildtypes}}); } - foreach my $type (@{$pkg->{buildtypes}}) { - my @extra_deps; - my %deplines; - next unless $pkg->{"builddepends/$type"}; - foreach my $dep (@{$pkg->{"builddepends/$type"}}) { - my $suffix = ""; + foreach my $type ('', @{$src->{buildtypes}}) { + my $suffix = ''; + + $suffix = "/$type" if $type; + + next unless $src->{"builddepends$suffix"}; + + defined $deplines{$suffix} or $deplines{$suffix} = {}; + + foreach my $dep (@{$src->{"builddepends$suffix"}}) { + my $depsuffix = ""; + my $deptype = ""; my $condition; if ($dep =~ /^(.+):(.+)/) { $condition = $1; $dep = $2; } - if ($dep =~ /^(.+)(\/.+)/) { + if ($dep =~ /^(.+)\/(.+)/) { $dep = $1; - $suffix = $2; + $deptype = $2; + $depsuffix = "/$2"; } - my $idx = ""; - my $pkg_dep = $package{$dep}; - if (defined($pkg_dep) && defined($pkg_dep->{src})) { - $idx = $pkg_dep->{subdir}.$pkg_dep->{src}; - } elsif (defined($srcpackage{$dep})) { - $idx = $subdir{$dep}.$dep; - } else { + next if $srcname.$suffix eq $dep.$depsuffix; + + my $src_dep = $srcpackage{$dep}; + unless (defined($src_dep) && (!$deptype || grep { $_ eq $deptype } @{$src_dep->{buildtypes}})) { + warn sprintf "WARNING: Makefile '%s' has a build dependency on '%s', which does not exist\n", + $src->{makefile}, $dep.$depsuffix; next; } - my $depstr = "\$(curdir)/$idx$suffix/compile"; + + my $depstr = sprintf '$(curdir)/%s/compile', $src_dep->{path}.$depsuffix; my $depline = get_conditional_dep($condition, $depstr); if ($depline) { - $deplines{$depline}++; + $deplines{$suffix}{$depline}++; } } - my $depline = join(" ", sort keys %deplines); - if ($depline) { - $line .= "\$(curdir)/".$pkg->{subdir}."$pkg->{src}/$type/compile += $depline\n"; - } } - my $hasdeps = 0; - my %deplines; - foreach my $deps (@srcdeps) { - my $idx; - my $condition; - my $prefix = ""; - my $suffix = ""; - - if ($deps =~ /^(.+):(.+)/) { - $condition = $1; - $deps = $2; - } - if ($deps =~ /^(.+)(\/.+)/) { - $deps = $1; - $suffix = $2; - } - - my $pkg_dep = $package{$deps}; - my @deps; - - if ($pkg_dep->{vdepends}) { - @deps = @{$pkg_dep->{vdepends}}; - } else { - @deps = ($deps); - } - - foreach my $dep (@deps) { - $pkg_dep = $package{$deps}; - if (defined $pkg_dep->{src}) { - ($pkg->{src} ne $pkg_dep->{src}.$suffix) and $idx = $pkg_dep->{subdir}.$pkg_dep->{src}; - } elsif (defined($srcpackage{$dep})) { - $idx = $subdir{$dep}.$dep; - } - undef $idx if $idx eq 'base-files'; - if ($idx) { - $idx .= $suffix; - - my $depline; - next if $pkg->{src} eq $pkg_dep->{src}.$suffix; - next if $dep{$condition.":".$pkg->{src}."->".$idx}; - next if $dep{$pkg->{src}."->($dep)".$idx} and $pkg_dep->{vdepends}; - my $depstr; - - if ($pkg_dep->{vdepends}) { - $depstr = "\$(if \$(CONFIG_PACKAGE_$dep),\$(curdir)/$idx/compile)"; - $dep{$pkg->{src}."->($dep)".$idx} = 1; - } else { - $depstr = "\$(curdir)/$idx/compile"; - $dep{$pkg->{src}."->".$idx} = 1; - } - $depline = get_conditional_dep($condition, $depstr); - if ($depline) { - $deplines{$depline}++; - } - } + foreach my $suffix (sort keys %deplines) { + my $depline = join(" ", sort keys %{$deplines{$suffix}}); + if ($depline) { + $line .= sprintf "\$(curdir)/%s/compile += %s\n", $src->{path}.$suffix, $depline; } } - my $depline = join(" ", sort keys %deplines); - if ($depline) { - $line .= "\$(curdir)/".$pkg->{subdir}."$pkg->{src}/compile += $depline\n"; - } } if ($line ne "") { print "\n$line"; } - foreach my $preconfig (keys %preconfig) { - my $cmds; - foreach my $cfg (keys %{$preconfig{$preconfig}}) { - my $conf = $preconfig{$preconfig}->{$cfg}->{id}; - $conf =~ tr/\.-/__/; - $cmds .= "\techo \"uci set '$preconfig{$preconfig}->{$cfg}->{id}=\$(subst \",,\$(CONFIG_UCI_PRECONFIG_$conf))'\"; \\\n"; - } - next unless $cmds; - print < \$@ - -ifneq (\$(IMAGEOPT)\$(CONFIG_IMAGEOPT),) - package/preconfig: \$(TARGET_DIR)/etc/uci-defaults/$preconfig -endif -endif - -EOF - } } sub gen_package_source() { @@ -594,13 +529,25 @@ sub gen_package_source() { } } -sub gen_package_subdirs() { +sub gen_package_auxiliary() { parse_package_metadata($ARGV[0]) or exit 1; foreach my $name (sort {uc($a) cmp uc($b)} keys %package) { my $pkg = $package{$name}; if ($pkg->{name} && $pkg->{repository}) { print "Package/$name/subdir = $pkg->{repository}\n"; } + my %depends; + foreach my $dep (@{$pkg->{depends} || []}) { + if ($dep =~ m!^\+?(?:[^:]+:)?([^@]+)$!) { + $depends{$1}++; + } + } + my @depends = sort keys %depends; + if (@depends > 0) { + foreach my $n (@{$pkg->{provides}}) { + print "Package/$n/depends = @depends\n"; + } + } } } @@ -619,7 +566,7 @@ sub gen_package_license($) { } else { if ($level == 1) { print "$pkg->{name}: Missing license! "; - print "Please fix $pkg->{makefile}\n"; + print "Please fix $pkg->{src}{makefile}\n"; } } } @@ -642,6 +589,208 @@ sub gen_usergroup_list() { } } +sub gen_package_manifest_json() { + my $json; + parse_package_metadata($ARGV[0]) or exit 1; + foreach my $name (sort {uc($a) cmp uc($b)} keys %package) { + my %depends; + my $pkg = $package{$name}; + foreach my $dep (@{$pkg->{depends} || []}) { + if ($dep =~ m!^\+?(?:[^:]+:)?([^@]+)$!) { + $depends{$1}++; + } + } + my @depends = sort keys %depends; + my $pkg_deps = join ' ', map { qq/"$_",/ } @depends; + $pkg_deps =~ s/\,$//; + + my $pkg_maintainer = join ' ', map { qq/"$_",/ } @{$pkg->{maintainer} || []}; + $pkg_maintainer =~ s/\,$//; + + $json = <<"END_JSON"; +${json}{ +"name":"$name", +"version":"$pkg->{version}", +"category":"$pkg->{category}", +"license":"$pkg->{license}", +"cpe_id":"$pkg->{cpe_id}", +"maintainer": [$pkg_maintainer], +"depends":[$pkg_deps]}, +END_JSON + } + + $json =~ s/[\n\r]//g; + $json =~ s/\,$//; + print "[$json]"; +} + +sub image_manifest_packages($) +{ + my %packages; + my $imgmanifest = shift; + + open FILE, "<$imgmanifest" or return; + while () { + /^(.+?) - (.+)$/ and $packages{$1} = $2; + } + close FILE; + + return %packages; +} + +sub dump_cyclonedxsbom_json { + my (@components) = @_; + + my $uuid = sprintf( + "%04x%04x-%04x-%04x-%04x-%04x%04x%04x", + rand(0xffff), rand(0xffff), rand(0xffff), + rand(0x0fff) | 0x4000, + rand(0x3fff) | 0x8000, + rand(0xffff), rand(0xffff), rand(0xffff) + ); + + my $cyclonedx = { + bomFormat => "CycloneDX", + specVersion => "1.4", + serialNumber => "urn:uuid:$uuid", + version => 1, + metadata => { + timestamp => gmtime->datetime, + }, + "components" => [@components], + }; + + return encode_json($cyclonedx); +} + +sub gen_image_cyclonedxsbom() { + my $pkginfo = shift @ARGV; + my $imgmanifest = shift @ARGV; + my @components; + my %image_packages; + + %image_packages = image_manifest_packages($imgmanifest); + %image_packages or exit 1; + parse_package_metadata($pkginfo) or exit 1; + + $package{"kernel"} = { + license => "GPL-2.0", + cpe_id => "cpe:/o:linux:linux_kernel", + name => "kernel", + }; + + my %abimap; + my @abipkgs = grep { defined $package{$_}->{abi_version} } keys %package; + foreach my $name (@abipkgs) { + my $pkg = $package{$name}; + my $abipkg = $name . $pkg->{abi_version}; + $abimap{$abipkg} = $name; + } + + foreach my $name (sort {uc($a) cmp uc($b)} keys %image_packages) { + my $pkg = $package{$name}; + if (!$pkg) { + $pkg = $package{$abimap{$name}}; + next if !$pkg; + } + + my @licenses; + my @license = split(/\s+/, $pkg->{license}); + foreach my $lic (@license) { + push @licenses, ( + { "license" => { "name" => $lic } } + ); + } + my $type; + if ($pkg->{category}) { + my $category = $pkg->{category}; + my %cat_type = ( + "Firmware" => "firmware", + "Libraries" => "library" + ); + + if ($cat_type{$category}) { + $type = $cat_type{$category}; + } else { + $type = "application"; + } + } + + my $version = $pkg->{version}; + if ($image_packages{$name}) { + $version = $image_packages{$name}; + } + $version =~ s/-\d+$// if $version; + if ($name =~ /^(kernel|kmod-)/ and $version =~ /^(\d+\.\d+\.\d+)/) { + $version = $1; + } + + push @components, { + name => $pkg->{name}, + version => $version, + @licenses > 0 ? (licenses => [ @licenses ]) : (), + $pkg->{cpe_id} ? (cpe => $pkg->{cpe_id}.":".$version) : (), + $type ? (type => $type) : (), + $version ? (version => $version) : (), + }; + } + + print dump_cyclonedxsbom_json(@components); +} + +sub gen_package_cyclonedxsbom() { + my $pkgmanifest = shift @ARGV; + my @components; + my %mpkgs; + + %mpkgs = parse_package_manifest_metadata($pkgmanifest); + %mpkgs or exit 1; + + foreach my $name (sort {uc($a) cmp uc($b)} keys %mpkgs) { + my $pkg = $mpkgs{$name}; + + my @licenses; + my @license = split(/\s+/, $pkg->{license}); + foreach my $lic (@license) { + push @licenses, ( + { "license" => { "name" => $lic } } + ); + } + + my $type; + if ($pkg->{section}) { + my $section = $pkg->{section}; + my %section_type = ( + "firmware" => "firmware", + "libs" => "library" + ); + + if ($section_type{$section}) { + $type = $section_type{$section}; + } else { + $type = "application"; + } + } + + my $version = $pkg->{version}; + $version =~ s/-\d+$// if $version; + if ($name =~ /^(kernel|kmod-)/ and $version =~ /^(\d+\.\d+\.\d+)/) { + $version = $1; + } + + push @components, { + name => $name, + version => $version, + @licenses > 0 ? (licenses => [ @licenses ]) : (), + $pkg->{cpe_id} ? (cpe => $pkg->{cpe_id}.":".$version) : (), + $type ? (type => $type) : (), + $version ? (version => $version) : (), + }; + } + + print dump_cyclonedxsbom_json(@components); +} + sub parse_command() { GetOptions("ignore=s", \@ignore); my $cmd = shift @ARGV; @@ -650,7 +799,10 @@ sub parse_command() { /^config$/ and return gen_package_config(); /^kconfig/ and return gen_kconfig_overrides(); /^source$/ and return gen_package_source(); - /^subdirs$/ and return gen_package_subdirs(); + /^pkgaux$/ and return gen_package_auxiliary(); + /^pkgmanifestjson$/ and return gen_package_manifest_json(); + /^imgcyclonedxsbom$/ and return gen_image_cyclonedxsbom(); + /^pkgcyclonedxsbom$/ and return gen_package_cyclonedxsbom(); /^license$/ and return gen_package_license(0); /^licensefull$/ and return gen_package_license(1); /^usergroup$/ and return gen_usergroup_list(); @@ -658,14 +810,17 @@ sub parse_command() { } die < [manifest] Image package manifest in CycloneDX SBOM JSON format + $0 pkgcyclonedxsbom Package manifest in CycloneDX SBOM JSON format + $0 license [file] Package license information $0 licensefull [file] Package license information (full list) - $0 usergroup [file] Package usergroup allocation list + $0 usergroup [file] Package usergroup allocation list $0 version_filter [patchver] [list...] Filter list of version tagged strings Options: