ruby: add auxiliar scripts in order to help tests

ruby_find_pkgsdeps: look for file dependencies (checks
require and Encoding references) and extrapolate it to pkgs
deps. Also checks whether a dep is redundant or missing in pkgs.
Must run inside an OpenWRT with all ruby* pkgs installed.

ruby_missingfiles: list files in staging/target and from files
comparing side by side its contents. It helps to easly visualize
which file is not packaged in an ipk.

Signed-off-by: Luiz Angelo Daros de Luca <luizluca@gmail.com>

diff --git a/lang/ruby/ruby_find_pkgsdeps b/lang/ruby/ruby_find_pkgsdeps
new file mode 100644 (file)
index 0000000..06587ec
--- /dev/null
+++ b/lang/ruby/ruby_find_pkgsdeps
@@ -0,0 +1,265 @@
+#!/usr/bin/ruby -Eutf-8
+# encoding: utf-8
+#
+# Find dependencies between ruby packages
+#
+# Must run inside a openwrt with all *ruby* packages installed
+#
+
+failed = false
+
+puts "Looking for installed ruby packages..."
+packages=`opkg list-installed '*ruby*' | cut -d' ' -f 1`.split("\n")
+
+puts "Looking for packages files..."
+package_files=Hash.new([])
+packages.each do
+       |pkg|
+       files=`opkg files "#{pkg}" | sed -e 1d`.split("\n")
+       package_files[pkg]=files if files
+end
+
+require_regex=/^require ["']([^"']+)["'].*/
+require_regex_ignore=/^require ([a-zA-Z\$]|["']$|.*\/$)/
+require_ignore=%w{drb/invokemethod16 foo rubygems/defaults/operating_system win32console java Win32API
+                  builder/xchar json/pure simplecov win32/sspi rdoc/markdown/literals_1_8 enumerator win32/resolv rbtree
+                  nqxml/streamingparser nqxml/treeparser xmlscan/parser xmlscan/scanner xmltreebuilder xml/parser xmlparser xml/encoding-ja xmlencoding-ja
+                  iconv uconv}
+
+builtin_enc=[
+       Encoding.find("ASCII-8BIT"),
+       Encoding.find("UTF-8"),
+       Encoding.find("UTF-7"),
+       Encoding.find("US-ASCII"),
+]
+
+puts "Looking for requires in files..."
+files_requires=Hash.new([])
+packages.each do
+        |pkg|
+       package_files[pkg].each do
+               |file|
+               next if not File.file?(file)
+
+               if not file =~ /.rb$/
+                       if File.executable?(file)
+                               magic=`head -c50 '#{file}' | head -1`
+                               begin
+                                       if not magic =~ /ruby/
+                                               next
+                                       end
+                               rescue
+                                       next
+                               end
+                       else
+                               next
+                       end
+               end
+               #puts "Checking #{file}..."
+               File.open(file, "r") do
+                       |f|
+                       lineno=0
+                       while line=f.gets() do
+                               lineno+=1; encs=[]; requires=[]; need_encdb=false
+
+                               line=line.chomp.gsub!(/^[[:blank:]]*/,"")
+
+                               case line
+                               when /^#.*coding *:/
+                                       if lineno <= 2
+                                               enc=line.sub(/.*coding *: */,"").sub(/ .*/,"")
+                                               encs << Encoding.find(enc)
+                                       end
+                               end
+                               line.gsub!(/#.*/,"")
+                               case line
+                               when "__END__"
+                                       break
+                               when /^require /
+                                       #puts "#{file}:#{line}"
+                                       if require_regex_ignore =~ line
+                                               #puts "Ignoring #{line} at #{file}:#{lineno} (REGEX)..."
+                                               next
+                                       end
+                                       if not require_regex =~ line
+                                               $stderr.puts "Unknown require: '#{line}' at file #{file}:#{lineno}"
+                                               failed=true
+                                       end
+                                       require=line.gsub(require_regex,"\\1")
+                                       require.gsub!(/\.(so|rb)$/,"")
+
+                                       if require_ignore.include?(require)
+                                               #puts "Ignoring #{line} at #{file}:#{lineno} (STR)..."
+                                               next
+                                       end
+
+                                       files_requires[file]=files_requires[file] + [require]
+
+                               when /Encoding::/
+                                       encs=line.scan(/Encoding::[[:alnum:]_]+/).collect {|enc| eval(enc) }.select {|enc| enc.kind_of? Encoding }
+                                       need_encdb=true
+                               end
+
+                               next if encs.empty?
+                               required_encs = (encs - builtin_enc).collect {|enc| "enc/#{enc.name.downcase.gsub("-","_")}" }
+                               required_encs << "enc/encdb" if need_encdb
+
+                               files_requires[file] = files_requires[file] + required_encs
+                       end
+               end
+       end
+end
+exit(1) if failed
+
+# Add deps from .so
+package_files.each do |(pkg,files)| files.each do |file|
+       case file
+       when /\/nkf\.so$/
+               files_requires[file]= files_requires[file] + ["enc/encdb"]
+       end
+end; end
+
+puts "Merging requirements into packages..."
+package_requires = Hash[packages.collect { |pkg| [pkg, package_files[pkg].collect {|file| files_requires[file] }.inject([],:+).uniq] }]
+
+weak_dependency=Hash.new([])
+weak_dependency.merge!({
+"ruby-misc"=>["ruby-openssl"],                                         #securerandom.rb
+"ruby-debuglib"=>["ruby-readline"],                            #debug.rb
+"ruby-drb"=>["ruby-openssl"],                                  #drb/ssl.rb
+"ruby-irb"=>["ruby-rdoc", "ruby-readline"],                    #irb/cmd/help.rb
+"ruby-gems"=>["ruby-openssl","ruby-io-console",                #rubygems/commands/cert_command.rb rubygems/user_interaction.rb
+             "ruby-minitest", "ruby-webrick"],                 #rubygems/test_case.rb rubygems/server.rb
+"ruby-mkmf"=>["ruby-webrick"],                                         #un.rb
+"ruby-net"=>["ruby-openssl","ruby-io-console","ruby-zlib"],    #net/*.rb
+"ruby-optparse"=>["ruby-uri","ruby-datetime"],                 #optparse/date.rb optparse/uri.rb
+"ruby-rake"=>["ruby-net","ruby-testunit","ruby-gems"],         #rake/contrib/ftptools.rb rake/runtest.rb /usr/bin/rake
+"ruby-rdoc"=>["ruby-gems","ruby-readline","ruby-webrick",
+             "ruby-minitest"],                                 #/usr/bin/rdoc and others
+"ruby-testunit"=>["ruby-gems", "ruby-io-console"],             #test/unit/parallel.rb test/unit.rb
+"ruby-webrick"=>["ruby-openssl"],                              #webrick/ssl.rb
+})
+
+puts "Looking for package dependencies..."
+package_provides = {}
+package_dependencies = Hash.new([])
+package_requires.each do
+       |(pkg,requires)|
+
+       requires.each do
+               |require|
+               if package_provides.include?(require)
+                       found = package_provides[require]
+               else
+                       found = package_files.detect {|(pkg,files)| files.detect {|file| $:.detect {|path| "#{path}/#{require}" == file.gsub(/\.(so|rb)$/,"") } } }
+                       if not found
+                               $stderr.puts "#{pkg}: Nobody provides #{require}"
+                               failed = true
+                               next
+                       end
+                       found = found.first
+                       package_provides[require]=found
+               end
+               if weak_dependency[pkg].include?(found)
+                       puts "#{pkg}: #{found} provides #{require} (ignored WEAK dep)"
+               else
+                       puts "#{pkg}: #{found} provides #{require}"
+                       package_dependencies[pkg]=package_dependencies[pkg] + [found]
+               end
+       end
+end
+exit(1) if failed
+
+package_dependencies.each do
+        |(pkg,deps)|
+        package_dependencies[pkg]=deps.uniq.sort - [pkg]
+end
+
+puts "Expanding dependencies..."
+begin
+       changed=false
+       package_dependencies.each do
+               |(pkg,deps)|
+
+               next if deps.empty?
+
+               deps_new = deps.collect {|dep| [dep] + package_dependencies[dep] }.inject([],:+).uniq.sort
+               if not deps == deps_new
+                       puts "#{pkg}: #{deps.join(",")}"
+                       puts "#{pkg}: #{deps_new.join(",")}"
+                       package_dependencies[pkg]=deps_new
+                       changed=true
+               end
+       end
+end if not changed
+
+puts "Checking for mutual dependencies..."
+package_dependencies.each do
+       |(pkg,deps)|
+       if deps.include? pkg
+               $stderr.puts "#{pkg}: Cycle dependency detected! "
+               failed = true
+       end
+end
+exit(1) if failed
+
+puts "Removing redundant dependencies..."
+package_dependencies.each do
+       |(pkg,deps)|
+       package_dependencies[pkg]=deps.uniq - [pkg]
+end
+
+package_dependencies2=package_dependencies.dup
+package_dependencies.each do
+       |(pkg,deps)|
+
+       # Ignore dependencies that are aready required by another dependency
+       deps_clean = deps.reject {|dep_suspect| deps.detect {|dep_provider|
+                       if package_dependencies[dep_provider].include?(dep_suspect)
+                               puts "#{pkg}: #{dep_suspect} is already required by #{dep_provider}"
+                               true
+                       end
+                } }
+
+       if not deps==deps_clean
+               puts "before: #{deps.join(",")}"
+               puts "after: #{deps_clean.join(",")}"
+               package_dependencies2[pkg]=deps_clean
+       end
+end
+package_dependencies=package_dependencies2
+
+puts "Checking current packages dependencies..."
+ok=true
+package_dependencies.each do
+       |(pkg,deps)|
+       current_deps=`opkg depends #{pkg} | sed -r -e '1d;s/^[[:blank:]]*//'`.split("\n")
+       current_deps.reject!{|dep| dep =~ /^lib/ }
+       current_deps -= ["ruby"]
+
+       extra_dep = current_deps - deps
+       $stderr.puts "Package #{pkg} does not need to depend on #{extra_dep.join(" ")} " if not extra_dep.empty?
+       missing_dep = deps - current_deps
+       $stderr.puts "Package #{pkg} need to depend on #{missing_dep.join(" ")} " if not missing_dep.empty?
+
+       if not extra_dep.empty? or not missing_dep.empty?
+               $stderr.puts "define Package/#{pkg}"
+               $stderr.puts "  DEPENDS:=ruby#{([""] +deps).join(" +")}"
+               ok=false
+       end
+end
+
+puts "All dependencies are OK." if ok
+
+
+__END__
+
+puts RUBY_VERSION, RUBY_PLATFORM
+puts 123
+
+puts Object.contants
+
+#RUBY_VER=2.1
+#RUBY_ARCH=i486-linux-gnu
+#RUBYLIB=/usr/lib/ruby/$RUBY_VER/
+#RUBYLIB_A=/usr/lib/ruby/$RUBY_ARCH/$RUBY_VER/

diff --git a/lang/ruby/ruby_missingfiles b/lang/ruby/ruby_missingfiles
new file mode 100644 (file)
index 0000000..f05e9fe
--- /dev/null
+++ b/lang/ruby/ruby_missingfiles
@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+
+function list_staging_files {
+       cd staging_dir/target-*/; find \
+               \( \( -name "root-x86" -or -name "packages" -or -name "stamp" -or -name "pkginfo" \) -prune \) -or -true \
+               \( -path "*ruby*" -or -name "erb" -or -name "gem" -or -name "irb" -or -name "rake" -or -name "rdoc" -or -name "ri" -or -name "testrb" \) \
+               -print | sort
+}
+
+function list_ipkg_files {
+       for OPKG in bin/*/packages/packages/*ruby*; do
+        tar --to-stdout -xzf "$OPKG" ./data.tar.gz | tar tz | sed -e 's%/$%%'
+       done | sort -u
+}
+
+
+echo "          Staging                    Packages"
+diff -y <(list_staging_files) <(list_ipkg_files)
+