#!/usr/bin/env python3
from pathlib import Path
+import urllib.request
import argparse
import json
import sys
import os
+import re
+
parser = argparse.ArgumentParser()
-parser.add_argument("input_path", nargs='?', help="Input folder that is traversed for OpenWrt JSON device files.")
-parser.add_argument('--url',
- action="store", dest="url", default="",
- help="Link to get the image from. May contain {target}, {release} and {commit}")
-parser.add_argument('--formatted',
- action="store_true", dest="formatted", help="Output formatted JSON data.")
+parser.add_argument("--formatted", action="store_true",
+ help="Output formatted JSON data.")
+subparsers = parser.add_subparsers(dest='action', required=True)
+
+parser_merge = subparsers.add_parser('merge',
+ help='Create a grid structure with horizontal and vertical connections.')
+parser_merge.add_argument("input_path", nargs="+",
+ help="Input folder that is traversed for OpenWrt JSON device files.")
+parser_merge.add_argument("--download-url", action="store", default="",
+ help="Link to get the image from. May contain {target}, {version} and {commit}")
+#parser_merge.add_argument("--change-prefix",
+# help="Change the openwrt- file name prefix.")
+
+parser_scrape = subparsers.add_parser('scrape',
+ help='Create a grid structure of horizontal, vertical and vertical connections.')
+parser_scrape.add_argument('domain',
+ help='Domain to scrape. E.g. https://downloads.openwrt.org')
+parser_scrape.add_argument('selector',
+ help='Path the config.js file is in.')
args = parser.parse_args()
SUPPORTED_METADATA_VERSION = 1
+# accepts {<file-path>: <file-content>}
+def merge_profiles(profiles, download_url):
+ # json output data
+ output = {}
+
+ def get_title_name(title):
+ if "title" in title:
+ return title["title"]
+ else:
+ return "{} {} {}".format(title.get("vendor", ""), title["model"], title.get("variant", "")).strip()
+
+ def add_profile(id, target, profile, code=None):
+ images = []
+ for image in profile["images"]:
+ images.append({"name": image["name"], "type": image["type"]})
+
+ if target is None:
+ target = profile["target"]
-# OpenWrt JSON device files
-paths = []
+ #if args.change_prefix:
+ # change_prefix(images, "openwrt-", args.change_prefix)
-if args.input_path:
- if not os.path.isdir(args.input_path):
- sys.stderr.write("Folder does not exists: {}\n".format(args.input_path))
- exit(1)
+ for title in profile["titles"]:
+ name = get_title_name(title)
- for path in Path(args.input_path).rglob('*.json'):
- paths.append(path)
+ if len(name) == 0:
+ sys.stderr.write(f"Empty title. Skip title in {path}\n")
+ continue
+
+ output["models"][name] = {"id": id, "target": target, "images": images}
-# json output data
-output = {}
-for path in paths:
- with open(path, "r") as file:
- try:
- obj = json.load(file)
+ if code is not None:
+ output["models"][name]["code"] = code
- if obj['metadata_version'] != SUPPORTED_METADATA_VERSION:
- sys.stderr.write('{} has unsupported metadata version: {} => skip\n'.format(path, obj['metadata_version']))
+ for path, content in profiles.items():
+ obj = json.loads(content)
+
+ if obj["metadata_version"] != SUPPORTED_METADATA_VERSION:
+ sys.stderr.write(f"{path} has unsupported metadata version: {obj['metadata_version']} => skip\n")
continue
- version = obj['version_number']
- commit = obj['version_commit']
+ code = obj.get("version_code", obj.get("version_commit"))
- if not 'version_commit' in output:
+ if not "version_code" in output:
output = {
- 'version_commit': commit,
- 'url': args.url,
- 'models' : {}
+ "version_code": code,
+ "download_url": download_url,
+ "models" : {}
}
- # only support a version_number with images of a single version_commit
- if output['version_commit'] != commit:
- sys.stderr.write('mixed revisions for a release ({} and {}) => abort\n'.format(output['version_commit'], commit))
- exit(1)
-
- images = []
- for image in obj['images']:
- images.append({'name': image['name'], 'type': image['type']})
+ # if we have mixed codes/commits, store in device object
+ if output["version_code"] == code:
+ code = None;
- target = obj['target']
- id = obj['id']
- for title in obj['titles']:
- if 'title' in title:
- name = title['title']
- output['models'][name] = {'id': id, 'target': target, 'images': images}
+ try:
+ if "profiles" in obj:
+ for id in obj["profiles"]:
+ add_profile(id, obj.get("target"), obj["profiles"][id], code)
else:
- name = "{} {} {}".format(title.get('vendor', ''), title['model'], title.get('variant', '')).strip()
- output['models'][name] = {'id': id, 'target': target, 'images': images}
-
- except json.decoder.JSONDecodeError as e:
- sys.stderr.write("Skip {}\n {}\n".format(path, e))
- continue
- except KeyError as e:
- sys.stderr.write("Abort on {}\n Missing key {}\n".format(path, e))
+ add_profile(obj["id"], obj["target"], obj, code)
+ except json.decoder.JSONDecodeError as e:
+ sys.stderr.write(f"Skip {path}\n {e}\n")
+ except KeyError as e:
+ sys.stderr.write(f"Abort on {path}\n Missing key {e}\n")
+ exit(1)
+
+ return output
+
+def scrape(url, selector_path):
+ config_path = f"{selector_path}/config.js"
+ data_path = f"{selector_path}/data"
+ versions = {}
+
+ def update_config(config_path, versions):
+ content = ''
+ with open(config_path, 'r') as file:
+ content = file.read()
+
+ content = re.sub('versions:[\\s]*{[^}]*}', f'versions: {versions}' , content)
+ with open(config_path, 'w+') as file:
+ # save updated config
+ file.write(content)
+
+ def handle_release(target):
+ profiles = {}
+ with urllib.request.urlopen(f"{target}/?json") as file:
+ array = json.loads(file.read().decode('utf-8'))
+ for profile in filter(lambda x: x.endswith('/profiles.json'), array):
+ #print(profile)
+ with urllib.request.urlopen(f"{target}/{profile}") as file:
+ profiles[f"{target}/{profile}"] = file.read()
+ return profiles
+
+ if not os.path.isfile(config_path):
+ print(f"file not found: {config_path}")
exit(1)
-if args.formatted:
- json.dump(output, sys.stdout, indent=" ", sort_keys = True)
-else:
- json.dump(output, sys.stdout, sort_keys = True)
+ # fetch release URLs
+ with urllib.request.urlopen(url) as infile:
+ for path in re.findall(r'href=["\']?([^\'" >]+)', str(infile.read())):
+ if not path.startswith('/') and path.endswith('targets/'):
+ release = path.strip('/').split('/')[-2]
+ download_url = f"{url}/{path}/{{target}}"
+
+ profiles = handle_release(f"{url}/{path}")
+ output = merge_profiles(profiles, download_url)
+ if len(output) > 0:
+ Path(f"{data_path}/{release}").mkdir(parents=True, exist_ok=True)
+ # write overview.json
+ with open(f"{data_path}/{release}/overview.json", 'w') as outfile:
+ if args.formatted:
+ json.dump(output, outfile, indent=" ", sort_keys=True)
+ else:
+ json.dump(output, outfile, sort_keys=True)
+
+ versions[release.upper()] = f"data/{release}/overview.json"
+
+ update_config(config_path, versions)
+
+'''
+def change_prefix(images, old_prefix, new_prefix):
+ for image in images:
+ if image["name"].startswith(old_prefix):
+ image["name"] = new_prefix + image["name"][len(old_prefix):]
+'''
+
+def merge(input_paths):
+ # OpenWrt JSON device files
+ profiles = {}
+
+ def add_path(path):
+ #paths.append(path)
+ with open(path, "r") as file:
+ profiles[path] = file.read()
+
+ for path in input_paths:
+ if os.path.isdir(path):
+ for filepath in Path(path).rglob("*.json"):
+ add_path(filepath)
+ else:
+ if not path.endswith(".json"):
+ sys.stderr.write(f"Folder does not exists: {path}\n")
+ exit(1)
+ add_path(path)
+
+ output = merge_profiles(profiles, args.download_url)
+
+ if args.formatted:
+ json.dump(output, sys.stdout, indent=" ", sort_keys=True)
+ else:
+ json.dump(output, sys.stdout, sort_keys=True)
+
+if args.action == "merge":
+ merge(args.input_path)
+
+if args.action == "scrape":
+ scrape(args.domain, args.selector)
projects / web / firmware-selector-openwrt-org.git / blobdiff