3 Tool to create overview.json files and update the config.js.
6 from pathlib
import Path
16 SUPPORTED_METADATA_VERSION
= 1
18 assert sys
.version_info
>= (3, 5), "Python version too old. Python >=3.5.0 needed."
21 # accepts {<file-path>: <file-content>}
22 def merge_profiles(profiles
, download_url
):
30 return "{} {} {}".format(
31 title
.get("vendor", ""), title
["model"], title
.get("variant", "")
34 def add_profile(id, target
, profile
, code
=None):
36 for image
in profile
["images"]:
37 images
.append({"name": image
["name"], "type": image
["type"]})
40 target
= profile
["target"]
42 for entry
in profile
["titles"]:
43 title
= get_title(entry
)
46 sys
.stderr
.write("Empty title. Skip title for {}".format(id))
49 output
["models"][title
] = {"id": id, "target": target
, "images": images
}
52 output
["models"][title
]["code"] = code
54 for path
, content
in profiles
.items():
55 obj
= json
.loads(content
)
57 if obj
["metadata_version"] != SUPPORTED_METADATA_VERSION
:
59 "{} has unsupported metadata version: {} => skip".format(
60 path
, obj
["metadata_version"]
65 code
= obj
.get("version_code", obj
.get("version_commit"))
67 if "version_code" not in output
:
68 output
= {"version_code": code
, "download_url": download_url
, "models": {}}
70 # if we have mixed codes/commits, store in device object
71 if output
["version_code"] == code
:
76 for id in obj
["profiles"]:
77 add_profile(id, obj
.get("target"), obj
["profiles"][id], code
)
79 add_profile(obj
["id"], obj
["target"], obj
, code
)
80 except json
.decoder
.JSONDecodeError
as e
:
81 sys
.stderr
.write("Skip {}\n {}\n".format(path
, e
))
83 sys
.stderr
.write("Abort on {}\n Missing key {}\n".format(path
, e
))
89 def update_config(config_path
, versions
):
91 with
open(str(config_path
), "r") as file:
94 content
= re
.sub("versions:[\\s]*{[^}]*}", "versions: {}".format(versions
), content
)
95 with
open(str(config_path
), "w+") as file:
100 Scrape profiles.json using links like https://downloads.openwrt.org/releases/19.07.3/targets/?json
101 Merge into overview.json files.
108 selector_path
= args
.selector
109 config_path
= "{}/config.js".format(selector_path
)
110 data_path
= "{}/data".format(selector_path
)
113 def handle_release(target
):
115 with urllib
.request
.urlopen("{}/?json".format(target
)) as file:
116 array
= json
.loads(file.read().decode("utf-8"))
117 for profile
in filter(lambda x
: x
.endswith("/profiles.json"), array
):
118 with urllib
.request
.urlopen("{}/{}".format(target
, profile
)) as file:
119 profiles
["{}/{}".format(target
, profile
)] = file.read().decode(
124 if not os
.path
.isfile(config_path
):
125 print("file not found: {}".format(config_path
))
129 with urllib
.request
.urlopen(url
) as infile
:
130 for path
in re
.findall(r
"href=[\"']?([^'\" >]+)", str(infile.read())):
131 if not path.startswith("/") and path.endswith("targets
/"):
132 release = path.strip("/").split("/")[-2]
133 download_url = "{}/{}/{{target}
}".format(url, path)
135 profiles = handle_release("{}/{}".format(url, path))
136 output = merge_profiles(profiles, download_url)
138 os.makedirs("{}/{}".format(data_path, release), exist_ok=True)
139 # write overview.json
141 "{}/{}/overview
.json
".format(data_path, release), "w
"
144 json.dump(output, outfile, indent=" ", sort_keys=True)
146 json.dump(output, outfile, sort_keys=True)
148 versions[release.upper()] = "data
/{}/overview
.json
".format(release)
150 update_config(config_path, versions)
154 Scrape profiles.json using wget (slower but more generic).
155 Merge into overview.json files.
160 def scrape_wget(args):
162 selector_path = args.selector
163 config_path = "{}/config
.js
".format(selector_path)
164 data_path = "{}/data
".format(selector_path)
167 with tempfile.TemporaryDirectory() as tmp_dir:
168 # download all profiles.json files
170 "wget
-c
-r
-P
{} -A
'profiles.json' --reject
-regex
'kmods|packages' --no
-parent
{}".format(
175 # delete empty folders
176 os.system("find
{}/* -type d
-empty
-delete
".format(tmp_dir))
178 # create overview.json files
179 for path in glob.glob("{}/*/snapshots
".format(tmp_dir)) + glob.glob(
180 "{}/*/releases
/*".format(tmp_dir)
182 release = os.path.basename(path)
183 base = path[len(tmp_dir) + 1 :]
186 for ppath in Path(path).rglob("profiles
.json
"):
187 with open(str(ppath), "r
") as file:
188 profiles[ppath] = file.read()
190 if len(profiles) == 0:
193 versions[release.upper()] = "data
/{}/overview
.json
".format(release)
195 output = merge_profiles(
196 profiles, "https
://{}/targets
/{{target}
}".format(base)
198 os.makedirs("{}/{}".format(data_path, release), exist_ok=True)
200 # write overview.json
201 with open("{}/{}/overview
.json
".format(data_path, release), "w
") as outfile:
203 json.dump(output, outfile, indent=" ", sort_keys=True)
205 json.dump(output, outfile, sort_keys=True)
207 update_config(config_path, versions)
211 Find and merge json files for a single release.
216 input_paths = args.input_path
217 # OpenWrt JSON device files
221 with open(str(path), "r
") as file:
222 profiles[path] = file.read()
224 for path in input_paths:
225 if os.path.isdir(path):
226 for filepath in Path(path).rglob("*.json
"):
229 if not path.endswith(".json
"):
230 sys.stderr.write("Folder does
not exists
: {}\n".format(path))
234 output = merge_profiles(profiles, args.download_url)
237 json.dump(output, sys.stdout, indent=" ", sort_keys=True)
239 json.dump(output, sys.stdout, sort_keys=True)
243 Scan local directory for releases with profiles.json.
244 Merge into overview.json files.
250 selector_path = args.selector
251 config_path = "{}/config
.js
".format(selector_path)
252 data_path = "{}/data
".format(selector_path)
255 # create overview.json files
256 for path in glob.glob("{}/snapshots
".format(args.directory)) + glob.glob(
257 "{}/releases
/*".format(args.directory)
259 release = os.path.basename(path)
260 base_dir = path[len(args.directory) + 1 :]
263 for ppath in Path(path).rglob("profiles
.json
"):
264 with open(str(ppath), "r
", encoding="utf
-8") as file:
265 profiles[ppath] = file.read()
267 if len(profiles) == 0:
270 versions[release.upper()] = "data
/{}/overview
.json
".format(release)
272 output = merge_profiles(
273 profiles, "https
://{}/{}/targets
/{{target}
}".format(args.domain, base_dir)
275 os.makedirs("{}/{}".format(data_path, release), exist_ok=True)
277 # write overview.json
278 with open("{}/{}/overview
.json
".format(data_path, release), "w
") as outfile:
280 json.dump(output, outfile, indent=" ", sort_keys=True)
282 json.dump(output, outfile, sort_keys=True)
284 update_config(config_path, versions)
288 parser = argparse.ArgumentParser()
290 "--formatted
", action="store_true
", help="Output formatted JSON data
."
292 subparsers = parser.add_subparsers(dest="action
")
293 subparsers.required = True
295 parser_merge = subparsers.add_parser(
297 help="Create a grid structure with horizontal
and vertical connections
.",
299 parser_merge.add_argument(
302 help="Input folder that
is traversed
for OpenWrt JSON device files
.",
304 parser_merge.add_argument(
308 help="Link to get the image
from. May contain {target}
, {version}
and {commit}
",
311 parser_scrape = subparsers.add_parser("scrape
", help="Scrape webpage
for releases
.")
312 parser_scrape.add_argument(
313 "domain
", help="Domain to scrape
. E
.g
. https
://downloads
.openwrt
.org
"
315 parser_scrape.add_argument("selector
", help="Path the config
.js
file is in.")
316 parser_scrape.add_argument(
317 "--use
-wget
", action="store_true
", help="Use wget to scrape the site
."
320 parser_scan = subparsers.add_parser("scan
", help="Scan directory
for releases
.")
321 parser_scan.add_argument(
323 help="Domain
for download_url attribute
in overview
.json
. E
.g
. https
://downloads
.openwrt
.org
",
325 parser_scan.add_argument("directory
", help="Directory to scan
for releases
.")
326 parser_scan.add_argument("selector
", help="Path the config
.js
file is in.")
328 args = parser.parse_args()
330 if args.action == "merge
":
333 if args.action == "scan
":
336 if args.action == "scrape
":
343 if __name__ == "__main__
":